tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it./workspace/data/public_pairs.json
1 {
2 "schema_version": "1.0",
3 "description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
4 "pairs": [
5 {
6 "pair_id": "P001",
7 "program_a": [
8 {"cmd": "param", "name": "w", "expr": "40"},
9 {"cmd": "param", "name": "h", "expr": "20"},
10 {"cmd": "param", "name": "d", "expr": "8"},
11 {"cmd": "sketch", "id": "s0", "plane": "XY"},
12 {"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
13 {"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
14 {"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
15 ],
16 "program_b": [
17 {"cmd": "param", "name": "depth", "expr": "4 + 4"},
18 {"cmd": "param", "name": "height", "expr": "5 * 4"},
19 {"cmd": "param", "name": "width", "expr": "20 * 2"},
20 {"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
21 {"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
22 {"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
23 ]
24 },
25 {
26 "pair_id": "P002",
27 "program_a": [
28 {"cmd": "param", "name": "r", "expr": "4"},
29 {"cmd": "sketch", "id": "s", "plane": "XY"},
30 {"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
31 {"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
32 ],
33 "program_b": [
34 {"cmd": "param", "name": "r2", "expr": "4.25"},
35 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
36 {"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
37 {"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
38 ]
39 },
40 {
41 "pair_id": "P003",
42 "program_a": [
43 {"cmd": "sketch", "id": "s", "plane": "XY"},
44 {"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
45 {"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
46 {"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
47 ],
48 "program_b": [
49 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
50 {"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
51 {"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
52 {"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
53 ]
54 }
55 ]
56 }
57
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a JSON file of program pairs and decides, for each pair, whether the two
compact CAD command programs describe the same canonical solid. The decision is
purely symbolic/structural: we normalize the aliases described in the task and
compare canonicalized field values.
"""
import sys
import json
import ast
import operator
import hashlib
# ---------------------------------------------------------------------------
# Command classification
# ---------------------------------------------------------------------------
# Annotation / metadata command kinds that never affect the solid.
METADATA_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
# Command kinds we interpret with defined semantics. Anything else that is not
# a construction command and not a metadata kind is an opaque solid-affecting
# payload.
PROFILE_KINDS = {"rect", "circle", "slot"}
DEFAULT_PARTITION = "\x00__default_partition__"
# ---------------------------------------------------------------------------
# Arithmetic expression evaluation
# ---------------------------------------------------------------------------
_BINOPS = {
ast.Add: operator.add,
ast.Sub: operator.sub,
ast.Mult: operator.mul,
ast.Div: operator.truediv,
}
def _eval_node(node, resolve):
if isinstance(node, ast.Expression):
return _eval_node(node.body, resolve)
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ValueError("non-numeric constant")
return float(node.value)
if isinstance(node, ast.Num): # pragma: no cover - legacy python
return float(node.n)
if isinstance(node, ast.BinOp) and type(node.op) in _BINOPS:
return _BINOPS[type(node.op)](
_eval_node(node.left, resolve), _eval_node(node.right, resolve)
)
if isinstance(node, ast.UnaryOp):
val = _eval_node(node.operand, resolve)
if isinstance(node.op, ast.USub):
return -val
if isinstance(node.op, ast.UAdd):
return +val
raise ValueError("bad unary op")
if isinstance(node, ast.Name):
return resolve(node.id)
raise ValueError("unsupported expression node: %r" % (node,))
def _eval_value(expr, resolve):
if isinstance(expr, bool):
raise ValueError("bool not numeric")
if isinstance(expr, (int, float)):
return float(expr)
if isinstance(expr, str):
tree = ast.parse(expr, mode="eval")
return _eval_node(tree, resolve)
raise ValueError("cannot evaluate %r" % (expr,))
class ParamResolver:
"""Lazily evaluates parameters, allowing params to reference other params."""
def __init__(self, defs):
self.defs = defs
self.cache = {}
self.active = set()
def resolve(self, name):
if name in self.cache:
return self.cache[name]
if name in self.active:
raise ValueError("cyclic parameter reference: %s" % name)
if name not in self.defs:
raise ValueError("unknown parameter: %s" % name)
self.active.add(name)
try:
val = _eval_value(self.defs[name], self.resolve)
finally:
self.active.discard(name)
self.cache[name] = val
return val
# ---------------------------------------------------------------------------
# Numeric canonicalization
# ---------------------------------------------------------------------------
def _fnum(x):
"""Round a number to the comparison tolerance and format deterministically."""
x = round(float(x) + 0.0, 6)
if x == 0:
x = 0.0
return format(x, ".6f")
def _leaf(value, resolve):
"""Canonicalize a scalar geometry value (number or arithmetic expression)."""
try:
return _fnum(_eval_value(value, resolve))
except Exception:
return "raw:" + str(value)
def _canon_val(value, resolve):
if isinstance(value, list):
return [_canon_val(v, resolve) for v in value]
return _leaf(value, resolve)
# ---------------------------------------------------------------------------
# Profile & extrude canonicalization
# ---------------------------------------------------------------------------
def _profile_sig(cmd, plane, resolve):
kind = cmd.get("cmd")
canon = {}
for key, val in cmd.items():
if key in ("cmd", "id", "sketch", "construction"):
continue
if kind == "slot" and key == "angle":
try:
ang = _eval_value(val, resolve) % 180.0
canon[key] = _fnum(ang)
except Exception:
canon[key] = "raw:" + str(val)
else:
canon[key] = _canon_val(val, resolve)
items = [[k, canon[k]] for k in sorted(canon.keys())]
plane_str = plane if plane else "\x00__noplane__"
return ["profile", kind, plane_str, items]
def _feature_sig(cmd, profile_table, resolve):
op = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = str(cmd.get("direction", "normal")).lower()
if "depth" in cmd:
depth = _leaf(cmd["depth"], resolve)
else:
depth = _fnum(0)
pids = []
if "profile" in cmd:
pids.append(cmd["profile"])
if "profiles" in cmd:
extend = cmd["profiles"]
if isinstance(extend, list):
pids.extend(extend)
else:
pids.append(extend)
prof_strings = []
for pid in pids:
if pid in profile_table:
prof_strings.append(json.dumps(profile_table[pid], sort_keys=True))
prof_strings.sort()
sig = ["extrude", op, extent, direction, depth, prof_strings]
if "body" in cmd:
label = cmd["body"]
elif "channel" in cmd:
label = cmd["channel"]
else:
label = None
return sig, label
# ---------------------------------------------------------------------------
# Program signature
# ---------------------------------------------------------------------------
def program_signature(program):
params = {}
sketches = {}
profile_cmds = []
extrude_cmds = []
opaque = []
if not isinstance(program, list):
program = []
for cmd in program:
if not isinstance(cmd, dict):
opaque.append(json.dumps(cmd, sort_keys=True))
continue
if cmd.get("construction") is True:
continue
kind = cmd.get("cmd")
if kind in METADATA_KINDS:
continue
if kind == "param":
params[cmd.get("name")] = cmd.get("expr")
elif kind == "sketch":
plane = cmd.get("plane")
sketches[cmd.get("id")] = str(plane).lower() if plane is not None else ""
elif kind in PROFILE_KINDS:
profile_cmds.append(cmd)
elif kind == "extrude":
extrude_cmds.append(cmd)
else:
payload = {k: v for k, v in cmd.items() if k != "id"}
opaque.append(json.dumps(payload, sort_keys=True))
resolver = ParamResolver(params).resolve
profile_table = {}
for cmd in profile_cmds:
plane = sketches.get(cmd.get("sketch"))
profile_table[cmd.get("id")] = _profile_sig(cmd, plane, resolver)
features = []
for cmd in extrude_cmds:
sig, label = _feature_sig(cmd, profile_table, resolver)
features.append((sig, label))
# Partition topology: group features by body/channel label; unlabeled
# features share a single default partition. Labels are renamable, so the
# canonical form is the multiset of groups (each group a multiset of feature
# signatures) with the label strings dropped.
groups = {}
for sig, label in features:
key = label if label is not None else DEFAULT_PARTITION
groups.setdefault(key, []).append(json.dumps(sig, sort_keys=True))
partition = sorted(json.dumps(sorted(g)) for g in groups.values())
opaque_sig = sorted(opaque)
return json.dumps({"partition": partition, "opaque": opaque_sig}, sort_keys=True)
def _short(sig):
return hashlib.md5(sig.encode("utf-8")).hexdigest()
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main(argv):
if len(argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
return 2
with open(argv[1], "r") as f:
data = json.load(f)
pairs = data.get("pairs", []) if isinstance(data, dict) else []
predictions = []
for pair in pairs:
pid = pair.get("pair_id")
sig_a = program_signature(pair.get("program_a", []))
sig_b = program_signature(pair.get("program_b", []))
predictions.append({
"pair_id": pid,
"equivalent": sig_a == sig_b,
"signature_a": _short(sig_a),
"signature_b": _short(sig_b),
})
with open(argv[2], "w") as f:
json.dump({"predictions": predictions}, f, indent=2)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
/workspace/solve.py
1 #!/usr/bin/env python3
2 """Weak starter baseline for DeepCAD canonical-equivalence.
3
4 This intentionally predicts every pair as non-equivalent. It is useful only as
5 a schema example and should score reward 0.0 on the hidden verifier.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import sys
12 from pathlib import Path
13
14
15 def main() -> int:
16 input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
17 output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
18
19 payload = json.loads(input_path.read_text(encoding="utf-8"))
20 predictions = [
21 {"pair_id": pair["pair_id"], "equivalent": False}
22 for pair in payload.get("pairs", [])
23 ]
24
25 output_path.parent.mkdir(parents=True, exist_ok=True)
26 output_path.write_text(
27 json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
28 encoding="utf-8",
29 )
30 return 0
31
32
33 if __name__ == "__main__":
34 raise SystemExit(main())
35
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a JSON file of program pairs and decides, for each pair, whether the two
compact CAD command programs describe the same canonical solid. The decision is
purely symbolic/structural: we normalize the aliases described in the task and
compare canonicalized field values.
"""
import sys
import json
import ast
import operator
import hashlib
# ---------------------------------------------------------------------------
# Command classification
# ---------------------------------------------------------------------------
# Annotation / metadata command kinds that never affect the solid.
METADATA_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
# Command kinds we interpret with defined semantics. Anything else that is not
# a construction command and not a metadata kind is an opaque solid-affecting
# payload.
PROFILE_KINDS = {"rect", "circle", "slot"}
DEFAULT_PARTITION = "\x00__default_partition__"
# ---------------------------------------------------------------------------
# Arithmetic expression evaluation
# ---------------------------------------------------------------------------
_BINOPS = {
ast.Add: operator.add,
ast.Sub: operator.sub,
ast.Mult: operator.mul,
ast.Div: operator.truediv,
}
def _eval_node(node, resolve):
if isinstance(node, ast.Expression):
return _eval_node(node.body, resolve)
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ValueError("non-numeric constant")
return float(node.value)
if isinstance(node, ast.Num): # pragma: no cover - legacy python
return float(node.n)
if isinstance(node, ast.BinOp) and type(node.op) in _BINOPS:
return _BINOPS[type(node.op)](
_eval_node(node.left, resolve), _eval_node(node.right, resolve)
)
if isinstance(node, ast.UnaryOp):
val = _eval_node(node.operand, resolve)
if isinstance(node.op, ast.USub):
return -val
if isinstance(node.op, ast.UAdd):
return +val
raise ValueError("bad unary op")
if isinstance(node, ast.Name):
return resolve(node.id)
raise ValueError("unsupported expression node: %r" % (node,))
def _eval_value(expr, resolve):
if isinstance(expr, bool):
raise ValueError("bool not numeric")
if isinstance(expr, (int, float)):
return float(expr)
if isinstance(expr, str):
tree = ast.parse(expr, mode="eval")
return _eval_node(tree, resolve)
raise ValueError("cannot evaluate %r" % (expr,))
class ParamResolver:
"""Lazily evaluates parameters, allowing params to reference other params."""
def __init__(self, defs):
self.defs = defs
self.cache = {}
self.active = set()
def resolve(self, name):
if name in self.cache:
return self.cache[name]
if name in self.active:
raise ValueError("cyclic parameter reference: %s" % name)
if name not in self.defs:
raise ValueError("unknown parameter: %s" % name)
self.active.add(name)
try:
val = _eval_value(self.defs[name], self.resolve)
finally:
self.active.discard(name)
self.cache[name] = val
return val
# ---------------------------------------------------------------------------
# Numeric canonicalization
# ---------------------------------------------------------------------------
def _fnum(x):
"""Round a number to the comparison tolerance and format deterministically."""
x = round(float(x) + 0.0, 6)
if x == 0:
x = 0.0
return format(x, ".6f")
def _leaf(value, resolve):
"""Canonicalize a scalar geometry value (number or arithmetic expression)."""
try:
return _fnum(_eval_value(value, resolve))
except Exception:
return "raw:" + str(value)
def _canon_val(value, resolve):
if isinstance(value, list):
return [_canon_val(v, resolve) for v in value]
return _leaf(value, resolve)
# ---------------------------------------------------------------------------
# Profile & extrude canonicalization
# ---------------------------------------------------------------------------
def _profile_sig(cmd, plane, resolve):
kind = cmd.get("cmd")
canon = {}
for key, val in cmd.items():
if key in ("cmd", "id", "sketch", "construction"):
continue
if kind == "slot" and key == "angle":
try:
ang = _eval_value(val, resolve) % 180.0
canon[key] = _fnum(ang)
except Exception:
canon[key] = "raw:" + str(val)
else:
canon[key] = _canon_val(val, resolve)
items = [[k, canon[k]] for k in sorted(canon.keys())]
plane_str = plane if plane else "\x00__noplane__"
return ["profile", kind, plane_str, items]
def _feature_sig(cmd, profile_table, resolve):
op = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = str(cmd.get("direction", "normal")).lower()
if "depth" in cmd:
depth = _leaf(cmd["depth"], resolve)
else:
depth = _fnum(0)
pids = []
if "profile" in cmd:
pids.append(cmd["profile"])
if "profiles" in cmd:
extra = cmd["profiles"]
if isinstance(extra, list):
pids.extend(extra)
else:
pids.append(extra)
prof_strings = []
for pid in pids:
if pid in profile_table:
prof_strings.append(json.dumps(profile_table[pid], sort_keys=True))
prof_strings.sort()
sig = ["extrude", op, extent, direction, depth, prof_strings]
if "body" in cmd:
label = cmd["body"]
elif "channel" in cmd:
label = cmd["channel"]
else:
label = None
return sig, label
# ---------------------------------------------------------------------------
# Program signature
# ---------------------------------------------------------------------------
def program_signature(program):
params = {}
sketches = {}
profile_cmds = []
extrude_cmds = []
opaque = []
if not isinstance(program, list):
program = []
for cmd in program:
if not isinstance(cmd, dict):
opaque.append(json.dumps(cmd, sort_keys=True))
continue
if cmd.get("construction") is True:
continue
kind = cmd.get("cmd")
if kind in METADATA_KINDS:
continue
if kind == "param":
params[cmd.get("name")] = cmd.get("expr")
elif kind == "sketch":
plane = cmd.get("plane")
sketches[cmd.get("id")] = str(plane).lower() if plane is not None else ""
elif kind in PROFILE_KINDS:
profile_cmds.append(cmd)
elif kind == "extrude":
extrude_cmds.append(cmd)
else:
payload = {k: v for k, v in cmd.items() if k != "id"}
opaque.append(json.dumps(payload, sort_keys=True))
resolver = ParamResolver(params).resolve
profile_table = {}
for cmd in profile_cmds:
plane = sketches.get(cmd.get("sketch"))
profile_table[cmd.get("id")] = _profile_sig(cmd, plane, resolver)
features = []
for cmd in extrude_cmds:
sig, label = _feature_sig(cmd, profile_table, resolver)
features.append((sig, label))
# Partition topology: group features by body/channel label; unlabeled
# features share a single default partition. Labels are renamable, so the
# canonical form is the multiset of groups (each group a multiset of feature
# signatures) with the label strings dropped.
groups = {}
for sig, label in features:
key = label if label is not None else DEFAULT_PARTITION
groups.setdefault(key, []).append(json.dumps(sig, sort_keys=True))
partition = sorted(json.dumps(sorted(g)) for g in groups.values())
opaque_sig = sorted(opaque)
return json.dumps({"partition": partition, "opaque": opaque_sig}, sort_keys=True)
def _short(sig):
return hashlib.md5(sig.encode("utf-8")).hexdigest()
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main(argv):
if len(argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
return 2
with open(argv[1], "r") as f:
data = json.load(f)
pairs = data.get("pairs", []) if isinstance(data, dict) else []
predictions = []
for pair in pairs:
pid = pair.get("pair_id")
sig_a = program_signature(pair.get("program_a", []))
sig_b = program_signature(pair.get("program_b", []))
predictions.append({
"pair_id": pid,
"equivalent": sig_a == sig_b,
"signature_a": _short(sig_a),
"signature_b": _short(sig_b),
})
with open(argv[2], "w") as f:
json.dump({"predictions": predictions}, f, indent=2)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
python3 solve.py data/public_pairs.json predictions.json && cat predictions.json
{
"predictions": [
{
"pair_id": "P001",
"equivalent": true,
"signature_a": "1820ab1f30e27d89258e393171c7d74e",
"signature_b": "1820ab1f30e27d89258e393171c7d74e"
},
{
"pair_id": "P002",
"equivalent": false,
"signature_a": "bc79d9147a935a321062a7ef750c5c1f",
"signature_b": "929d48cdd2c74ec372b9c2468a7f0cce"
},
{
"pair_id": "P003",
"equivalent": true,
"signature_a": "09d0e8fb3b98a50cf2b1762666481a9f",
"signature_b": "09d0e8fb3b98a50cf2b1762666481a9f"
}
]
}
[stdout]
{
"predictions": [
{
"pair_id": "P001",
"equivalent": true,
"signature_a": "1820ab1f30e27d89258e393171c7d74e",
"signature_b": "1820ab1f30e27d89258e393171c7d74e"
},
{
"pair_id": "P002",
"equivalent": false,
"signature_a": "bc79d9147a935a321062a7ef750c5c1f",
"signature_b": "929d48cdd2c74ec372b9c2468a7f0cce"
},
{
"pair_id": "P003",
"equivalent": true,
"signature_a": "09d0e8fb3b98a50cf2b1762666481a9f",
"signature_b": "09d0e8fb3b98a50cf2b1762666481a9f"
}
]
}cat > /tmp/t.json <<'EOF'
{"pairs":[
{"pair_id":"T1_body_vs_channel_group",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"q","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":3,"body":"B1"},
{"cmd":"extrude","id":"e2","profile":"q","depth":3,"body":"B1"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"q","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"q","depth":3,"channel":"C9"},
{"cmd":"extrude","id":"e2","profile":"r","depth":3,"channel":"C9"}]},
{"pair_id":"T2_default_vs_split",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"q","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":3},
{"cmd":"extrude","id":"e2","profile":"q","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"q","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":3,"body":"A"},
{"cmd":"extrude","id":"e2","profile":"q","depth":3,"body":"B"}]},
{"pair_id":"T3_slot_angle_mod180",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":30},
{"cmd":"extrude","id":"e","profile":"sl","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":210},
{"cmd":"extrude","id":"e","profile":"sl","depth":3}]},
{"pair_id":"T4_neg_depth_not_direction",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":-3,"direction":"normal"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":3,"direction":"reversed"}]},
{"pair_id":"T5_opaque_multiset",
"program_a":[{"cmd":"fillet","id":"f1","edges":["e1","e2"],"radius":"2"},
{"cmd":"chamfer","id":"c1","target":"x"}],
"program_b":[{"cmd":"chamfer","id":"zzz","target":"x"},
{"cmd":"fillet","id":"other","edges":["e1","e2"],"radius":"2"}]},
{"pair_id":"T6_rect_ordered",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":3}]}
]}
EOF
python3 solve.py /tmp/t.json /tmp/o.json && python3 -c "import json;print('\n'.join(f'{p[\"pair_id\"]}: {p[\"equivalent\"]}' for p in json.load(open('/tmp/o.json'))['predictions']))"T1_body_vs_channel_group: True T2_default_vs_split: False T3_slot_angle_mod180: True T4_neg_depth_not_direction: False T5_opaque_multiset: True T6_rect_ordered: False [stdout] T1_body_vs_channel_group: True T2_default_vs_split: False T3_slot_angle_mod180: True T4_neg_depth_not_direction: False T5_opaque_multiset: True T6_rect_ordered: False
No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_f16c8d7e693e46d1. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_f16c8d7e693e46d1 · verifier authoritative; classifier explanatory.