tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it./workspace/data/public_pairs.json
1 {
2 "schema_version": "1.0",
3 "description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
4 "pairs": [
5 {
6 "pair_id": "P001",
7 "program_a": [
8 {"cmd": "param", "name": "w", "expr": "40"},
9 {"cmd": "param", "name": "h", "expr": "20"},
10 {"cmd": "param", "name": "d", "expr": "8"},
11 {"cmd": "sketch", "id": "s0", "plane": "XY"},
12 {"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
13 {"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
14 {"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
15 ],
16 "program_b": [
17 {"cmd": "param", "name": "depth", "expr": "4 + 4"},
18 {"cmd": "param", "name": "height", "expr": "5 * 4"},
19 {"cmd": "param", "name": "width", "expr": "20 * 2"},
20 {"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
21 {"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
22 {"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
23 ]
24 },
25 {
26 "pair_id": "P002",
27 "program_a": [
28 {"cmd": "param", "name": "r", "expr": "4"},
29 {"cmd": "sketch", "id": "s", "plane": "XY"},
30 {"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
31 {"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
32 ],
33 "program_b": [
34 {"cmd": "param", "name": "r2", "expr": "4.25"},
35 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
36 {"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
37 {"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
38 ]
39 },
40 {
41 "pair_id": "P003",
42 "program_a": [
43 {"cmd": "sketch", "id": "s", "plane": "XY"},
44 {"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
45 {"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
46 {"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
47 ],
48 "program_b": [
49 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
50 {"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
51 {"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
52 {"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
53 ]
54 }
55 ]
56 }
57
List workspace and check python
ls -la /workspace/ /workspace/data/ 2>/dev/null; echo "---python---"; python3 --version
/workspace/: total 24 drwxr-xr-x 1 root root 4096 Jul 6 02:01 . drwxr-xr-x 1 root root 4096 Jul 6 02:20 .. -rw-r--r-- 1 root root 299 Jul 4 05:02 README.md drwxr-xr-x 2 root root 4096 Jul 4 05:02 data drwxr-xr-x 2 root root 4096 Jul 6 02:01 output -rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py /workspace/data/: total 12 drwxr-xr-x 2 root root 4096 Jul 4 05:02 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json ---python--- Python 3.11.15 [stdout] /workspace/: total 24 drwxr-xr-x 1 root root 4096 Jul 6 02:01 . drwxr-xr-x 1 root root 4096 Jul 6 02:20 .. -rw-r--r-- 1 root root 299 Jul 4 05:02 README.md drwxr-xr-x 2 root root 4096 Jul 4 05:02 data drwxr-xr-x 2 root root 4096 Jul 6 02:01 output -rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py /workspace/data/: total 12 drwxr-xr-x 2 root root 4096 Jul 4 05:02 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json ---python--- Python 3.11.15
/workspace/solve.py
1 #!/usr/bin/env python3
2 """Weak starter baseline for DeepCAD canonical-equivalence.
3
4 This intentionally predicts every pair as non-equivalent. It is useful only as
5 a schema example and should score reward 0.0 on the hidden verifier.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import sys
12 from pathlib import Path
13
14
15 def main() -> int:
16 input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
17 output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
18
19 payload = json.loads(input_path.read_text(encoding="utf-8"))
20 predictions = [
21 {"pair_id": pair["pair_id"], "equivalent": False}
22 for pair in payload.get("pairs", [])
23 ]
24
25 output_path.parent.mkdir(parents=True, exist_ok=True)
26 output_path.write_text(
27 json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
28 encoding="utf-8",
29 )
30 return 0
31
32
33 if __name__ == "__main__":
34 raise SystemExit(main())
35
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a pairs JSON, decides for each pair whether program_a and program_b
describe the same canonical solid, and writes predictions.json.
Approach: build a normalized, label-invariant signature for each program and
compare the two signatures for equality.
"""
from __future__ import annotations
import ast
import json
import sys
from pathlib import Path
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
META_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
PROFILE_KINDS = {"rect", "circle", "slot"}
PROFILE_SKIP_FIELDS = {"id", "cmd", "sketch", "construction"}
TOL_DECIMALS = 6
# ---------------------------------------------------------------------------
# Expression evaluation
# ---------------------------------------------------------------------------
class Evaluator:
"""Evaluate scalar arithmetic expressions with parameter references."""
def __init__(self, params):
# params: name -> raw value (str expression or number)
self.params = params
self.cache = {}
self.inprogress = set()
def eval_name(self, name):
if name in self.cache:
return self.cache[name]
if name not in self.params:
raise KeyError(name)
if name in self.inprogress:
raise ValueError("parameter cycle: %s" % name)
self.inprogress.add(name)
try:
val = self.eval_value(self.params[name])
finally:
self.inprogress.discard(name)
self.cache[name] = val
return val
def eval_value(self, v):
if isinstance(v, bool):
raise ValueError("bool is not numeric")
if isinstance(v, (int, float)):
return float(v)
if isinstance(v, str):
tree = ast.parse(v, mode="eval")
return self.eval_node(tree.body)
raise ValueError("not a scalar expression: %r" % (v,))
def eval_node(self, node):
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ValueError("non-numeric constant")
return float(node.value)
if isinstance(node, ast.Num): # pragma: no cover (py<3.8)
return float(node.n)
if isinstance(node, ast.BinOp):
left = self.eval_node(node.left)
right = self.eval_node(node.right)
op = node.op
if isinstance(op, ast.Add):
return left + right
if isinstance(op, ast.Sub):
return left - right
if isinstance(op, ast.Mult):
return left * right
if isinstance(op, ast.Div):
return left / right
raise ValueError("unsupported binary operator")
if isinstance(node, ast.UnaryOp):
operand = self.eval_node(node.operand)
if isinstance(node.op, ast.USub):
return -operand
if isinstance(node.op, ast.UAdd):
return operand
raise ValueError("unsupported unary operator")
if isinstance(node, ast.Name):
return self.eval_name(node.id)
raise ValueError("unsupported expression node: %r" % (node,))
# ---------------------------------------------------------------------------
# Numeric helpers
# ---------------------------------------------------------------------------
def round_num(x):
if x is None:
return None
r = round(float(x), TOL_DECIMALS)
if r == 0:
r = 0.0
return r
def ser(obj):
"""Canonical JSON string of a jsonable structure (for multiset sorting)."""
return json.dumps(obj, sort_keys=True, ensure_ascii=False)
def canon_field(v, ev):
"""Canonicalize a geometric field value, evaluating numeric expressions."""
if isinstance(v, bool):
return v
if isinstance(v, (int, float)):
return round_num(v)
if isinstance(v, str):
try:
return round_num(ev.eval_value(v))
except Exception:
return v
if isinstance(v, list):
return [canon_field(x, ev) for x in v]
if isinstance(v, dict):
return {k: canon_field(x, ev) for k, x in v.items()}
return v
def normalize_plane(p):
if isinstance(p, str):
return p.upper()
return p
# ---------------------------------------------------------------------------
# Signature construction
# ---------------------------------------------------------------------------
def canon_direction(d, ev):
if isinstance(d, str):
return d.lower()
if isinstance(d, bool):
return d
if isinstance(d, (int, float)):
return round_num(d)
if isinstance(d, list):
return [canon_field(x, ev) for x in d]
return d
def canon_profile(p, sketches, ev):
kind = p.get("cmd")
plane = sketches.get(p.get("sketch"))
fields = {}
for k, v in p.items():
if k in PROFILE_SKIP_FIELDS:
continue
if k == "angle":
try:
a = ev.eval_value(v) % 180.0
fields[k] = round_num(a)
continue
except Exception:
pass
fields[k] = canon_field(v, ev)
return ["profile", kind, plane, fields]
def extrude_profile_ids(ex):
if "profiles" in ex and isinstance(ex["profiles"], list):
return list(ex["profiles"])
if "profile" in ex:
prof = ex["profile"]
if isinstance(prof, list):
return list(prof)
return [prof]
return []
def feature_sig(ex, profiles, sketches, ev):
op = str(ex.get("operation", "new")).lower()
extent = str(ex.get("extent", "one_side")).lower()
direction = canon_direction(ex.get("direction", "normal"), ev)
depth = None
if "depth" in ex:
try:
depth = round_num(ev.eval_value(ex["depth"]))
except Exception:
depth = canon_field(ex["depth"], ev)
ids = extrude_profile_ids(ex)
prof_forms = []
for pid in ids:
if pid in profiles:
prof_forms.append(ser(canon_profile(profiles[pid], sketches, ev)))
prof_forms.sort()
return [op, extent, direction, depth, prof_forms]
def partition_label(ex):
if "body" in ex:
return ("L", ex["body"])
if "channel" in ex:
return ("L", ex["channel"])
return None
def build_signature(program):
if not isinstance(program, list):
program = []
params = {}
sketches = {}
profiles = {}
extrudes = []
opaque = []
for cmd in program:
if not isinstance(cmd, dict):
opaque.append(cmd)
continue
if cmd.get("construction") is True:
continue
kind = cmd.get("cmd")
if kind in META_KINDS:
continue
if kind == "param":
params[cmd.get("name")] = cmd.get("expr")
elif kind == "sketch":
sketches[cmd.get("id")] = normalize_plane(cmd.get("plane"))
elif kind in PROFILE_KINDS:
profiles[cmd.get("id")] = cmd
elif kind == "extrude":
extrudes.append(cmd)
else:
payload = {k: v for k, v in cmd.items() if k != "id"}
opaque.append(payload)
ev = Evaluator(params)
default_group = []
label_groups = {}
for ex in extrudes:
feat = feature_sig(ex, profiles, sketches, ev)
lab = partition_label(ex)
if lab is None:
default_group.append(feat)
else:
label_groups.setdefault(ser(list(lab)), []).append(feat)
partitions = []
if default_group:
partitions.append(["DEFAULT", sorted(ser(f) for f in default_group)])
for _key, feats in label_groups.items():
partitions.append(["EXPLICIT", sorted(ser(f) for f in feats)])
partitions_sig = sorted(ser(p) for p in partitions)
opaque_sig = sorted(ser(o) for o in opaque)
return ser(["SIG", partitions_sig, opaque_sig])
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = []
for pair in payload.get("pairs", []):
pid = pair.get("pair_id")
try:
sig_a = build_signature(pair.get("program_a", []))
sig_b = build_signature(pair.get("program_b", []))
equivalent = sig_a == sig_b
except Exception:
sig_a = sig_b = None
equivalent = False
predictions.append(
{
"pair_id": pid,
"equivalent": bool(equivalent),
"signature_a": sig_a,
"signature_b": sig_b,
}
)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Run solver on public data
python3 solve.py data/public_pairs.json predictions.json && python3 -c "import json; d=json.load(open('predictions.json')); [print(p['pair_id'], p['equivalent']) for p in d['predictions']]"P001 True P002 False P003 True [stdout] P001 True P002 False P003 True
Test edge cases
cat > /tmp/t.json << 'EOF'
{"pairs":[
{"pair_id":"E01","program_a":[{"cmd":"sketch","id":"s","plane":"XZ"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"ZX"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3}]},
{"pair_id":"E02","program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,4]},{"cmd":"extrude","id":"e","profile":"r","depth":3}]},
{"pair_id":"E03","program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3,"direction":"normal"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":-3,"direction":"normal"}]},
{"pair_id":"E04_defVSlabel","program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"rect","id":"r2","sketch":"s","center":[1,1],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3},{"cmd":"extrude","id":"e2","profile":"r2","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"rect","id":"r2","sketch":"s","center":[1,1],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3,"body":"g"},{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"body":"g"}]},
{"pair_id":"E05_bodyVSchannel","program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"rect","id":"r2","sketch":"s","center":[1,1],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3,"body":"g"},{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"body":"g"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"rect","id":"r2","sketch":"s","center":[1,1],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3,"channel":"z"},{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"channel":"z"}]},
{"pair_id":"E06_slotangle","program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"r","sketch":"s","center":[0,0],"length":10,"width":2,"angle":10},{"cmd":"extrude","id":"e","profile":"r","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"r","sketch":"s","center":[0,0],"length":10,"width":2,"angle":190},{"cmd":"extrude","id":"e","profile":"r","depth":3}]},
{"pair_id":"E07_opaque","program_a":[{"cmd":"fillet","id":"f","edges":["a","b"],"radius":2}],
"program_b":[{"cmd":"fillet","id":"fZ","edges":["a","b"],"radius":2}]},
{"pair_id":"E08_opaque_diff","program_a":[{"cmd":"fillet","id":"f","edges":["a","b"],"radius":2}],
"program_b":[{"cmd":"fillet","id":"f","edges":["b","a"],"radius":2}]},
{"pair_id":"E09_op_distinct","program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3,"operation":"add"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,2]},{"cmd":"extrude","id":"e","profile":"r","depth":3,"operation":"new"}]}
]}
EOF
python3 solve.py /tmp/t.json /tmp/o.json && python3 -c "import json;[print(p['pair_id'],p['equivalent']) for p in json.load(open('/tmp/o.json'))['predictions']]"E01 False E02 False E03 False E04_defVSlabel False E05_bodyVSchannel True E06_slotangle True E07_opaque True E08_opaque_diff False E09_op_distinct False [stdout] E01 False E02 False E03 False E04_defVSlabel False E05_bodyVSchannel True E06_slotangle True E07_opaque True E08_opaque_diff False E09_op_distinct False
No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_3cbd36c1865f4028. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_3cbd36c1865f4028 · verifier authoritative; classifier explanatory.