tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it./workspace/data/public_pairs.json
1 {
2 "schema_version": "1.0",
3 "description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
4 "pairs": [
5 {
6 "pair_id": "P001",
7 "program_a": [
8 {"cmd": "param", "name": "w", "expr": "40"},
9 {"cmd": "param", "name": "h", "expr": "20"},
10 {"cmd": "param", "name": "d", "expr": "8"},
11 {"cmd": "sketch", "id": "s0", "plane": "XY"},
12 {"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
13 {"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
14 {"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
15 ],
16 "program_b": [
17 {"cmd": "param", "name": "depth", "expr": "4 + 4"},
18 {"cmd": "param", "name": "height", "expr": "5 * 4"},
19 {"cmd": "param", "name": "width", "expr": "20 * 2"},
20 {"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
21 {"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
22 {"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
23 ]
24 },
25 {
26 "pair_id": "P002",
27 "program_a": [
28 {"cmd": "param", "name": "r", "expr": "4"},
29 {"cmd": "sketch", "id": "s", "plane": "XY"},
30 {"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
31 {"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
32 ],
33 "program_b": [
34 {"cmd": "param", "name": "r2", "expr": "4.25"},
35 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
36 {"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
37 {"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
38 ]
39 },
40 {
41 "pair_id": "P003",
42 "program_a": [
43 {"cmd": "sketch", "id": "s", "plane": "XY"},
44 {"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
45 {"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
46 {"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
47 ],
48 "program_b": [
49 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
50 {"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
51 {"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
52 {"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
53 ]
54 }
55 ]
56 }
57
#!/usr/bin/env python3
"""DeepCAD Canonical Equivalence solver.
Reads an input JSON of program pairs, decides for each pair whether both
programs describe the same canonical solid, and writes predictions.json.
Usage:
python3 solve.py <input_json> <output_json>
"""
import ast
import json
import sys
ANNOTATION = {"constraint", "dimension", "note", "metadata", "view", "comment"}
PROFILE_KINDS = {"rect", "circle", "slot"}
def round6(x):
"""Quantize to the 1e-6 tolerance grid, normalizing negative zero."""
r = round(float(x), 6)
if r == 0:
r = 0.0
return r
def canon(obj):
"""Canonical, order-stable JSON string.
sort_keys canonicalizes object key order (JSON objects are unordered)
while preserving list order, which is required for opaque payloads.
"""
return json.dumps(obj, sort_keys=True, separators=(",", ":"))
class Evaluator:
"""Evaluates scalar arithmetic expressions over a parameter environment."""
def __init__(self, params):
self.params = params # name -> expr (string or number)
self.cache = {}
self.stack = set()
def get(self, name):
if name in self.cache:
return self.cache[name]
if name not in self.params:
raise KeyError(name)
if name in self.stack:
raise ValueError("cyclic parameter reference")
self.stack.add(name)
try:
val = self.eval_expr(self.params[name])
finally:
self.stack.discard(name)
self.cache[name] = val
return val
def eval_expr(self, expr):
node = ast.parse(str(expr), mode="eval").body
return self._ev(node)
def _ev(self, node):
if isinstance(node, ast.BinOp):
l = self._ev(node.left)
r = self._ev(node.right)
if isinstance(node.op, ast.Add):
return l + r
if isinstance(node.op, ast.Sub):
return l - r
if isinstance(node.op, ast.Mult):
return l * r
if isinstance(node.op, ast.Div):
return l / r
raise ValueError("unsupported operator")
if isinstance(node, ast.UnaryOp):
v = self._ev(node.operand)
if isinstance(node.op, ast.USub):
return -v
if isinstance(node.op, ast.UAdd):
return v
raise ValueError("unsupported unary operator")
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ValueError("non-numeric constant")
return float(node.value)
if isinstance(node, ast.Name):
return self.get(node.id)
raise ValueError("unsupported expression node")
def eval_value(v, ev):
"""Evaluate a scalar field to a rounded float, or leave non-numeric as-is."""
if isinstance(v, bool):
return v
if isinstance(v, (int, float)):
return round6(v)
if isinstance(v, str):
try:
return round6(ev.eval_expr(v))
except Exception:
return v
return v
def deep_eval(v, ev):
if isinstance(v, list):
return [deep_eval(x, ev) for x in v]
return eval_value(v, ev)
def parse_program(prog):
params, sketches, profiles, extrudes, opaque = {}, {}, {}, [], []
for cmd in prog:
if not isinstance(cmd, dict):
# Non-dict command: treat as opaque literal payload.
opaque.append(cmd)
continue
if cmd.get("construction") is True:
continue
kind = cmd.get("cmd")
if kind in ANNOTATION:
continue
if kind == "param":
params[cmd.get("name")] = cmd.get("expr")
elif kind == "sketch":
plane = cmd.get("plane")
sketches[cmd.get("id")] = plane.upper() if isinstance(plane, str) else plane
elif kind in PROFILE_KINDS:
profiles[cmd.get("id")] = cmd
elif kind == "extrude":
extrudes.append(cmd)
else:
opaque.append(cmd)
return params, sketches, profiles, extrudes, opaque
def profile_sig(pcmd, sketches, ev):
plane = sketches.get(pcmd.get("sketch"))
kind = pcmd.get("cmd")
fields = {}
for k, v in pcmd.items():
if k in ("cmd", "id", "sketch", "construction"):
continue
fields[k] = deep_eval(v, ev)
if kind == "slot" and isinstance(fields.get("angle"), (int, float)) and not isinstance(fields.get("angle"), bool):
fields["angle"] = round6(fields["angle"] % 180)
return {"kind": kind, "plane": plane, "fields": fields}
def feature_sig(ecmd, sketches, profiles, ev):
op = ecmd.get("operation", "new")
op = op.lower() if isinstance(op, str) else op
extent = ecmd.get("extent", "one_side")
extent = extent.lower() if isinstance(extent, str) else extent
direction = ecmd.get("direction", "normal")
if isinstance(direction, str):
direction = direction.lower()
else:
direction = deep_eval(direction, ev)
depth = ecmd.get("depth")
depth = deep_eval(depth, ev) if depth is not None else None
if "profiles" in ecmd:
prof_ids = ecmd["profiles"]
if not isinstance(prof_ids, list):
prof_ids = [prof_ids]
elif "profile" in ecmd:
prof_ids = [ecmd["profile"]]
else:
prof_ids = []
psigs = sorted(
canon(profile_sig(profiles[pid], sketches, ev))
for pid in prof_ids
if pid in profiles
)
return {
"op": op,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": psigs,
}
def partition_key(ecmd):
"""Partition label used for co-membership (value only, not namespace)."""
if "body" in ecmd:
return ("L", canon(ecmd["body"]))
if "channel" in ecmd:
return ("L", canon(ecmd["channel"]))
return ("DEFAULT",)
def program_signature(prog):
params, sketches, profiles, extrudes, opaque = parse_program(prog)
ev = Evaluator(params)
# Group extrude features by partition; discard label spelling, keep topology.
groups = {}
for ecmd in extrudes:
fsig = canon(feature_sig(ecmd, sketches, profiles, ev))
pk = partition_key(ecmd)
groups.setdefault(pk, []).append(fsig)
group_list = sorted(sorted(v) for v in groups.values())
partition_sig = canon(group_list)
# Opaque commands: unordered multiset, payload literal minus top-level id.
opaque_sigs = []
for c in opaque:
if isinstance(c, dict):
payload = {k: v for k, v in c.items() if k != "id"}
else:
payload = c
opaque_sigs.append(canon(payload))
opaque_sigs.sort()
return canon([partition_sig, opaque_sigs])
def main():
if len(sys.argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
sys.exit(2)
with open(sys.argv[1], "r") as f:
data = json.load(f)
predictions = []
for pair in data.get("pairs", []):
pid = pair.get("pair_id")
sig_a = program_signature(pair.get("program_a", []))
sig_b = program_signature(pair.get("program_b", []))
predictions.append({
"pair_id": pid,
"equivalent": sig_a == sig_b,
"signature_a": sig_a,
"signature_b": sig_b,
})
with open(sys.argv[2], "w") as f:
json.dump({"predictions": predictions}, f, indent=2)
if __name__ == "__main__":
main()
/workspace/solve.py
1 #!/usr/bin/env python3
2 """Weak starter baseline for DeepCAD canonical-equivalence.
3
4 This intentionally predicts every pair as non-equivalent. It is useful only as
5 a schema example and should score reward 0.0 on the hidden verifier.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import sys
12 from pathlib import Path
13
14
15 def main() -> int:
16 input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
17 output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
18
19 payload = json.loads(input_path.read_text(encoding="utf-8"))
20 predictions = [
21 {"pair_id": pair["pair_id"], "equivalent": False}
22 for pair in payload.get("pairs", [])
23 ]
24
25 output_path.parent.mkdir(parents=True, exist_ok=True)
26 output_path.write_text(
27 json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
28 encoding="utf-8",
29 )
30 return 0
31
32
33 if __name__ == "__main__":
34 raise SystemExit(main())
35
#!/usr/bin/env python3
"""DeepCAD Canonical Equivalence solver.
Reads an input JSON of program pairs, decides for each pair whether both
programs describe the same canonical solid, and writes predictions.json.
Usage:
python3 solve.py <input_json> <output_json>
"""
import ast
import json
import sys
ANNOTATION = {"constraint", "dimension", "note", "metadata", "view", "comment"}
PROFILE_KINDS = {"rect", "circle", "slot"}
def round6(x):
"""Quantize to the 1e-6 tolerance grid, normalizing negative zero."""
r = round(float(x), 6)
if r == 0:
r = 0.0
return r
def canon(obj):
"""Canonical, order-stable JSON string.
sort_keys canonicalizes object key order (JSON objects are unordered)
while preserving list order, which is required for opaque payloads.
"""
return json.dumps(obj, sort_keys=True, separators=(",", ":"))
class Evaluator:
"""Evaluates scalar arithmetic expressions over a parameter environment."""
def __init__(self, params):
self.params = params # name -> expr (string or number)
self.cache = {}
self.stack = set()
def get(self, name):
if name in self.cache:
return self.cache[name]
if name not in self.params:
raise KeyError(name)
if name in self.stack:
raise ValueError("cyclic parameter reference")
self.stack.add(name)
try:
val = self.eval_expr(self.params[name])
finally:
self.stack.discard(name)
self.cache[name] = val
return val
def eval_expr(self, expr):
node = ast.parse(str(expr), mode="eval").body
return self._ev(node)
def _ev(self, node):
if isinstance(node, ast.BinOp):
l = self._ev(node.left)
r = self._ev(node.right)
if isinstance(node.op, ast.Add):
return l + r
if isinstance(node.op, ast.Sub):
return l - r
if isinstance(node.op, ast.Mult):
return l * r
if isinstance(node.op, ast.Div):
return l / r
raise ValueError("unsupported operator")
if isinstance(node, ast.UnaryOp):
v = self._ev(node.operand)
if isinstance(node.op, ast.USub):
return -v
if isinstance(node.op, ast.UAdd):
return v
raise ValueError("unsupported unary operator")
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ValueError("non-numeric constant")
return float(node.value)
if isinstance(node, ast.Name):
return self.get(node.id)
raise ValueError("unsupported expression node")
def eval_value(v, ev):
"""Evaluate a scalar field to a rounded float, or leave non-numeric as-is."""
if isinstance(v, bool):
return v
if isinstance(v, (int, float)):
return round6(v)
if isinstance(v, str):
try:
return round6(ev.eval_expr(v))
except Exception:
return v
return v
def deep_eval(v, ev):
if isinstance(v, list):
return [deep_eval(x, ev) for x in v]
return eval_value(v, ev)
def parse_program(prog):
params, sketches, profiles, extrudes, opaque = {}, {}, {}, [], []
for cmd in prog:
if not isinstance(cmd, dict):
# Non-dict command: treat as opaque literal payload.
opaque.append(cmd)
continue
if cmd.get("construction") is True:
continue
kind = cmd.get("cmd")
if kind in ANNOTATION:
continue
if kind == "param":
params[cmd.get("name")] = cmd.get("expr")
elif kind == "sketch":
plane = cmd.get("plane")
sketches[cmd.get("id")] = plane.upper() if isinstance(plane, str) else plane
elif kind in PROFILE_KINDS:
profiles[cmd.get("id")] = cmd
elif kind == "extrude":
extrudes.append(cmd)
else:
opaque.append(cmd)
return params, sketches, profiles, extrudes, opaque
def profile_sig(pcmd, sketches, ev):
plane = sketches.get(pcmd.get("sketch"))
kind = pcmd.get("cmd")
fields = {}
for k, v in pcmd.items():
if k in ("cmd", "id", "sketch", "construction"):
continue
fields[k] = deep_eval(v, ev)
angle = fields.get("angle")
if kind == "slot" and isinstance(angle, (int, float)) and not isinstance(angle, bool):
fields["angle"] = round6(angle % 180)
return {"kind": kind, "plane": plane, "fields": fields}
def feature_sig(ecmd, sketches, profiles, ev):
op = ecmd.get("operation", "new")
op = op.lower() if isinstance(op, str) else op
extent = ecmd.get("extent", "one_side")
extent = extent.lower() if isinstance(extent, str) else extent
direction = ecmd.get("direction", "normal")
if isinstance(direction, str):
direction = direction.lower()
else:
direction = deep_eval(direction, ev)
depth = ecmd.get("depth")
depth = deep_eval(depth, ev) if depth is not None else None
if "profiles" in ecmd:
prof_ids = ecmd["profiles"]
if not isinstance(prof_ids, list):
prof_ids = [prof_ids]
elif "profile" in ecmd:
prof_ids = [ecmd["profile"]]
else:
prof_ids = []
psigs = sorted(
canon(profile_sig(profiles[pid], sketches, ev))
for pid in prof_ids
if pid in profiles
)
return {
"op": op,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": psigs,
}
def partition_key(ecmd):
"""Partition label used for co-membership (value only, not namespace)."""
if "body" in ecmd:
return ("L", canon(ecmd["body"]))
if "channel" in ecmd:
return ("L", canon(ecmd["channel"]))
return ("DEFAULT",)
def program_signature(prog):
params, sketches, profiles, extrudes, opaque = parse_program(prog)
ev = Evaluator(params)
# Group extrude features by partition; discard label spelling, keep topology.
groups = {}
for ecmd in extrudes:
fsig = canon(feature_sig(ecmd, sketches, profiles, ev))
pk = partition_key(ecmd)
groups.setdefault(pk, []).append(fsig)
group_list = sorted(sorted(v) for v in groups.values())
partition_sig = canon(group_list)
# Opaque commands: unordered multiset, payload literal minus top-level id.
opaque_sigs = []
for c in opaque:
if isinstance(c, dict):
payload = {k: v for k, v in c.items() if k != "id"}
else:
payload = c
opaque_sigs.append(canon(payload))
opaque_sigs.sort()
return canon([partition_sig, opaque_sigs])
def main():
if len(sys.argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
return 2
with open(sys.argv[1], "r", encoding="utf-8") as f:
data = json.load(f)
predictions = []
for pair in data.get("pairs", []):
pid = pair.get("pair_id")
sig_a = program_signature(pair.get("program_a", []))
sig_b = program_signature(pair.get("program_b", []))
predictions.append({
"pair_id": pid,
"equivalent": bool(sig_a == sig_b),
"signature_a": sig_a,
"signature_b": sig_b,
})
with open(sys.argv[2], "w", encoding="utf-8") as f:
json.dump({"predictions": predictions}, f, indent=2)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Run solver on public example
python3 solve.py data/public_pairs.json predictions.json && cat predictions.json
{
"predictions": [
{
"pair_id": "P001",
"equivalent": true,
"signature_a": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":8.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"new\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[0.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[40.0,20.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]",
"signature_b": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":8.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"new\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[0.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[40.0,20.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]"
},
{
"pair_id": "P002",
"equivalent": false,
"signature_a": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":10.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"cut\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[8.0,0.0],\\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\":4.0},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]",
"signature_b": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":10.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"cut\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[8.0,0.0],\\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\":4.25},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]"
},
{
"pair_id": "P003",
"equivalent": true,
"signature_a": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":5.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"add\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[-6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]",
"signature_b": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":5.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"add\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[-6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]"
}
]
}
[stdout]
{
"predictions": [
{
"pair_id": "P001",
"equivalent": true,
"signature_a": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":8.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"new\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[0.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[40.0,20.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]",
"signature_b": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":8.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"new\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[0.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[40.0,20.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]"
},
{
"pair_id": "P002",
"equivalent": false,
"signature_a": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":10.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"cut\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[8.0,0.0],\\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\":4.0},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]",
"signature_b": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":10.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"cut\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[8.0,0.0],\\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\":4.25},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]"
},
{
"pair_id": "P003",
"equivalent": true,
"signature_a": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":5.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"add\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[-6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]",
"signature_b": "[\"[[\\\"{\\\\\\\"depth\\\\\\\":5.0,\\\\\\\"direction\\\\\\\":\\\\\\\"normal\\\\\\\",\\\\\\\"extent\\\\\\\":\\\\\\\"one_side\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"add\\\\\\\",\\\\\\\"profiles\\\\\\\":[\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[-6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\",\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\":[6.0,0.0],\\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\":[2.0,12.0]},\\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]]\",[]]"
}
]
}Test semantic edge cases
cat > /tmp/edge.json <<'EOF'
{"pairs":[
{"pair_id":"E1","comment":"body vs channel same grouping -> equivalent",
"program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":5,"body":"A"},
{"cmd":"extrude","id":"e2","profile":"r","depth":5,"body":"A"}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":5,"channel":"Q"},
{"cmd":"extrude","id":"e2","profile":"r","depth":5,"channel":"Q"}]},
{"pair_id":"E2","comment":"grouped-together vs separate partitions -> NOT equivalent",
"program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":5,"body":"A"},
{"cmd":"extrude","id":"e2","profile":"r","depth":5,"body":"A"}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":5,"body":"A"},
{"cmd":"extrude","id":"e2","profile":"r","depth":5,"body":"B"}]},
{"pair_id":"E3","comment":"plane XZ vs ZX -> NOT equivalent",
"program_a":[{"cmd":"sketch","id":"s","plane":"XZ"},{"cmd":"circle","id":"c","sketch":"s","center":[0,0],"radius":3},{"cmd":"extrude","id":"e","profile":"c","depth":5}],
"program_b":[{"cmd":"sketch","id":"s","plane":"ZX"},{"cmd":"circle","id":"c","sketch":"s","center":[0,0],"radius":3},{"cmd":"extrude","id":"e","profile":"c","depth":5}]},
{"pair_id":"E4","comment":"opaque fillet: alpha-rename NOT allowed, literal payload -> NOT equivalent",
"program_a":[{"cmd":"fillet","id":"f1","edges":["edge_7"],"radius":"2"}],
"program_b":[{"cmd":"fillet","id":"f2","edges":["edge_9"],"radius":"2"}]},
{"pair_id":"E5","comment":"opaque same payload diff id -> equivalent",
"program_a":[{"cmd":"fillet","id":"f1","edges":["edge_7"],"radius":"2"}],
"program_b":[{"cmd":"fillet","id":"zzz","edges":["edge_7"],"radius":"2"}]},
{"pair_id":"E6","comment":"slot angle mod 180 -> equivalent",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":10},{"cmd":"extrude","id":"e","profile":"sl","depth":5}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":190},{"cmd":"extrude","id":"e","profile":"sl","depth":5}]},
{"pair_id":"E7","comment":"negative depth != reversed direction -> NOT equivalent",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e","profile":"r","depth":-5,"direction":"normal"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e","profile":"r","depth":5,"direction":"reversed"}]}
]}
EOF
python3 /workspace/solve.py /tmp/edge.json /tmp/out.json
python3 -c "import json;print([(p['pair_id'],p['equivalent']) for p in json.load(open('/tmp/out.json'))['predictions']])"[('E1', True), ('E2', False), ('E3', False), ('E4', False), ('E5', True), ('E6', True), ('E7', False)]
[stdout]
[('E1', True), ('E2', False), ('E3', False), ('E4', False), ('E5', True), ('E6', True), ('E7', False)]Test partition and default-field edge cases
cat > /tmp/edge2.json <<'EOF'
{"pairs":[
{"pair_id":"D1","comment":"two unlabeled (1 group) vs unlabeled+labeled (2 groups) -> NOT equiv",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":5},{"cmd":"extrude","id":"e2","profile":"r","depth":5}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r","depth":5},{"cmd":"extrude","id":"e2","profile":"r","depth":5,"body":"X"}]},
{"pair_id":"D2","comment":"default one_side/new/normal implicit vs explicit -> equiv",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":5}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":5,"operation":"new","extent":"one_side","direction":"normal"}]},
{"pair_id":"D3","comment":"add vs join distinct (no synonym) -> NOT equiv",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":5,"operation":"add"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":5,"operation":"join"}]},
{"pair_id":"D4","comment":"unconsumed profile & construction & annotation ignored -> equiv",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"circle","id":"unused","sketch":"s","center":[9,9],"radius":1},
{"cmd":"dimension","id":"d","value":5},{"cmd":"rect","id":"cx","sketch":"s","center":[3,3],"size":[1,1],"construction":true},
{"cmd":"extrude","id":"e","profile":"r","depth":5}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"extrude","id":"e","profile":"r","depth":5}]}
]}
EOF
python3 /workspace/solve.py /tmp/edge2.json /tmp/out2.json
python3 -c "import json;print([(p['pair_id'],p['equivalent']) for p in json.load(open('/tmp/out2.json'))['predictions']])"[('D1', False), ('D2', True), ('D3', False), ('D4', True)]
[stdout]
[('D1', False), ('D2', True), ('D3', False), ('D4', True)]No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_3f27431bad454069. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_3f27431bad454069 · verifier authoritative; classifier explanatory.