tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it./workspace/data/public_pairs.json
1 {
2 "schema_version": "1.0",
3 "description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
4 "pairs": [
5 {
6 "pair_id": "P001",
7 "program_a": [
8 {"cmd": "param", "name": "w", "expr": "40"},
9 {"cmd": "param", "name": "h", "expr": "20"},
10 {"cmd": "param", "name": "d", "expr": "8"},
11 {"cmd": "sketch", "id": "s0", "plane": "XY"},
12 {"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
13 {"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
14 {"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
15 ],
16 "program_b": [
17 {"cmd": "param", "name": "depth", "expr": "4 + 4"},
18 {"cmd": "param", "name": "height", "expr": "5 * 4"},
19 {"cmd": "param", "name": "width", "expr": "20 * 2"},
20 {"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
21 {"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
22 {"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
23 ]
24 },
25 {
26 "pair_id": "P002",
27 "program_a": [
28 {"cmd": "param", "name": "r", "expr": "4"},
29 {"cmd": "sketch", "id": "s", "plane": "XY"},
30 {"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
31 {"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
32 ],
33 "program_b": [
34 {"cmd": "param", "name": "r2", "expr": "4.25"},
35 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
36 {"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
37 {"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
38 ]
39 },
40 {
41 "pair_id": "P003",
42 "program_a": [
43 {"cmd": "sketch", "id": "s", "plane": "XY"},
44 {"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
45 {"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
46 {"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
47 ],
48 "program_b": [
49 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
50 {"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
51 {"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
52 {"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
53 ]
54 }
55 ]
56 }
57
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a pairs JSON, decides for each pair whether program_a and program_b
describe the same canonical solid, and writes predictions.json.
"""
import json
import sys
# ---------------------------------------------------------------------------
# Command classification
# ---------------------------------------------------------------------------
METADATA_KINDS = {
"constraint", "dimension", "note", "metadata", "view", "comment",
}
PROFILE_KINDS = {"rect", "circle", "slot"}
SUPPORTED_KINDS = {"param", "sketch", "extrude"} | PROFILE_KINDS
TOL_DECIMALS = 6
def q(x):
"""Quantize a numeric value to the comparison tolerance grid."""
v = round(float(x), TOL_DECIMALS)
if v == 0.0:
v = 0.0 # normalize -0.0 -> 0.0
return v
# ---------------------------------------------------------------------------
# Arithmetic expression evaluation
# ---------------------------------------------------------------------------
class ExprError(Exception):
pass
def tokenize(s):
tokens = []
i = 0
n = len(s)
while i < n:
c = s[i]
if c.isspace():
i += 1
continue
if c in "+-*/()":
tokens.append((c, c))
i += 1
continue
if c.isdigit() or c == ".":
j = i
while j < n and (s[j].isdigit() or s[j] in ".eE" or
(s[j] in "+-" and j > i and s[j - 1] in "eE")):
j += 1
tokens.append(("num", float(s[i:j])))
i = j
continue
if c.isalpha() or c == "_":
j = i
while j < n and (s[j].isalnum() or s[j] == "_"):
j += 1
tokens.append(("id", s[i:j]))
i = j
continue
raise ExprError("bad char %r in %r" % (c, s))
return tokens
class Parser:
def __init__(self, tokens, resolver):
self.tokens = tokens
self.pos = 0
self.resolver = resolver
def peek(self):
return self.tokens[self.pos] if self.pos < len(self.tokens) else (None, None)
def advance(self):
tok = self.tokens[self.pos]
self.pos += 1
return tok
def parse(self):
val = self.expr()
if self.pos != len(self.tokens):
raise ExprError("trailing tokens")
return val
def expr(self):
val = self.term()
while self.peek()[0] in ("+", "-"):
op = self.advance()[0]
rhs = self.term()
val = val + rhs if op == "+" else val - rhs
return val
def term(self):
val = self.factor()
while self.peek()[0] in ("*", "/"):
op = self.advance()[0]
rhs = self.factor()
val = val * rhs if op == "*" else val / rhs
return val
def factor(self):
typ, v = self.peek()
if typ == "+":
self.advance()
return self.factor()
if typ == "-":
self.advance()
return -self.factor()
if typ == "num":
self.advance()
return v
if typ == "id":
self.advance()
return self.resolver(v)
if typ == "(":
self.advance()
val = self.expr()
if self.peek()[0] != ")":
raise ExprError("missing )")
self.advance()
return val
raise ExprError("unexpected token %r" % (typ,))
class ParamEnv:
"""Resolves parameter names to numeric values with memoization."""
def __init__(self, params):
# params: name -> raw expr (string or number)
self.params = params
self.cache = {}
self.stack = set()
def resolve(self, name):
if name in self.cache:
return self.cache[name]
if name in self.stack:
raise ExprError("cyclic param %r" % name)
if name not in self.params:
raise ExprError("unknown param %r" % name)
self.stack.add(name)
val = self.eval(self.params[name])
self.stack.discard(name)
self.cache[name] = val
return val
def eval(self, raw):
if isinstance(raw, bool):
return float(raw)
if isinstance(raw, (int, float)):
return float(raw)
if isinstance(raw, str):
toks = tokenize(raw)
return Parser(toks, self.resolve).parse()
raise ExprError("cannot eval %r" % (raw,))
def eval_value(v, env):
"""Evaluate a geometric field value into a canonical form."""
if isinstance(v, bool):
return v
if isinstance(v, (int, float)):
return q(v)
if isinstance(v, str):
return q(env.eval(v))
if isinstance(v, list):
return tuple(eval_value(e, env) for e in v)
return v
# ---------------------------------------------------------------------------
# Program canonicalization
# ---------------------------------------------------------------------------
PROFILE_SKIP_FIELDS = {"cmd", "id", "sketch", "construction"}
def profile_signature(prof, sketches, env):
kind = prof.get("cmd")
plane = sketches.get(prof.get("sketch"))
fields = []
for key in sorted(prof.keys()):
if key in PROFILE_SKIP_FIELDS:
continue
val = eval_value(prof[key], env)
if key == "angle":
val = angle_mod(val)
fields.append((key, val))
return (kind, plane, tuple(fields))
def angle_mod(val):
if isinstance(val, tuple):
return tuple(angle_mod(v) for v in val)
if isinstance(val, (int, float)):
return q(((float(val) % 180.0) + 180.0) % 180.0)
return val
def canon_direction(v, env):
if isinstance(v, str):
return v.lower()
if isinstance(v, list):
return tuple(eval_value(e, env) for e in v)
if isinstance(v, (int, float)):
return q(v)
return v
DEFAULT_PARTITION = ("__default__",)
def partition_key(cmd):
if "body" in cmd and cmd["body"] is not None:
return ("label", str(cmd["body"]))
if "channel" in cmd and cmd["channel"] is not None:
return ("label", str(cmd["channel"]))
return DEFAULT_PARTITION
def extrude_content_sig(cmd, profiles, sketches, env):
operation = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = canon_direction(cmd.get("direction", "normal"), env)
depth_raw = cmd.get("depth", None)
if depth_raw is None:
depth = None
else:
depth = eval_value(depth_raw, env)
# collect referenced profile ids
ref_ids = []
if "profiles" in cmd and cmd["profiles"] is not None:
ref_ids.extend(cmd["profiles"])
if "profile" in cmd and cmd["profile"] is not None:
ref_ids.append(cmd["profile"])
prof_sigs = []
for pid in ref_ids:
prof = profiles.get(pid)
if prof is None:
continue
prof_sigs.append(profile_signature(prof, sketches, env))
prof_sigs.sort()
return ("extrude", operation, extent, direction, depth, tuple(prof_sigs))
def opaque_signature(cmd):
payload = {k: v for k, v in cmd.items() if k != "id"}
return json.dumps(payload, sort_keys=True, ensure_ascii=True)
def canonicalize(program):
params = {}
sketches = {}
profiles = {}
extrudes = []
opaque = []
for cmd in program:
if not isinstance(cmd, dict):
continue
if cmd.get("construction") is True:
continue
kind = cmd.get("cmd")
if kind in METADATA_KINDS:
continue
if kind == "param":
raw = cmd.get("expr", cmd.get("value"))
params[cmd.get("name")] = raw
elif kind == "sketch":
plane = cmd.get("plane")
if isinstance(plane, str):
plane = plane.upper()
sketches[cmd.get("id")] = plane
elif kind in PROFILE_KINDS:
profiles[cmd.get("id")] = cmd
elif kind == "extrude":
extrudes.append(cmd)
else:
opaque.append(cmd)
env = ParamEnv(params)
# Build extrude features with partition grouping.
groups = {}
for cmd in extrudes:
content = extrude_content_sig(cmd, profiles, sketches, env)
key = partition_key(cmd)
groups.setdefault(key, []).append(content)
group_sigs = []
for key, members in groups.items():
members.sort()
group_sigs.append(tuple(members))
group_sigs.sort()
opaque_sigs = sorted(opaque_signature(c) for c in opaque)
return (tuple(group_sigs), tuple(opaque_sigs))
def signature_repr(sig):
return repr(sig)
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
if len(sys.argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
sys.exit(2)
in_path, out_path = sys.argv[1], sys.argv[2]
with open(in_path, "r") as f:
data = json.load(f)
predictions = []
for pair in data.get("pairs", []):
pid = pair.get("pair_id")
try:
sig_a = canonicalize(pair.get("program_a", []))
sig_b = canonicalize(pair.get("program_b", []))
equivalent = sig_a == sig_b
except Exception as e: # noqa: BLE001
sys.stderr.write("error on %s: %s\n" % (pid, e))
sig_a = sig_b = None
equivalent = False
predictions.append({
"pair_id": pid,
"equivalent": bool(equivalent),
"signature_a": signature_repr(sig_a),
"signature_b": signature_repr(sig_b),
})
with open(out_path, "w") as f:
json.dump({"predictions": predictions}, f, indent=2)
if __name__ == "__main__":
main()
/workspace/solve.py
1 #!/usr/bin/env python3
2 """Weak starter baseline for DeepCAD canonical-equivalence.
3
4 This intentionally predicts every pair as non-equivalent. It is useful only as
5 a schema example and should score reward 0.0 on the hidden verifier.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import sys
12 from pathlib import Path
13
14
15 def main() -> int:
16 input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
17 output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
18
19 payload = json.loads(input_path.read_text(encoding="utf-8"))
20 predictions = [
21 {"pair_id": pair["pair_id"], "equivalent": False}
22 for pair in payload.get("pairs", [])
23 ]
24
25 output_path.parent.mkdir(parents=True, exist_ok=True)
26 output_path.write_text(
27 json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
28 encoding="utf-8",
29 )
30 return 0
31
32
33 if __name__ == "__main__":
34 raise SystemExit(main())
35
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a pairs JSON, decides for each pair whether program_a and program_b
describe the same canonical solid, and writes predictions.json.
"""
import json
import sys
# ---------------------------------------------------------------------------
# Command classification
# ---------------------------------------------------------------------------
METADATA_KINDS = {
"constraint", "dimension", "note", "metadata", "view", "comment",
}
PROFILE_KINDS = {"rect", "circle", "slot"}
SUPPORTED_KINDS = {"param", "sketch", "extrude"} | PROFILE_KINDS
TOL_DECIMALS = 6
def q(x):
"""Quantize a numeric value to the comparison tolerance grid."""
v = round(float(x), TOL_DECIMALS)
if v == 0.0:
v = 0.0 # normalize -0.0 -> 0.0
return v
# ---------------------------------------------------------------------------
# Arithmetic expression evaluation
# ---------------------------------------------------------------------------
class ExprError(Exception):
pass
def tokenize(s):
tokens = []
i = 0
n = len(s)
while i < n:
c = s[i]
if c.isspace():
i += 1
continue
if c in "+-*/()":
tokens.append((c, c))
i += 1
continue
if c.isdigit() or c == ".":
j = i
while j < n and (s[j].isdigit() or s[j] in ".eE" or
(s[j] in "+-" and j > i and s[j - 1] in "eE")):
j += 1
tokens.append(("num", float(s[i:j])))
i = j
continue
if c.isalpha() or c == "_":
j = i
while j < n and (s[j].isalnum() or s[j] == "_"):
j += 1
tokens.append(("id", s[i:j]))
i = j
continue
raise ExprError("bad char %r in %r" % (c, s))
return tokens
class Parser:
def __init__(self, tokens, resolver):
self.tokens = tokens
self.pos = 0
self.resolver = resolver
def peek(self):
return self.tokens[self.pos] if self.pos < len(self.tokens) else (None, None)
def advance(self):
tok = self.tokens[self.pos]
self.pos += 1
return tok
def parse(self):
val = self.expr()
if self.pos != len(self.tokens):
raise ExprError("trailing tokens")
return val
def expr(self):
val = self.term()
while self.peek()[0] in ("+", "-"):
op = self.advance()[0]
rhs = self.term()
val = val + rhs if op == "+" else val - rhs
return val
def term(self):
val = self.factor()
while self.peek()[0] in ("*", "/"):
op = self.advance()[0]
rhs = self.factor()
val = val * rhs if op == "*" else val / rhs
return val
def factor(self):
typ, v = self.peek()
if typ == "+":
self.advance()
return self.factor()
if typ == "-":
self.advance()
return -self.factor()
if typ == "num":
self.advance()
return v
if typ == "id":
self.advance()
return self.resolver(v)
if typ == "(":
self.advance()
val = self.expr()
if self.peek()[0] != ")":
raise ExprError("missing )")
self.advance()
return val
raise ExprError("unexpected token %r" % (typ,))
class ParamEnv:
"""Resolves parameter names to numeric values with memoization."""
def __init__(self, params):
# params: name -> raw expr (string or number)
self.params = params
self.cache = {}
self.stack = set()
def resolve(self, name):
if name in self.cache:
return self.cache[name]
if name in self.stack:
raise ExprError("cyclic param %r" % name)
if name not in self.params:
raise ExprError("unknown param %r" % name)
self.stack.add(name)
val = self.eval(self.params[name])
self.stack.discard(name)
self.cache[name] = val
return val
def eval(self, raw):
if isinstance(raw, bool):
return float(raw)
if isinstance(raw, (int, float)):
return float(raw)
if isinstance(raw, str):
toks = tokenize(raw)
return Parser(toks, self.resolve).parse()
raise ExprError("cannot eval %r" % (raw,))
def eval_value(v, env):
"""Evaluate a geometric field value into a canonical form."""
if isinstance(v, bool):
return v
if isinstance(v, (int, float)):
return q(v)
if isinstance(v, str):
return q(env.eval(v))
if isinstance(v, list):
return tuple(eval_value(e, env) for e in v)
return v
# ---------------------------------------------------------------------------
# Program canonicalization
# ---------------------------------------------------------------------------
PROFILE_SKIP_FIELDS = {"cmd", "id", "sketch", "construction"}
def angle_mod(val):
if isinstance(val, tuple):
return tuple(angle_mod(v) for v in val)
if isinstance(val, (int, float)):
return q(((float(val) % 180.0) + 180.0) % 180.0)
return val
def profile_signature(prof, sketches, env):
kind = prof.get("cmd")
plane = sketches.get(prof.get("sketch"))
fields = []
for key in sorted(prof.keys()):
if key in PROFILE_SKIP_FIELDS:
continue
val = eval_value(prof[key], env)
if key == "angle":
val = angle_mod(val)
fields.append((key, val))
return (kind, plane, tuple(fields))
def canon_direction(v, env):
if isinstance(v, str):
return v.lower()
if isinstance(v, list):
return tuple(eval_value(e, env) for e in v)
if isinstance(v, (int, float)):
return q(v)
return v
DEFAULT_PARTITION = ("__default__",)
def partition_key(cmd):
if "body" in cmd and cmd["body"] is not None:
return ("label", str(cmd["body"]))
if "channel" in cmd and cmd["channel"] is not None:
return ("label", str(cmd["channel"]))
return DEFAULT_PARTITION
def extrude_content_sig(cmd, profiles, sketches, env):
operation = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = canon_direction(cmd.get("direction", "normal"), env)
depth_raw = cmd.get("depth", None)
depth = None if depth_raw is None else eval_value(depth_raw, env)
ref_ids = []
if cmd.get("profiles") is not None:
ref_ids.extend(cmd["profiles"])
if cmd.get("profile") is not None:
ref_ids.append(cmd["profile"])
prof_sigs = []
for pid in ref_ids:
prof = profiles.get(pid)
if prof is None:
continue
prof_sigs.append(profile_signature(prof, sketches, env))
prof_sigs.sort()
return ("extrude", operation, extent, direction, depth, tuple(prof_sigs))
def opaque_signature(cmd):
payload = {k: v for k, v in cmd.items() if k != "id"}
return json.dumps(payload, sort_keys=True, ensure_ascii=True)
def canonicalize(program):
params = {}
sketches = {}
profiles = {}
extrudes = []
opaque = []
for cmd in program:
if not isinstance(cmd, dict):
continue
if cmd.get("construction") is True:
continue
kind = cmd.get("cmd")
if kind in METADATA_KINDS:
continue
if kind == "param":
raw = cmd.get("expr", cmd.get("value"))
params[cmd.get("name")] = raw
elif kind == "sketch":
plane = cmd.get("plane")
if isinstance(plane, str):
plane = plane.upper()
sketches[cmd.get("id")] = plane
elif kind in PROFILE_KINDS:
profiles[cmd.get("id")] = cmd
elif kind == "extrude":
extrudes.append(cmd)
else:
opaque.append(cmd)
env = ParamEnv(params)
# Build extrude features grouped by body/channel partition topology.
groups = {}
for cmd in extrudes:
content = extrude_content_sig(cmd, profiles, sketches, env)
key = partition_key(cmd)
groups.setdefault(key, []).append(content)
group_sigs = []
for members in groups.values():
members.sort()
group_sigs.append(tuple(members))
group_sigs.sort()
opaque_sigs = sorted(opaque_signature(c) for c in opaque)
return (tuple(group_sigs), tuple(opaque_sigs))
def signature_repr(sig):
return repr(sig)
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
if len(sys.argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
sys.exit(2)
in_path, out_path = sys.argv[1], sys.argv[2]
with open(in_path, "r") as f:
data = json.load(f)
predictions = []
for pair in data.get("pairs", []):
pid = pair.get("pair_id")
try:
sig_a = canonicalize(pair.get("program_a", []))
sig_b = canonicalize(pair.get("program_b", []))
equivalent = sig_a == sig_b
except Exception as e: # noqa: BLE001
sys.stderr.write("error on %s: %s\n" % (pid, e))
sig_a = sig_b = None
equivalent = False
predictions.append({
"pair_id": pid,
"equivalent": bool(equivalent),
"signature_a": signature_repr(sig_a),
"signature_b": signature_repr(sig_b),
})
with open(out_path, "w") as f:
json.dump({"predictions": predictions}, f, indent=2)
if __name__ == "__main__":
main()
Run solver on public example
python3 solve.py data/public_pairs.json predictions.json && cat predictions.json
{
"predictions": [
{
"pair_id": "P001",
"equivalent": true,
"signature_a": "(((('extrude', 'new', 'one_side', 'normal', 8.0, (('rect', 'XY', (('center', (0.0, 0.0)), ('size', (40.0, 20.0)))),)),),), ())",
"signature_b": "(((('extrude', 'new', 'one_side', 'normal', 8.0, (('rect', 'XY', (('center', (0.0, 0.0)), ('size', (40.0, 20.0)))),)),),), ())"
},
{
"pair_id": "P002",
"equivalent": false,
"signature_a": "(((('extrude', 'cut', 'one_side', 'normal', 10.0, (('circle', 'XY', (('center', (8.0, 0.0)), ('radius', 4.0))),)),),), ())",
"signature_b": "(((('extrude', 'cut', 'one_side', 'normal', 10.0, (('circle', 'XY', (('center', (8.0, 0.0)), ('radius', 4.25))),)),),), ())"
},
{
"pair_id": "P003",
"equivalent": true,
"signature_a": "(((('extrude', 'add', 'one_side', 'normal', 5.0, (('rect', 'XY', (('center', (-6.0, 0.0)), ('size', (2.0, 12.0)))), ('rect', 'XY', (('center', (6.0, 0.0)), ('size', (2.0, 12.0)))))),),), ())",
"signature_b": "(((('extrude', 'add', 'one_side', 'normal', 5.0, (('rect', 'XY', (('center', (-6.0, 0.0)), ('size', (2.0, 12.0)))), ('rect', 'XY', (('center', (6.0, 0.0)), ('size', (2.0, 12.0)))))),),), ())"
}
]
}
[stdout]
{
"predictions": [
{
"pair_id": "P001",
"equivalent": true,
"signature_a": "(((('extrude', 'new', 'one_side', 'normal', 8.0, (('rect', 'XY', (('center', (0.0, 0.0)), ('size', (40.0, 20.0)))),)),),), ())",
"signature_b": "(((('extrude', 'new', 'one_side', 'normal', 8.0, (('rect', 'XY', (('center', (0.0, 0.0)), ('size', (40.0, 20.0)))),)),),), ())"
},
{
"pair_id": "P002",
"equivalent": false,
"signature_a": "(((('extrude', 'cut', 'one_side', 'normal', 10.0, (('circle', 'XY', (('center', (8.0, 0.0)), ('radius', 4.0))),)),),), ())",
"signature_b": "(((('extrude', 'cut', 'one_side', 'normal', 10.0, (('circle', 'XY', (('center', (8.0, 0.0)), ('radius', 4.25))),)),),), ())"
},
{
"pair_id": "P003",
"equivalent": true,
"signature_a": "(((('extrude', 'add', 'one_side', 'normal', 5.0, (('rect', 'XY', (('center', (-6.0, 0.0)), ('size', (2.0, 12.0)))), ('rect', 'XY', (('center', (6.0, 0.0)), ('size', (2.0, 12.0)))))),),), ())",
"signature_b": "(((('extrude', 'add', 'one_side', 'normal', 5.0, (('rect', 'XY', (('center', (-6.0, 0.0)), ('size', (2.0, 12.0)))), ('rect', 'XY', (('center', (6.0, 0.0)), ('size', (2.0, 12.0)))))),),), ())"
}
]
}Edge-case tests
cat > /tmp/t.json <<'EOF'
{"pairs":[
{"pair_id":"T1_partition_default_vs_label",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e1","profile":"r","depth":1},{"cmd":"extrude","id":"e2","profile":"r","depth":1}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e1","profile":"r","depth":1,"body":"A"},{"cmd":"extrude","id":"e2","profile":"r","depth":1,"body":"B"}]},
{"pair_id":"T2_body_channel_topology_equal",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e1","profile":"r","depth":1,"body":"X"},{"cmd":"extrude","id":"e2","profile":"r","depth":1,"body":"X"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e1","profile":"r","depth":1,"channel":"Q"},{"cmd":"extrude","id":"e2","profile":"r","depth":1,"channel":"Q"}]},
{"pair_id":"T3_slot_angle_mod180",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":30},{"cmd":"extrude","id":"e","profile":"sl","depth":1}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":210},{"cmd":"extrude","id":"e","profile":"sl","depth":1}]},
{"pair_id":"T4_direction_sign_distinct",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e","profile":"r","depth":-1}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e","profile":"r","depth":1}]},
{"pair_id":"T5_opaque_multiset",
"program_a":[{"cmd":"fillet","id":"f1","edges":["a"],"radius":2},{"cmd":"chamfer","id":"c1","edges":["b"],"dist":1}],
"program_b":[{"cmd":"chamfer","id":"c9","edges":["b"],"dist":1},{"cmd":"fillet","id":"f9","edges":["a"],"radius":2}]},
{"pair_id":"T6_plane_xz_vs_zx",
"program_a":[{"cmd":"sketch","id":"s","plane":"XZ"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e","profile":"r","depth":1}],
"program_b":[{"cmd":"sketch","id":"s","plane":"ZX"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,2]},{"cmd":"extrude","id":"e","profile":"r","depth":1}]}
]}
EOF
python3 solve.py /tmp/t.json /tmp/o.json
python3 -c "import json;print('\n'.join(f\"{p['pair_id']}: {p['equivalent']}\" for p in json.load(open('/tmp/o.json'))['predictions']))"T1_partition_default_vs_label: False T2_body_channel_topology_equal: True T3_slot_angle_mod180: True T4_direction_sign_distinct: False T5_opaque_multiset: True T6_plane_xz_vs_zx: False [stdout] T1_partition_default_vs_label: False T2_body_channel_topology_equal: True T3_slot_angle_mod180: True T4_direction_sign_distinct: False T5_opaque_multiset: True T6_plane_xz_vs_zx: False
No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_bf8af958b3ae4c6f. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_bf8af958b3ae4c6f · verifier authoritative; classifier explanatory.