tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.View public pairs example
cat /workspace/data/public_pairs.json
{
"schema_version": "1.0",
"description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
"pairs": [
{
"pair_id": "P001",
"program_a": [
{"cmd": "param", "name": "w", "expr": "40"},
{"cmd": "param", "name": "h", "expr": "20"},
{"cmd": "param", "name": "d", "expr": "8"},
{"cmd": "sketch", "id": "s0", "plane": "XY"},
{"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
{"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
{"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "depth", "expr": "4 + 4"},
{"cmd": "param", "name": "height", "expr": "5 * 4"},
{"cmd": "param", "name": "width", "expr": "20 * 2"},
{"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
{"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
{"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
]
},
{
"pair_id": "P002",
"program_a": [
{"cmd": "param", "name": "r", "expr": "4"},
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
{"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "r2", "expr": "4.25"},
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
{"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
]
},
{
"pair_id": "P003",
"program_a": [
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
],
"program_b": [
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
]
}
]
}
[stdout]
{
"schema_version": "1.0",
"description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
"pairs": [
{
"pair_id": "P001",
"program_a": [
{"cmd": "param", "name": "w", "expr": "40"},
{"cmd": "param", "name": "h", "expr": "20"},
{"cmd": "param", "name": "d", "expr": "8"},
{"cmd": "sketch", "id": "s0", "plane": "XY"},
{"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
{"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
{"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "depth", "expr": "4 + 4"},
{"cmd": "param", "name": "height", "expr": "5 * 4"},
{"cmd": "param", "name": "width", "expr": "20 * 2"},
{"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
{"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
{"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
]
},
{
"pair_id": "P002",
"program_a": [
{"cmd": "param", "name": "r", "expr": "4"},
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
{"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "r2", "expr": "4.25"},
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
{"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
]
},
{
"pair_id": "P003",
"program_a": [
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
],
"program_b": [
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
]
}
]
}#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a pairs JSON, decides for each pair whether program_a and program_b
describe the same canonical solid, and writes predictions.json.
"""
import json
import re
import sys
from collections import Counter
# --------------------------------------------------------------------------
# Command classification
# --------------------------------------------------------------------------
SUPPORTED = {"param", "sketch", "rect", "circle", "slot", "extrude"}
PROFILE_KINDS = {"rect", "circle", "slot"}
ANNOTATION = {"constraint", "dimension", "note", "metadata", "view", "comment"}
def is_construction(cmd):
return bool(cmd.get("construction", False))
# --------------------------------------------------------------------------
# Numeric helpers
# --------------------------------------------------------------------------
def norm_num(x):
"""Quantise to the 1e-6 tolerance and kill negative zero."""
v = round(float(x), 6) + 0.0
if v == 0:
v = 0.0
return v
# --------------------------------------------------------------------------
# Expression evaluator: numeric constants, params, + - * /, parentheses
# --------------------------------------------------------------------------
_TOKEN_NUM = re.compile(r"[0-9]*\.?[0-9]+([eE][+-]?[0-9]+)?")
_TOKEN_ID = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
def tokenize(s):
tokens = []
i = 0
n = len(s)
while i < n:
c = s[i]
if c.isspace():
i += 1
continue
if c in "+-*/()":
tokens.append(c)
i += 1
continue
if c.isdigit() or c == ".":
m = _TOKEN_NUM.match(s, i)
if not m:
raise ValueError("bad number in %r" % s)
tokens.append(("num", float(m.group())))
i = m.end()
continue
m = _TOKEN_ID.match(s, i)
if m:
tokens.append(("id", m.group()))
i = m.end()
continue
raise ValueError("bad char %r in %r" % (c, s))
return tokens
class Evaluator:
def __init__(self, params):
# params: name -> raw expr (string or number)
self.params = params
self.memo = {}
self.stack = set()
def value(self, name):
if name in self.memo:
return self.memo[name]
if name not in self.params:
raise ValueError("undefined param %r" % name)
if name in self.stack:
raise ValueError("cyclic param %r" % name)
self.stack.add(name)
v = self.eval(self.params[name])
self.stack.discard(name)
self.memo[name] = v
return v
def eval(self, expr):
if isinstance(expr, bool):
raise ValueError("bool not numeric")
if isinstance(expr, (int, float)):
return float(expr)
toks = tokenize(str(expr))
self._toks = toks
self._pos = 0
v = self._expr()
if self._pos != len(toks):
raise ValueError("trailing tokens in %r" % expr)
return v
def _peek(self):
if self._pos < len(self._toks):
return self._toks[self._pos]
return None
def _next(self):
t = self._toks[self._pos]
self._pos += 1
return t
def _expr(self):
v = self._term()
while True:
t = self._peek()
if t == "+":
self._next()
v += self._term()
elif t == "-":
self._next()
v -= self._term()
else:
break
return v
def _term(self):
v = self._factor()
while True:
t = self._peek()
if t == "*":
self._next()
v *= self._factor()
elif t == "/":
self._next()
v /= self._factor()
else:
break
return v
def _factor(self):
t = self._peek()
if t == "+":
self._next()
return self._factor()
if t == "-":
self._next()
return -self._factor()
if t == "(":
self._next()
v = self._expr()
if self._peek() != ")":
raise ValueError("missing )")
self._next()
return v
if isinstance(t, tuple) and t[0] == "num":
self._next()
return t[1]
if isinstance(t, tuple) and t[0] == "id":
self._next()
return self.value(t[1])
raise ValueError("unexpected token %r" % (t,))
# --------------------------------------------------------------------------
# Profile canonicalisation
# --------------------------------------------------------------------------
def canon_field(name, v, ev):
if isinstance(v, list):
return [canon_field(name, x, ev) for x in v]
if isinstance(v, bool):
return v
if isinstance(v, (int, float)):
num = float(v)
elif isinstance(v, str):
try:
num = ev.eval(v)
except Exception:
return v # opaque token inside a profile field
else:
return v
if name == "angle":
num = num % 180.0
return norm_num(num)
def canon_profile(cmd, ev, planes):
plane = planes.get(cmd.get("sketch"))
fields = {}
for k, v in cmd.items():
if k in ("cmd", "id", "sketch", "construction"):
continue
fields[k] = canon_field(k, v, ev)
return json.dumps(
{"kind": cmd["cmd"], "plane": plane, "fields": fields},
sort_keys=True,
)
# --------------------------------------------------------------------------
# Extrude canonicalisation
# --------------------------------------------------------------------------
def canon_direction(v, ev):
if isinstance(v, str):
return v.lower()
if isinstance(v, list):
return [canon_field("dir", x, ev) for x in v]
if isinstance(v, (int, float)) and not isinstance(v, bool):
return norm_num(v)
return v
def canon_extrude_content(cmd, ev, profiles):
op = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = canon_direction(cmd.get("direction", "normal"), ev)
depth = norm_num(ev.eval(cmd.get("depth", 0)))
refs = []
if "profiles" in cmd:
refs.extend(cmd["profiles"])
if "profile" in cmd:
refs.append(cmd["profile"])
prof_sigs = []
for r in refs:
if r in profiles:
prof_sigs.append(profiles[r])
prof_sigs.sort()
content = {
"op": op,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": prof_sigs,
}
return json.dumps(content, sort_keys=True)
def partition_key(cmd):
if "body" in cmd:
return ("L", str(cmd["body"]))
if "channel" in cmd:
return ("L", str(cmd["channel"]))
return ("DEFAULT",)
# --------------------------------------------------------------------------
# Program signature
# --------------------------------------------------------------------------
def program_signature(program):
# Build param table
param_table = {}
for cmd in program:
if cmd.get("cmd") == "param":
param_table[cmd.get("name")] = cmd.get("expr")
ev = Evaluator(param_table)
# Build sketch plane table
planes = {}
for cmd in program:
if cmd.get("cmd") == "sketch":
plane = cmd.get("plane")
if isinstance(plane, str):
plane = plane.upper()
planes[cmd.get("id")] = plane
# Build profile table (non-construction profiles only)
profiles = {}
for cmd in program:
if cmd.get("cmd") in PROFILE_KINDS and not is_construction(cmd):
profiles[cmd.get("id")] = canon_profile(cmd, ev, planes)
# Collect extrude features and opaque payloads
features = [] # (partition_key, content_json)
opaque = Counter()
for cmd in program:
if is_construction(cmd):
continue
kind = cmd.get("cmd")
if kind in ANNOTATION:
continue
if kind == "extrude":
content = canon_extrude_content(cmd, ev, profiles)
features.append((partition_key(cmd), content))
elif kind in SUPPORTED:
continue # param/sketch/profile: consumed via tables
else:
payload = {k: v for k, v in cmd.items() if k != "id"}
opaque[json.dumps(payload, sort_keys=True)] += 1
# Group features by partition, canonicalise as multiset of group-multisets
groups = {}
for key, content in features:
groups.setdefault(key, []).append(content)
group_sigs = Counter()
for key, contents in groups.items():
contents.sort()
group_sigs[json.dumps(contents, sort_keys=True)] += 1
return group_sigs, opaque
def equivalent(program_a, program_b):
ga, oa = program_signature(program_a)
gb, ob = program_signature(program_b)
return ga == gb and oa == ob
def sig_string(program):
g, o = program_signature(program)
return json.dumps(
{"groups": sorted(g.elements()), "opaque": sorted(o.elements())},
sort_keys=True,
)
# --------------------------------------------------------------------------
# Main
# --------------------------------------------------------------------------
def main():
if len(sys.argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
sys.exit(2)
with open(sys.argv[1]) as f:
data = json.load(f)
predictions = []
for pair in data.get("pairs", []):
pa = pair.get("program_a", [])
pb = pair.get("program_b", [])
try:
eq = equivalent(pa, pb)
except Exception:
eq = False
rec = {"pair_id": pair.get("pair_id"), "equivalent": bool(eq)}
try:
rec["signature_a"] = sig_string(pa)
rec["signature_b"] = sig_string(pb)
except Exception:
pass
predictions.append(rec)
with open(sys.argv[2], "w") as f:
json.dump({"predictions": predictions}, f, indent=2)
if __name__ == "__main__":
main()
/workspace/solve.py
1 #!/usr/bin/env python3
2 """Weak starter baseline for DeepCAD canonical-equivalence.
3
4 This intentionally predicts every pair as non-equivalent. It is useful only as
5 a schema example and should score reward 0.0 on the hidden verifier.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import sys
12 from pathlib import Path
13
14
15 def main() -> int:
16 input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
17 output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
18
19 payload = json.loads(input_path.read_text(encoding="utf-8"))
20 predictions = [
21 {"pair_id": pair["pair_id"], "equivalent": False}
22 for pair in payload.get("pairs", [])
23 ]
24
25 output_path.parent.mkdir(parents=True, exist_ok=True)
26 output_path.write_text(
27 json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
28 encoding="utf-8",
29 )
30 return 0
31
32
33 if __name__ == "__main__":
34 raise SystemExit(main())
35
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a pairs JSON, decides for each pair whether program_a and program_b
describe the same canonical solid, and writes predictions.json.
"""
import json
import re
import sys
from collections import Counter
# --------------------------------------------------------------------------
# Command classification
# --------------------------------------------------------------------------
SUPPORTED = {"param", "sketch", "rect", "circle", "slot", "extrude"}
PROFILE_KINDS = {"rect", "circle", "slot"}
ANNOTATION = {"constraint", "dimension", "note", "metadata", "view", "comment"}
def is_construction(cmd):
return bool(cmd.get("construction", False))
# --------------------------------------------------------------------------
# Numeric helpers
# --------------------------------------------------------------------------
def norm_num(x):
"""Quantise to the 1e-6 tolerance and kill negative zero."""
v = round(float(x), 6) + 0.0
if v == 0:
v = 0.0
return v
# --------------------------------------------------------------------------
# Expression evaluator: numeric constants, params, + - * /, parentheses
# --------------------------------------------------------------------------
_TOKEN_NUM = re.compile(r"[0-9]*\.?[0-9]+([eE][+-]?[0-9]+)?")
_TOKEN_ID = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
def tokenize(s):
tokens = []
i = 0
n = len(s)
while i < n:
c = s[i]
if c.isspace():
i += 1
continue
if c in "+-*/()":
tokens.append(c)
i += 1
continue
if c.isdigit() or c == ".":
m = _TOKEN_NUM.match(s, i)
if not m:
raise ValueError("bad number in %r" % s)
tokens.append(("num", float(m.group())))
i = m.end()
continue
m = _TOKEN_ID.match(s, i)
if m:
tokens.append(("id", m.group()))
i = m.end()
continue
raise ValueError("bad char %r in %r" % (c, s))
return tokens
class Evaluator:
def __init__(self, params):
# params: name -> raw expr (string or number)
self.params = params
self.memo = {}
self.stack = set()
def value(self, name):
if name in self.memo:
return self.memo[name]
if name not in self.params:
raise ValueError("undefined param %r" % name)
if name in self.stack:
raise ValueError("cyclic param %r" % name)
self.stack.add(name)
v = self.eval(self.params[name])
self.stack.discard(name)
self.memo[name] = v
return v
def eval(self, expr):
if isinstance(expr, bool):
raise ValueError("bool not numeric")
if isinstance(expr, (int, float)):
return float(expr)
toks = tokenize(str(expr))
self._toks = toks
self._pos = 0
v = self._expr()
if self._pos != len(toks):
raise ValueError("trailing tokens in %r" % expr)
return v
def _peek(self):
if self._pos < len(self._toks):
return self._toks[self._pos]
return None
def _next(self):
t = self._toks[self._pos]
self._pos += 1
return t
def _expr(self):
v = self._term()
while True:
t = self._peek()
if t == "+":
self._next()
v += self._term()
elif t == "-":
self._next()
v -= self._term()
else:
break
return v
def _term(self):
v = self._factor()
while True:
t = self._peek()
if t == "*":
self._next()
v *= self._factor()
elif t == "/":
self._next()
v /= self._factor()
else:
break
return v
def _factor(self):
t = self._peek()
if t == "+":
self._next()
return self._factor()
if t == "-":
self._next()
return -self._factor()
if t == "(":
self._next()
v = self._expr()
if self._peek() != ")":
raise ValueError("missing )")
self._next()
return v
if isinstance(t, tuple) and t[0] == "num":
self._next()
return t[1]
if isinstance(t, tuple) and t[0] == "id":
self._next()
return self.value(t[1])
raise ValueError("unexpected token %r" % (t,))
# --------------------------------------------------------------------------
# Profile canonicalisation
# --------------------------------------------------------------------------
def canon_field(name, v, ev):
if isinstance(v, list):
return [canon_field(name, x, ev) for x in v]
if isinstance(v, bool):
return v
if isinstance(v, (int, float)):
num = float(v)
elif isinstance(v, str):
try:
num = ev.eval(v)
except Exception:
return v # opaque token inside a profile field
else:
return v
if name == "angle":
num = num % 180.0
return norm_num(num)
def canon_profile(cmd, ev, planes):
plane = planes.get(cmd.get("sketch"))
fields = {}
for k, v in cmd.items():
if k in ("cmd", "id", "sketch", "construction"):
continue
fields[k] = canon_field(k, v, ev)
return json.dumps(
{"kind": cmd["cmd"], "plane": plane, "fields": fields},
sort_keys=True,
)
# --------------------------------------------------------------------------
# Extrude canonicalisation
# --------------------------------------------------------------------------
def canon_direction(v, ev):
if isinstance(v, str):
return v.lower()
if isinstance(v, list):
return [canon_field("dir", x, ev) for x in v]
if isinstance(v, (int, float)) and not isinstance(v, bool):
return norm_num(v)
return v
def canon_extrude_content(cmd, ev, profiles):
op = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = canon_direction(cmd.get("direction", "normal"), ev)
depth = norm_num(ev.eval(cmd.get("depth", 0)))
refs = []
if "profiles" in cmd:
refs.extend(cmd["profiles"])
if "profile" in cmd:
refs.append(cmd["profile"])
prof_sigs = []
for r in refs:
if r in profiles:
prof_sigs.append(profiles[r])
prof_sigs.sort()
content = {
"op": op,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": prof_sigs,
}
return json.dumps(content, sort_keys=True)
def partition_key(cmd):
if "body" in cmd:
return ("L", str(cmd["body"]))
if "channel" in cmd:
return ("L", str(cmd["channel"]))
return ("DEFAULT",)
# --------------------------------------------------------------------------
# Program signature
# --------------------------------------------------------------------------
def program_signature(program):
# Build param table
param_table = {}
for cmd in program:
if cmd.get("cmd") == "param":
param_table[cmd.get("name")] = cmd.get("expr")
ev = Evaluator(param_table)
# Build sketch plane table
planes = {}
for cmd in program:
if cmd.get("cmd") == "sketch":
plane = cmd.get("plane")
if isinstance(plane, str):
plane = plane.upper()
planes[cmd.get("id")] = plane
# Build profile table (non-construction profiles only)
profiles = {}
for cmd in program:
if cmd.get("cmd") in PROFILE_KINDS and not is_construction(cmd):
profiles[cmd.get("id")] = canon_profile(cmd, ev, planes)
# Collect extrude features and opaque payloads
features = [] # (partition_key, content_json)
opaque = Counter()
for cmd in program:
if is_construction(cmd):
continue
kind = cmd.get("cmd")
if kind in ANNOTATION:
continue
if kind == "extrude":
content = canon_extrude_content(cmd, ev, profiles)
features.append((partition_key(cmd), content))
elif kind in SUPPORTED:
continue # param/sketch/profile consumed via lookup tables
else:
payload = {k: v for k, v in cmd.items() if k != "id"}
opaque[json.dumps(payload, sort_keys=True)] += 1
# Group features by partition -> multiset of group-multisets
groups = {}
for key, content in features:
groups.setdefault(key, []).append(content)
group_sigs = Counter()
for key, contents in groups.items():
contents.sort()
group_sigs[json.dumps(contents, sort_keys=True)] += 1
return group_sigs, opaque
def equivalent(program_a, program_b):
ga, oa = program_signature(program_a)
gb, ob = program_signature(program_b)
return ga == gb and oa == ob
def sig_string(program):
g, o = program_signature(program)
return json.dumps(
{"groups": sorted(g.elements()), "opaque": sorted(o.elements())},
sort_keys=True,
)
# --------------------------------------------------------------------------
# Main
# --------------------------------------------------------------------------
def main():
if len(sys.argv) != 3:
sys.stderr.write("usage: solve.py <input_json> <output_json>\n")
return 2
with open(sys.argv[1]) as f:
data = json.load(f)
predictions = []
for pair in data.get("pairs", []):
pa = pair.get("program_a", [])
pb = pair.get("program_b", [])
try:
eq = equivalent(pa, pb)
except Exception:
eq = False
rec = {"pair_id": pair.get("pair_id"), "equivalent": bool(eq)}
try:
rec["signature_a"] = sig_string(pa)
rec["signature_b"] = sig_string(pb)
except Exception:
pass
predictions.append(rec)
with open(sys.argv[2], "w") as f:
json.dump({"predictions": predictions}, f, indent=2)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Run solver on public pairs
python3 solve.py data/public_pairs.json predictions.json && python3 -c "import json; d=json.load(open('predictions.json')); [print(p['pair_id'], p['equivalent']) for p in d['predictions']]"P001 False P002 False P003 True [stdout] P001 False P002 False P003 True
Inspect P001 signatures
python3 -c "
import json
d=json.load(open('predictions.json'))
p=d['predictions'][0]
print('A',p['signature_a'])
print('B',p['signature_b'])
"Exit code 1
A {"groups": ["[\"{\\\"depth\\\": 8.0, \\\"direction\\\": \\\"normal\\\", \\\"extent\\\": \\\"one_side\\\", \\\"op\\\": \\\"new\\\", \\\"profiles\\\": [\\\"{\\\\\\\"fields\\\\\\\": {\\\\\\\"center\\\\\\\": [0.0, 0.0], \\\\\\\"size\\\\\\\": [40.0, 20.0]}, \\\\\\\"kind\\\\\\\": \\\\\\\"rect\\\\\\\", \\\\\\\"plane\\\\\\\": \\\\\\\"XY\\\\\\\"}\\\"]}\"]"], "opaque": []}
Traceback (most recent call last):
File "<string>", line 6, in <module>
KeyError: 'signature_b'
[error] tool reported failureDebug program_b signature
python3 -c "
import json, solve
d=json.load(open('data/public_pairs.json'))
pb=d['pairs'][0]['program_b']
print(solve.sig_string(pb))
"Exit code 1
Traceback (most recent call last):
File "<string>", line 5, in <module>
File "/workspace/solve.py", line 315, in sig_string
g, o = program_signature(program)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/solve.py", line 288, in program_signature
content = canon_extrude_content(cmd, ev, profiles)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/solve.py", line 220, in canon_extrude_content
depth = norm_num(ev.eval(cmd.get("depth", 0)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/solve.py", line 103, in eval
raise ValueError("trailing tokens in %r" % expr)
ValueError: trailing tokens in 'depth'
[error] tool reported failure/workspace/solve.py
def eval(self, expr):
if isinstance(expr, bool):
raise ValueError("bool not numeric")
if isinstance(expr, (int, float)):
return float(expr)
toks = tokenize(str(expr))
self._toks = toks
self._pos = 0
v = self._expr()
if self._pos != len(toks):
raise ValueError("trailing tokens in %r" % expr)
return v
def _peek(self):
if self._pos < len(self._toks):
return self._toks[self._pos]
return None
def _next(self):
t = self._toks[self._pos]
self._pos += 1
return t
def _expr(self):
v = self._term()
while True:
t = self._peek()
if t == "+":
self._next()
v += self._term()
elif t == "-":
self._next()
v -= self._term()
else:
break
return v
def _term(self):
v = self._factor()
while True:
t = self._peek()
if t == "*":
self._next()
v *= self._factor()
elif t == "/":
self._next()
v /= self._factor()
else:
break
return v
def _factor(self):
t = self._peek()
if t == "+":
self._next()
return self._factor()
if t == "-":
self._next()
return -self._factor()
if t == "(":
self._next()
v = self._expr()
if self._peek() != ")":
raise ValueError("missing )")
self._next()
return v
if isinstance(t, tuple) and t[0] == "num":
self._next()
return t[1]
if isinstance(t, tuple) and t[0] == "id":
self._next()
return self.value(t[1])
raise ValueError("unexpected token %r" % (t,)) def eval(self, expr):
if isinstance(expr, bool):
raise ValueError("bool not numeric")
if isinstance(expr, (int, float)):
return float(expr)
toks = tokenize(str(expr))
pos = [0]
def peek():
return toks[pos[0]] if pos[0] < len(toks) else None
def nxt():
t = toks[pos[0]]
pos[0] += 1
return t
def p_expr():
v = p_term()
while True:
t = peek()
if t == "+":
nxt()
v += p_term()
elif t == "-":
nxt()
v -= p_term()
else:
break
return v
def p_term():
v = p_factor()
while True:
t = peek()
if t == "*":
nxt()
v *= p_factor()
elif t == "/":
nxt()
v /= p_factor()
else:
break
return v
def p_factor():
t = peek()
if t == "+":
nxt()
return p_factor()
if t == "-":
nxt()
return -p_factor()
if t == "(":
nxt()
v = p_expr()
if peek() != ")":
raise ValueError("missing )")
nxt()
return v
if isinstance(t, tuple) and t[0] == "num":
nxt()
return t[1]
if isinstance(t, tuple) and t[0] == "id":
nxt()
return self.value(t[1])
raise ValueError("unexpected token %r" % (t,))
v = p_expr()
if pos[0] != len(toks):
raise ValueError("trailing tokens in %r" % expr)
return vRe-run solver
python3 solve.py data/public_pairs.json predictions.json && python3 -c "import json; d=json.load(open('predictions.json')); [print(p['pair_id'], p['equivalent']) for p in d['predictions']]"P001 True P002 False P003 True [stdout] P001 True P002 False P003 True
Validate edge-case rules
python3 -c "
import solve
def eq(a,b): return solve.equivalent(a,b)
# 1. opaque multiset, id stripped, order-independent, literal
a=[{'cmd':'fillet','id':'f1','edges':['e1'],'radius':2},{'cmd':'chamfer','id':'c','dist':1}]
b=[{'cmd':'chamfer','id':'zzz','dist':1},{'cmd':'fillet','id':'other','edges':['e1'],'radius':2}]
print('opaque reorder eq:', eq(a,b)) # True
# 2. opaque literal: numeric 2 vs 2.0 -> different (no tolerance)
a=[{'cmd':'fillet','radius':2}]
b=[{'cmd':'fillet','radius':2.0}]
print('opaque 2 vs 2.0:', eq(a,b)) # False
# 3. plane XZ vs ZX different
a=[{'cmd':'sketch','id':'s','plane':'XZ'},{'cmd':'circle','id':'c','sketch':'s','center':[0,0],'radius':1},{'cmd':'extrude','id':'e','profile':'c','depth':5}]
b=[{'cmd':'sketch','id':'s','plane':'ZX'},{'cmd':'circle','id':'c','sketch':'s','center':[0,0],'radius':1},{'cmd':'extrude','id':'e','profile':'c','depth':5}]
print('XZ vs ZX:', eq(a,b)) # False
# 4. depth sign / direction not folded
a=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'circle','id':'c','sketch':'s','center':[0,0],'radius':1},{'cmd':'extrude','id':'e','profile':'c','depth':5,'direction':'normal'}]
b=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'circle','id':'c','sketch':'s','center':[0,0],'radius':1},{'cmd':'extrude','id':'e','profile':'c','depth':-5,'direction':'reverse'}]
print('depth sign fold:', eq(a,b)) # False
# 5. slot angle mod 180
a=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'slot','id':'sl','sketch':'s','center':[0,0],'length':10,'width':2,'angle':30},{'cmd':'extrude','id':'e','profile':'sl','depth':5}]
b=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'slot','id':'sl','sketch':'s','center':[0,0],'length':10,'width':2,'angle':210},{'cmd':'extrude','id':'e','profile':'sl','depth':5}]
print('slot angle mod180:', eq(a,b)) # True
# 6. partition topology: grouped vs separate
prof=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'circle','id':'c','sketch':'s','center':[0,0],'radius':1}]
a=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5,'body':'g'},{'cmd':'extrude','id':'e2','profile':'c','depth':5,'body':'g'}]
b=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5,'body':'g'},{'cmd':'extrude','id':'e2','profile':'c','depth':5,'body':'h'}]
print('grouped vs separate:', eq(a,b)) # False
# 7. body vs channel same grouping -> equivalent
a=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5,'body':'g'},{'cmd':'extrude','id':'e2','profile':'c','depth':5,'body':'g'}]
b=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5,'channel':'x'},{'cmd':'extrude','id':'e2','profile':'c','depth':5,'channel':'x'}]
print('body vs channel same group:', eq(a,b)) # True
# 8. add vs join distinct operations
a=prof+[{'cmd':'extrude','id':'e','profile':'c','depth':5,'operation':'add'}]
b=prof+[{'cmd':'extrude','id':'e','profile':'c','depth':5,'operation':'join'}]
print('add vs join:', eq(a,b)) # False
# 9. construction command ignored
a=prof+[{'cmd':'extrude','id':'e','profile':'c','depth':5}]
b=prof+[{'cmd':'extrude','id':'e','profile':'c','depth':5},{'cmd':'fillet','construction':True,'radius':9}]
print('construction ignored:', eq(a,b)) # True
# 10. annotation ignored
b2=prof+[{'cmd':'extrude','id':'e','profile':'c','depth':5},{'cmd':'note','text':'hi'}]
print('annotation ignored:', eq(a,b2)) # True
"opaque reorder eq: True opaque 2 vs 2.0: False XZ vs ZX: False depth sign fold: False slot angle mod180: True grouped vs separate: False body vs channel same group: True add vs join: False construction ignored: True annotation ignored: True [stdout] opaque reorder eq: True opaque 2 vs 2.0: False XZ vs ZX: False depth sign fold: False slot angle mod180: True grouped vs separate: False body vs channel same group: True add vs join: False construction ignored: True annotation ignored: True
Validate partition and ordering edge cases
python3 -c "
import solve
prof=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'circle','id':'c','sketch':'s','center':[0,0],'radius':1}]
# default (unlabeled, grouped together) vs both-labeled-same -> same topology -> equivalent
a=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5},{'cmd':'extrude','id':'e2','profile':'c','depth':5}]
b=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5,'body':'g'},{'cmd':'extrude','id':'e2','profile':'c','depth':5,'body':'g'}]
print('default vs labeled same topology:', solve.equivalent(a,b)) # True
# unlabeled does not alias labeled: A one unlabeled+one labeled (2 separate groups) vs B both same label (1 group)
a=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5},{'cmd':'extrude','id':'e2','profile':'c','depth':5,'body':'g'}]
b=prof+[{'cmd':'extrude','id':'e1','profile':'c','depth':5,'body':'g'},{'cmd':'extrude','id':'e2','profile':'c','depth':5,'body':'g'}]
print('no-alias case:', solve.equivalent(a,b)) # False
# rect size ordering
a=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[40,20]},{'cmd':'extrude','id':'e','profile':'r','depth':5}]
b=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[20,40]},{'cmd':'extrude','id':'e','profile':'r','depth':5}]
print('rect 40x20 vs 20x40:', solve.equivalent(a,b)) # False
# unused profile does not matter
a=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[40,20]},{'cmd':'circle','id':'unused','sketch':'s','center':[9,9],'radius':3},{'cmd':'extrude','id':'e','profile':'r','depth':5}]
b=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[40,20]},{'cmd':'extrude','id':'e','profile':'r','depth':5}]
print('unused profile ignored:', solve.equivalent(a,b)) # True
"default vs labeled same topology: True no-alias case: False rect 40x20 vs 20x40: False unused profile ignored: True [stdout] default vs labeled same topology: True no-alias case: False rect 40x20 vs 20x40: False unused profile ignored: True
Final validation of predictions artifact
python3 solve.py data/public_pairs.json predictions.json && python3 -c "
import json
d=json.load(open('predictions.json'))
assert set(d)=={'predictions'}
for p in d['predictions']:
assert isinstance(p['equivalent'], bool)
print(p['pair_id'], p['equivalent'])
print('schema OK')
"P001 True P002 False P003 True schema OK [stdout] P001 True P002 False P003 True schema OK
No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_62f7498d84254252. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_62f7498d84254252 · verifier authoritative; classifier explanatory.