tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it./workspace/data/public_pairs.json
1 {
2 "schema_version": "1.0",
3 "description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
4 "pairs": [
5 {
6 "pair_id": "P001",
7 "program_a": [
8 {"cmd": "param", "name": "w", "expr": "40"},
9 {"cmd": "param", "name": "h", "expr": "20"},
10 {"cmd": "param", "name": "d", "expr": "8"},
11 {"cmd": "sketch", "id": "s0", "plane": "XY"},
12 {"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
13 {"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
14 {"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
15 ],
16 "program_b": [
17 {"cmd": "param", "name": "depth", "expr": "4 + 4"},
18 {"cmd": "param", "name": "height", "expr": "5 * 4"},
19 {"cmd": "param", "name": "width", "expr": "20 * 2"},
20 {"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
21 {"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
22 {"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
23 ]
24 },
25 {
26 "pair_id": "P002",
27 "program_a": [
28 {"cmd": "param", "name": "r", "expr": "4"},
29 {"cmd": "sketch", "id": "s", "plane": "XY"},
30 {"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
31 {"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
32 ],
33 "program_b": [
34 {"cmd": "param", "name": "r2", "expr": "4.25"},
35 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
36 {"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
37 {"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
38 ]
39 },
40 {
41 "pair_id": "P003",
42 "program_a": [
43 {"cmd": "sketch", "id": "s", "plane": "XY"},
44 {"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
45 {"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
46 {"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
47 ],
48 "program_b": [
49 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
50 {"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
51 {"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
52 {"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
53 ]
54 }
55 ]
56 }
57
ls -la /workspace/ && echo "---" && ls -la /workspace/data/ && echo "---py---" && cat /workspace/solve.py 2>/dev/null || echo "no solve.py yet"
total 24
drwxr-xr-x 1 root root 4096 Jul 6 02:01 .
drwxr-xr-x 1 root root 4096 Jul 6 02:01 ..
-rw-r--r-- 1 root root 299 Jul 4 05:02 README.md
drwxr-xr-x 2 root root 4096 Jul 4 05:02 data
drwxr-xr-x 2 root root 4096 Jul 6 02:01 output
-rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py
---
total 12
drwxr-xr-x 2 root root 4096 Jul 4 05:02 .
drwxr-xr-x 1 root root 4096 Jul 6 02:01 ..
-rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json
---py---
#!/usr/bin/env python3
"""Weak starter baseline for DeepCAD canonical-equivalence.
This intentionally predicts every pair as non-equivalent. It is useful only as
a schema example and should score reward 0.0 on the hidden verifier.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = [
{"pair_id": pair["pair_id"], "equivalent": False}
for pair in payload.get("pairs", [])
]
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
[stdout]
total 24
drwxr-xr-x 1 root root 4096 Jul 6 02:01 .
drwxr-xr-x 1 root root 4096 Jul 6 02:01 ..
-rw-r--r-- 1 root root 299 Jul 4 05:02 README.md
drwxr-xr-x 2 root root 4096 Jul 4 05:02 data
drwxr-xr-x 2 root root 4096 Jul 6 02:01 output
-rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py
---
total 12
drwxr-xr-x 2 root root 4096 Jul 4 05:02 .
drwxr-xr-x 1 root root 4096 Jul 6 02:01 ..
-rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json
---py---
#!/usr/bin/env python3
"""Weak starter baseline for DeepCAD canonical-equivalence.
This intentionally predicts every pair as non-equivalent. It is useful only as
a schema example and should score reward 0.0 on the hidden verifier.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = [
{"pair_id": pair["pair_id"], "equivalent": False}
for pair in payload.get("pairs", [])
]
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())/workspace/README.md
1 # DeepCAD Canonical Equivalence Starter 2 3 `solve.py` is an intentionally weak starter baseline. It reads an input JSON with program pairs and writes a prediction for each pair. 4 5 The public file `data/public_pairs.json` is unlabeled and only shows the command schema. Hidden labels are verifier-only. 6
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
For each pair of compact CAD command programs, decide whether both programs
describe the same canonical solid.
Approach: normalize each program into a canonical *signature* that captures only
what affects the solid, then compare the two signatures for equality.
The solid is determined by:
* the multiset of *extrude* features (each canonicalized: operation, extent,
direction, depth, and the multiset of consumed profile geometries), grouped
into body/channel partitions; and
* the multiset of *opaque* solid-affecting payload commands compared literally.
Everything else (parameter names, ids, sketch ids, command order, construction
geometry, annotation commands) is normalized away or dropped.
"""
from __future__ import annotations
import ast
import json
import sys
from pathlib import Path
# Absolute tolerance for numeric comparison is 1e-6; rounding to 6 decimals
# yields a grid finer than that tolerance.
ROUND_DIGITS = 6
SUPPORTED_KINDS = {"param", "sketch", "rect", "circle", "slot", "extrude"}
PROFILE_KINDS = {"rect", "circle", "slot"}
ANNOTATION_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
# Fields on a profile command that are labels/wiring, not geometry.
PROFILE_NON_GEOM = {"cmd", "kind", "type", "id", "sketch", "construction", "plane"}
# --------------------------------------------------------------------------- #
# Expression evaluation
# --------------------------------------------------------------------------- #
class ExprError(Exception):
pass
def _eval_ast(node, resolver):
if isinstance(node, ast.Expression):
return _eval_ast(node.body, resolver)
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ExprError("non-numeric constant")
return float(node.value)
if isinstance(node, ast.Num): # pragma: no cover (py<3.8 compat)
return float(node.n)
if isinstance(node, ast.BinOp):
left = _eval_ast(node.left, resolver)
right = _eval_ast(node.right, resolver)
if isinstance(node.op, ast.Add):
return left + right
if isinstance(node.op, ast.Sub):
return left - right
if isinstance(node.op, ast.Mult):
return left * right
if isinstance(node.op, ast.Div):
return left / right
raise ExprError("unsupported binary operator")
if isinstance(node, ast.UnaryOp):
operand = _eval_ast(node.operand, resolver)
if isinstance(node.op, ast.UAdd):
return +operand
if isinstance(node.op, ast.USub):
return -operand
raise ExprError("unsupported unary operator")
if isinstance(node, ast.Name):
return resolver(node.id)
raise ExprError("unsupported expression node")
def eval_expr(expr, resolver):
tree = ast.parse(expr, mode="eval")
return _eval_ast(tree, resolver)
class ParamTable:
"""Resolves parameter names to numeric values, allowing references between
parameters (in any order) and detecting cycles."""
def __init__(self, exprs):
self._exprs = exprs
self._cache = {}
def resolve(self, name, stack=None):
if name in self._cache:
return self._cache[name]
if name not in self._exprs:
raise ExprError(f"unknown parameter: {name}")
if stack is None:
stack = ()
if name in stack:
raise ExprError(f"cyclic parameter: {name}")
raw = self._exprs[name]
value = self._eval_value(raw, stack + (name,))
self._cache[name] = value
return value
def _eval_value(self, raw, stack):
if isinstance(raw, bool):
raise ExprError("boolean parameter")
if isinstance(raw, (int, float)):
return float(raw)
if isinstance(raw, str):
tree = ast.parse(raw, mode="eval")
return _eval_ast(tree, lambda n: self.resolve(n, stack))
raise ExprError("unsupported parameter expression")
def resolver(self):
return lambda name: self.resolve(name)
def evaluate_value(value, params):
"""Evaluate a JSON value (number, expression string, or nested list) to a
canonical numeric form. If a string cannot be evaluated as an arithmetic
expression, it is preserved literally."""
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return round(float(value), ROUND_DIGITS)
if isinstance(value, list):
return [evaluate_value(v, params) for v in value]
if isinstance(value, str):
try:
return round(eval_expr(value, params.resolver()), ROUND_DIGITS)
except (ExprError, SyntaxError, ZeroDivisionError, ValueError, TypeError):
return value
return value
# --------------------------------------------------------------------------- #
# Command helpers
# --------------------------------------------------------------------------- #
def cmd_kind(command):
for key in ("cmd", "kind", "type"):
if key in command and isinstance(command[key], str):
return command[key].lower()
return None
def is_construction(command):
return command.get("construction") is True
def normalize_plane(plane):
if isinstance(plane, str):
return plane.upper()
return plane
def canonical_json(obj):
return json.dumps(obj, sort_keys=True, separators=(",", ":"))
# --------------------------------------------------------------------------- #
# Signature construction
# --------------------------------------------------------------------------- #
def build_signature(program):
if not isinstance(program, list):
program = []
# 1. Parameters (register regardless of construction flag).
param_exprs = {}
for command in program:
if cmd_kind(command) == "param" and "name" in command:
param_exprs[command["name"]] = command.get("expr")
params = ParamTable(param_exprs)
# 2. Sketch planes.
sketch_planes = {}
for command in program:
if cmd_kind(command) == "sketch" and "id" in command:
sketch_planes[command["id"]] = normalize_plane(command.get("plane"))
# 3. Profiles (skip construction profiles).
profiles = {}
for command in program:
kind = cmd_kind(command)
if kind in PROFILE_KINDS and not is_construction(command) and "id" in command:
profiles[command["id"]] = canonical_profile(command, kind, sketch_planes, params)
# 4. Extrude features and 5. opaque payloads.
features = [] # list of (partition_key, feature_sig)
opaque = []
for command in program:
if is_construction(command):
continue
kind = cmd_kind(command)
if kind == "extrude":
features.append(canonical_extrude(command, profiles, params))
elif kind in SUPPORTED_KINDS or kind in ANNOTATION_KINDS:
continue # param/sketch/profile already handled; annotations ignored
else:
opaque.append(canonical_opaque(command))
# 6. Partition grouping.
default_partition = []
labeled = {}
for partition_key, feature_sig in features:
if partition_key is None:
default_partition.append(feature_sig)
else:
labeled.setdefault(partition_key, []).append(feature_sig)
default_partition.sort()
labeled_partitions = sorted(canonical_json(sorted(group)) for group in labeled.values())
opaque.sort()
return {
"default_partition": default_partition,
"labeled_partitions": labeled_partitions,
"opaque": opaque,
}
def canonical_profile(command, kind, sketch_planes, params):
plane = command.get("plane")
if plane is None:
plane = sketch_planes.get(command.get("sketch"))
plane = normalize_plane(plane)
fields = {}
for key, value in command.items():
if key in PROFILE_NON_GEOM:
continue
evaluated = evaluate_value(value, params)
if key == "angle" and isinstance(evaluated, (int, float)) and not isinstance(evaluated, bool):
evaluated = round(((evaluated % 180.0) + 180.0) % 180.0, ROUND_DIGITS)
fields[key] = evaluated
return canonical_json({"type": kind, "plane": plane, "fields": fields})
def canonical_extrude(command, profiles, params):
operation = command.get("operation", "new")
extent = command.get("extent", "one_side")
direction = command.get("direction", "normal")
operation = operation.lower() if isinstance(operation, str) else operation
extent = extent.lower() if isinstance(extent, str) else extent
direction = direction.lower() if isinstance(direction, str) else direction
depth = command.get("depth")
depth = evaluate_value(depth, params) if depth is not None else None
# Resolve consumed profiles (order not semantic).
ref_ids = []
if "profiles" in command and isinstance(command["profiles"], list):
ref_ids.extend(command["profiles"])
if "profile" in command and command["profile"] is not None:
ref_ids.append(command["profile"])
profile_sigs = []
for pid in ref_ids:
if pid in profiles:
profile_sigs.append(profiles[pid])
profile_sigs.sort()
feature = {
"operation": operation,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": profile_sigs,
}
feature_sig = canonical_json(feature)
# Partition key: body or channel value (same namespace); None = default.
partition_key = None
if command.get("body") is not None:
partition_key = ("v", command["body"])
elif command.get("channel") is not None:
partition_key = ("v", command["channel"])
return partition_key, feature_sig
def canonical_opaque(command):
payload = {k: v for k, v in command.items() if k != "id"}
return canonical_json(payload)
# --------------------------------------------------------------------------- #
# Main
# --------------------------------------------------------------------------- #
def decide(pair):
sig_a = build_signature(pair.get("program_a", []))
sig_b = build_signature(pair.get("program_b", []))
equivalent = sig_a == sig_b
return equivalent, sig_a, sig_b
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = []
for pair in payload.get("pairs", []):
equivalent, sig_a, sig_b = decide(pair)
predictions.append(
{
"pair_id": pair["pair_id"],
"equivalent": bool(equivalent),
"signature_a": canonical_json(sig_a),
"signature_b": canonical_json(sig_b),
}
)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
/workspace/solve.py
1 #!/usr/bin/env python3 2 """Weak starter baseline for DeepCAD canonical-equivalence. 3
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
For each pair of compact CAD command programs, decide whether both programs
describe the same canonical solid.
Approach: normalize each program into a canonical *signature* that captures only
what affects the solid, then compare the two signatures for equality.
The solid is determined by:
* the multiset of *extrude* features (each canonicalized: operation, extent,
direction, depth, and the multiset of consumed profile geometries), grouped
into body/channel partitions; and
* the multiset of *opaque* solid-affecting payload commands compared literally.
Everything else (parameter names, ids, sketch ids, command order, construction
geometry, annotation commands) is normalized away or dropped.
"""
from __future__ import annotations
import ast
import json
import sys
from pathlib import Path
# Absolute tolerance for numeric comparison is 1e-6; rounding to 6 decimals
# yields a grid finer than that tolerance.
ROUND_DIGITS = 6
SUPPORTED_KINDS = {"param", "sketch", "rect", "circle", "slot", "extrude"}
PROFILE_KINDS = {"rect", "circle", "slot"}
ANNOTATION_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
# Fields on a profile command that are labels/wiring, not geometry.
PROFILE_NON_GEOM = {"cmd", "kind", "type", "id", "sketch", "construction", "plane"}
# --------------------------------------------------------------------------- #
# Expression evaluation
# --------------------------------------------------------------------------- #
class ExprError(Exception):
pass
def _eval_ast(node, resolver):
if isinstance(node, ast.Expression):
return _eval_ast(node.body, resolver)
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ExprError("non-numeric constant")
return float(node.value)
if isinstance(node, ast.Num): # pragma: no cover (py<3.8 compat)
return float(node.n)
if isinstance(node, ast.BinOp):
left = _eval_ast(node.left, resolver)
right = _eval_ast(node.right, resolver)
if isinstance(node.op, ast.Add):
return left + right
if isinstance(node.op, ast.Sub):
return left - right
if isinstance(node.op, ast.Mult):
return left * right
if isinstance(node.op, ast.Div):
return left / right
raise ExprError("unsupported binary operator")
if isinstance(node, ast.UnaryOp):
operand = _eval_ast(node.operand, resolver)
if isinstance(node.op, ast.UAdd):
return +operand
if isinstance(node.op, ast.USub):
return -operand
raise ExprError("unsupported unary operator")
if isinstance(node, ast.Name):
return resolver(node.id)
raise ExprError("unsupported expression node")
def eval_expr(expr, resolver):
tree = ast.parse(expr, mode="eval")
return _eval_ast(tree, resolver)
class ParamTable:
"""Resolves parameter names to numeric values, allowing references between
parameters (in any order) and detecting cycles."""
def __init__(self, exprs):
self._exprs = exprs
self._cache = {}
def resolve(self, name, stack=None):
if name in self._cache:
return self._cache[name]
if name not in self._exprs:
raise ExprError(f"unknown parameter: {name}")
if stack is None:
stack = ()
if name in stack:
raise ExprError(f"cyclic parameter: {name}")
raw = self._exprs[name]
value = self._eval_value(raw, stack + (name,))
self._cache[name] = value
return value
def _eval_value(self, raw, stack):
if isinstance(raw, bool):
raise ExprError("boolean parameter")
if isinstance(raw, (int, float)):
return float(raw)
if isinstance(raw, str):
tree = ast.parse(raw, mode="eval")
return _eval_ast(tree, lambda n: self.resolve(n, stack))
raise ExprError("unsupported parameter expression")
def resolver(self):
return lambda name: self.resolve(name)
def evaluate_value(value, params):
"""Evaluate a JSON value (number, expression string, or nested list) to a
canonical numeric form. If a string cannot be evaluated as an arithmetic
expression, it is preserved literally."""
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return round(float(value), ROUND_DIGITS)
if isinstance(value, list):
return [evaluate_value(v, params) for v in value]
if isinstance(value, str):
try:
return round(eval_expr(value, params.resolver()), ROUND_DIGITS)
except (ExprError, SyntaxError, ZeroDivisionError, ValueError, TypeError):
return value
return value
# --------------------------------------------------------------------------- #
# Command helpers
# --------------------------------------------------------------------------- #
def cmd_kind(command):
for key in ("cmd", "kind", "type"):
if key in command and isinstance(command[key], str):
return command[key].lower()
return None
def is_construction(command):
return command.get("construction") is True
def normalize_plane(plane):
if isinstance(plane, str):
return plane.upper()
return plane
def canonical_json(obj):
return json.dumps(obj, sort_keys=True, separators=(",", ":"))
# --------------------------------------------------------------------------- #
# Signature construction
# --------------------------------------------------------------------------- #
def build_signature(program):
if not isinstance(program, list):
program = []
# 1. Parameters (register regardless of construction flag).
param_exprs = {}
for command in program:
if isinstance(command, dict) and cmd_kind(command) == "param" and "name" in command:
param_exprs[command["name"]] = command.get("expr")
params = ParamTable(param_exprs)
# 2. Sketch planes.
sketch_planes = {}
for command in program:
if isinstance(command, dict) and cmd_kind(command) == "sketch" and "id" in command:
sketch_planes[command["id"]] = normalize_plane(command.get("plane"))
# 3. Profiles (skip construction profiles).
profiles = {}
for command in program:
if not isinstance(command, dict):
continue
kind = cmd_kind(command)
if kind in PROFILE_KINDS and not is_construction(command) and "id" in command:
profiles[command["id"]] = canonical_profile(command, kind, sketch_planes, params)
# 4. Extrude features and 5. opaque payloads.
features = [] # list of (partition_key, feature_sig)
opaque = []
for command in program:
if not isinstance(command, dict):
continue
if is_construction(command):
continue
kind = cmd_kind(command)
if kind == "extrude":
features.append(canonical_extrude(command, profiles, params))
elif kind in SUPPORTED_KINDS or kind in ANNOTATION_KINDS:
continue # param/sketch/profile already handled; annotations ignored
else:
opaque.append(canonical_opaque(command))
# 6. Partition grouping.
default_partition = []
labeled = {}
for partition_key, feature_sig in features:
if partition_key is None:
default_partition.append(feature_sig)
else:
labeled.setdefault(partition_key, []).append(feature_sig)
default_partition.sort()
labeled_partitions = sorted(canonical_json(sorted(group)) for group in labeled.values())
opaque.sort()
return {
"default_partition": default_partition,
"labeled_partitions": labeled_partitions,
"opaque": opaque,
}
def canonical_profile(command, kind, sketch_planes, params):
plane = command.get("plane")
if plane is None:
plane = sketch_planes.get(command.get("sketch"))
plane = normalize_plane(plane)
fields = {}
for key, value in command.items():
if key in PROFILE_NON_GEOM:
continue
evaluated = evaluate_value(value, params)
if key == "angle" and isinstance(evaluated, (int, float)) and not isinstance(evaluated, bool):
evaluated = round(((evaluated % 180.0) + 180.0) % 180.0, ROUND_DIGITS)
fields[key] = evaluated
return canonical_json({"type": kind, "plane": plane, "fields": fields})
def canonical_extrude(command, profiles, params):
operation = command.get("operation", "new")
extent = command.get("extent", "one_side")
direction = command.get("direction", "normal")
operation = operation.lower() if isinstance(operation, str) else operation
extent = extent.lower() if isinstance(extent, str) else extent
direction = direction.lower() if isinstance(direction, str) else direction
depth = command.get("depth")
depth = evaluate_value(depth, params) if depth is not None else None
# Resolve consumed profiles (order not semantic).
ref_ids = []
if "profiles" in command and isinstance(command["profiles"], list):
ref_ids.extend(command["profiles"])
if "profile" in command and command["profile"] is not None:
ref_ids.append(command["profile"])
profile_sigs = []
for pid in ref_ids:
if pid in profiles:
profile_sigs.append(profiles[pid])
profile_sigs.sort()
feature = {
"operation": operation,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": profile_sigs,
}
feature_sig = canonical_json(feature)
# Partition key: body or channel value (same namespace); None = default.
partition_key = None
if command.get("body") is not None:
partition_key = ("v", command["body"])
elif command.get("channel") is not None:
partition_key = ("v", command["channel"])
return partition_key, feature_sig
def canonical_opaque(command):
payload = {k: v for k, v in command.items() if k != "id"}
return canonical_json(payload)
# --------------------------------------------------------------------------- #
# Main
# --------------------------------------------------------------------------- #
def decide(pair):
sig_a = build_signature(pair.get("program_a", []))
sig_b = build_signature(pair.get("program_b", []))
equivalent = sig_a == sig_b
return equivalent, sig_a, sig_b
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = []
for pair in payload.get("pairs", []):
equivalent, sig_a, sig_b = decide(pair)
predictions.append(
{
"pair_id": pair["pair_id"],
"equivalent": bool(equivalent),
"signature_a": canonical_json(sig_a),
"signature_b": canonical_json(sig_b),
}
)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
python3 solve.py data/public_pairs.json predictions.json && echo "=== predictions ===" && python3 -c "
import json
d=json.load(open('predictions.json'))
for p in d['predictions']:
print(p['pair_id'], p['equivalent'])
"=== predictions === P001 True P002 False P003 True [stdout] === predictions === P001 True P002 False P003 True
python3 -c "
import solve, json
def eq(a,b): return solve.build_signature(a)==solve.build_signature(b)
# body vs channel same grouping -> equivalent
A=[{'cmd':'extrude','profile':'p','operation':'add','depth':5,'body':'g1'},
{'cmd':'extrude','profile':'p','operation':'add','depth':5,'body':'g1'}]
B=[{'cmd':'extrude','profile':'p','operation':'add','depth':5,'channel':'c9'},
{'cmd':'extrude','profile':'p','operation':'add','depth':5,'channel':'c9'}]
print('body==channel grouping:', eq(A,B)) # expect True
# shared partition vs separate partitions -> different
C=[{'cmd':'extrude','profile':'p','operation':'add','depth':5,'body':'g1'},
{'cmd':'extrude','profile':'p','operation':'add','depth':5,'body':'g2'}]
print('shared vs separate:', eq(A,C)) # expect False
# default unlabeled vs labeled distinction
D=[{'cmd':'extrude','profile':'p','operation':'add','depth':5}]
E=[{'cmd':'extrude','profile':'p','operation':'add','depth':5,'body':'x'}]
print('default vs labeled single:', eq(D,E)) # expect False
# depth sign preserved
F=[{'cmd':'extrude','profile':'p','depth':5}]
G=[{'cmd':'extrude','profile':'p','depth':-5}]
print('depth sign:', eq(F,G)) # expect False
# direction not folded
H=[{'cmd':'extrude','profile':'p','depth':5,'direction':'normal'}]
I=[{'cmd':'extrude','profile':'p','depth':5,'direction':'reverse'}]
print('direction diff:', eq(H,I)) # expect False
# add vs join distinct
J=[{'cmd':'extrude','profile':'p','operation':'add','depth':5}]
K=[{'cmd':'extrude','profile':'p','operation':'join','depth':5}]
print('add vs join:', eq(J,K)) # expect False
# slot angle mod 180
sA=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'slot','id':'sl','sketch':'s','center':[0,0],'length':10,'width':2,'angle':10},{'cmd':'extrude','profile':'sl','depth':3}]
sB=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'slot','id':'sl','sketch':'s','center':[0,0],'length':10,'width':2,'angle':190},{'cmd':'extrude','profile':'sl','depth':3}]
print('slot angle mod180:', eq(sA,sB)) # expect True
# opaque payload multiset, order-insensitive, id stripped
oA=[{'cmd':'fillet','id':'f1','edges':['e1'],'radius':2},{'cmd':'chamfer','id':'c1','dist':1}]
oB=[{'cmd':'chamfer','id':'zzz','dist':1},{'cmd':'fillet','id':'q','edges':['e1'],'radius':2}]
print('opaque reorder+id:', eq(oA,oB)) # expect True
# opaque literal difference (no tolerance)
oC=[{'cmd':'fillet','id':'f1','radius':2}]
oD=[{'cmd':'fillet','id':'f1','radius':2.0000001}]
print('opaque no tolerance:', eq(oC,oD)) # expect False
# param expr equivalence with tolerance & renaming
pA=[{'cmd':'param','name':'w','expr':'40'},{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':['w',20]},{'cmd':'extrude','profile':'r','depth':'8'}]
pB=[{'cmd':'param','name':'a','expr':'80/2'},{'cmd':'sketch','id':'q','plane':'xy'},{'cmd':'rect','id':'z','sketch':'q','center':[0,0],'size':['a',20]},{'cmd':'extrude','profile':'z','depth':'4+4'}]
print('param eval+plane case:', eq(pA,pB)) # expect True
# XZ vs ZX different
zA=[{'cmd':'sketch','id':'s','plane':'XZ'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[2,3]},{'cmd':'extrude','profile':'r','depth':1}]
zB=[{'cmd':'sketch','id':'s','plane':'ZX'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[2,3]},{'cmd':'extrude','profile':'r','depth':1}]
print('XZ vs ZX:', eq(zA,zB)) # expect False
# construction & annotations ignored
cA=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[2,3]},{'cmd':'line','id':'g','construction':True},{'cmd':'note','text':'hi'},{'cmd':'extrude','profile':'r','depth':1}]
cB=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[2,3]},{'cmd':'extrude','profile':'r','depth':1}]
print('construction/annotation ignored:', eq(cA,cB)) # expect True
# rect size ordering matters
rA=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[2,3]},{'cmd':'extrude','profile':'r','depth':1}]
rB=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[3,2]},{'cmd':'extrude','profile':'r','depth':1}]
print('rect size order:', eq(rA,rB)) # expect False
# unused profile does not affect solid
uA=[{'cmd':'sketch','id':'s','plane':'XY'},{'cmd':'rect','id':'r','sketch':'s','center':[0,0],'size':[2,3]},{'cmd':'circle','id':'unused','sketch':'s','center':[9,9],'radius':1},{'cmd':'extrude','profile':'r','depth':1}]
print('unused profile:', eq(uA,rB) or eq(uA,cB)) # expect True (matches cB)
"
body==channel grouping: True shared vs separate: False default vs labeled single: False depth sign: False direction diff: False add vs join: False slot angle mod180: True opaque reorder+id: True opaque no tolerance: False param eval+plane case: True XZ vs ZX: False construction/annotation ignored: True rect size order: False unused profile: True [stdout] body==channel grouping: True shared vs separate: False default vs labeled single: False depth sign: False direction diff: False add vs join: False slot angle mod180: True opaque reorder+id: True opaque no tolerance: False param eval+plane case: True XZ vs ZX: False construction/annotation ignored: True rect size order: False unused profile: True
No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_1421bf7acde24029. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_1421bf7acde24029 · verifier authoritative; classifier explanatory.