tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it./workspace/data/public_pairs.json
1 {
2 "schema_version": "1.0",
3 "description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
4 "pairs": [
5 {
6 "pair_id": "P001",
7 "program_a": [
8 {"cmd": "param", "name": "w", "expr": "40"},
9 {"cmd": "param", "name": "h", "expr": "20"},
10 {"cmd": "param", "name": "d", "expr": "8"},
11 {"cmd": "sketch", "id": "s0", "plane": "XY"},
12 {"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
13 {"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
14 {"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
15 ],
16 "program_b": [
17 {"cmd": "param", "name": "depth", "expr": "4 + 4"},
18 {"cmd": "param", "name": "height", "expr": "5 * 4"},
19 {"cmd": "param", "name": "width", "expr": "20 * 2"},
20 {"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
21 {"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
22 {"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
23 ]
24 },
25 {
26 "pair_id": "P002",
27 "program_a": [
28 {"cmd": "param", "name": "r", "expr": "4"},
29 {"cmd": "sketch", "id": "s", "plane": "XY"},
30 {"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
31 {"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
32 ],
33 "program_b": [
34 {"cmd": "param", "name": "r2", "expr": "4.25"},
35 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
36 {"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
37 {"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
38 ]
39 },
40 {
41 "pair_id": "P003",
42 "program_a": [
43 {"cmd": "sketch", "id": "s", "plane": "XY"},
44 {"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
45 {"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
46 {"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
47 ],
48 "program_b": [
49 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
50 {"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
51 {"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
52 {"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
53 ]
54 }
55 ]
56 }
57
/workspace/solve.py
1 #!/usr/bin/env python3
2 """Weak starter baseline for DeepCAD canonical-equivalence.
3
4 This intentionally predicts every pair as non-equivalent. It is useful only as
5 a schema example and should score reward 0.0 on the hidden verifier.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import sys
12 from pathlib import Path
13
14
15 def main() -> int:
16 input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
17 output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
18
19 payload = json.loads(input_path.read_text(encoding="utf-8"))
20 predictions = [
21 {"pair_id": pair["pair_id"], "equivalent": False}
22 for pair in payload.get("pairs", [])
23 ]
24
25 output_path.parent.mkdir(parents=True, exist_ok=True)
26 output_path.write_text(
27 json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
28 encoding="utf-8",
29 )
30 return 0
31
32
33 if __name__ == "__main__":
34 raise SystemExit(main())
35
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
For each pair of compact CAD command programs, decide whether both programs
describe the same canonical solid. We build a canonical signature for each
program by:
* evaluating parameter expressions to numeric values (tolerance 1e-6),
* resolving profiles (rect/circle/slot) and their sketch planes,
* canonicalizing extrude features (operation/extent/direction/depth/profiles),
* canonicalizing the body/channel partition topology (set partition of
features, labels are renamable and body/channel are not namespaces),
* comparing opaque solid-affecting commands as an unordered literal multiset,
* ignoring construction and metadata/annotation commands.
Two programs are equivalent iff their signatures match.
"""
from __future__ import annotations
import ast
import json
import sys
from pathlib import Path
# ---------------------------------------------------------------------------
# Command classification
# ---------------------------------------------------------------------------
# Kinds that define supported non-opaque geometry / values.
DEFINITIONAL_KINDS = {"param", "sketch", "rect", "circle", "slot"}
PROFILE_KINDS = {"rect", "circle", "slot"}
# Metadata / annotation kinds that never affect the solid.
METADATA_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
TOL_DECIMALS = 6 # absolute tolerance 1e-6
# ---------------------------------------------------------------------------
# Expression evaluation
# ---------------------------------------------------------------------------
class ExprError(Exception):
pass
def evaluate(value, params, stack=None):
"""Evaluate a scalar value that may be a number or an arithmetic expr."""
if isinstance(value, bool):
raise ExprError("bool is not a numeric scalar")
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, str):
return _eval_expr(value, params, stack if stack is not None else frozenset())
raise ExprError(f"unsupported value: {value!r}")
def _eval_expr(expr, params, stack):
try:
tree = ast.parse(expr, mode="eval")
except SyntaxError as exc: # not a parseable arithmetic expression
raise ExprError(str(exc))
return _eval_node(tree.body, params, stack)
def _eval_node(node, params, stack):
if isinstance(node, ast.Expression):
return _eval_node(node.body, params, stack)
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ExprError("non-numeric constant")
return float(node.value)
# Python <3.8 compatibility.
if isinstance(node, getattr(ast, "Num", ())): # pragma: no cover
return float(node.n)
if isinstance(node, ast.BinOp):
left = _eval_node(node.left, params, stack)
right = _eval_node(node.right, params, stack)
op = node.op
if isinstance(op, ast.Add):
return left + right
if isinstance(op, ast.Sub):
return left - right
if isinstance(op, ast.Mult):
return left * right
if isinstance(op, ast.Div):
return left / right
raise ExprError("unsupported binary operator")
if isinstance(node, ast.UnaryOp):
operand = _eval_node(node.operand, params, stack)
if isinstance(node.op, ast.UAdd):
return +operand
if isinstance(node.op, ast.USub):
return -operand
raise ExprError("unsupported unary operator")
if isinstance(node, ast.Name):
name = node.id
if name not in params:
raise ExprError(f"undefined parameter: {name}")
if name in stack:
raise ExprError(f"cyclic parameter: {name}")
return evaluate(params[name], params, stack | {name})
raise ExprError(f"unsupported expression node: {type(node).__name__}")
# ---------------------------------------------------------------------------
# Numeric normalization
# ---------------------------------------------------------------------------
def r(x):
"""Round to the comparison resolution and normalize negative zero."""
y = round(float(x), TOL_DECIMALS)
if y == 0.0:
y = 0.0
return y
# ---------------------------------------------------------------------------
# Profile canonicalization
# ---------------------------------------------------------------------------
def canon_scalar(key, value, params):
try:
x = evaluate(value, params)
except Exception:
# Non-numeric literal payload; keep it literally.
return ["LIT", value]
if key == "angle":
x = x % 180.0
return r(x)
def canon_value(key, value, params):
if isinstance(value, list):
return [canon_scalar(key, elem, params) for elem in value]
return canon_scalar(key, value, params)
def profile_signature(pid, params, sketches, profiles):
prof = profiles.get(pid)
if prof is None:
return json.dumps({"missing_profile": pid}, sort_keys=True)
kind = prof.get("cmd")
plane = sketches.get(prof.get("sketch"))
if isinstance(plane, str):
plane = plane.upper()
fields = {}
for k, v in prof.items():
if k in ("cmd", "id", "sketch", "construction"):
continue
fields[k] = canon_value(k, v, params)
struct = {"kind": kind, "plane": plane, "fields": fields}
return json.dumps(struct, sort_keys=True)
# ---------------------------------------------------------------------------
# Extrude canonicalization
# ---------------------------------------------------------------------------
def extrude_signature(cmd, params, sketches, profiles):
if "profiles" in cmd and isinstance(cmd["profiles"], list):
prof_ids = list(cmd["profiles"])
elif "profile" in cmd:
prof_ids = [cmd["profile"]]
else:
prof_ids = []
prof_sigs = sorted(
profile_signature(pid, params, sketches, profiles) for pid in prof_ids
)
op = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = str(cmd.get("direction", "normal")).lower()
depth = cmd.get("depth", None)
if depth is None:
depth_v = None
else:
try:
depth_v = r(evaluate(depth, params))
except Exception:
depth_v = ["LIT", depth]
feature = {
"operation": op,
"extent": extent,
"direction": direction,
"depth": depth_v,
"profiles": prof_sigs,
}
return json.dumps(feature, sort_keys=True)
# ---------------------------------------------------------------------------
# Program signature
# ---------------------------------------------------------------------------
def program_signature(program):
# Definitional maps (looked up by id regardless of construction flag).
params = {}
sketches = {}
profiles = {}
for cmd in program:
if not isinstance(cmd, dict):
continue
kind = cmd.get("cmd")
if kind == "param":
name = cmd.get("name")
if name is not None:
params[name] = cmd.get("expr")
elif kind == "sketch":
sid = cmd.get("id")
if sid is not None:
sketches[sid] = cmd.get("plane")
elif kind in PROFILE_KINDS:
pid = cmd.get("id")
if pid is not None:
profiles[pid] = cmd
# Collect extrude features (with partition labels) and opaque commands.
labeled_groups = {} # label value -> list of feature sigs
default_group = []
opaque = []
for cmd in program:
if not isinstance(cmd, dict):
continue
if cmd.get("construction"):
continue
kind = cmd.get("cmd")
if kind in DEFINITIONAL_KINDS:
continue
if kind in METADATA_KINDS:
continue
if kind == "extrude":
fsig = extrude_signature(cmd, params, sketches, profiles)
# body/channel are not namespaces: group by raw label value.
if "body" in cmd:
label = ("V", _hashable(cmd["body"]))
elif "channel" in cmd:
label = ("V", _hashable(cmd["channel"]))
else:
label = None
if label is None:
default_group.append(fsig)
else:
labeled_groups.setdefault(label, []).append(fsig)
else:
# Opaque solid-affecting payload: strip only top-level id.
payload = {k: v for k, v in cmd.items() if k != "id"}
opaque.append(json.dumps(payload, sort_keys=True))
# Partition topology: set partition of features. The unlabeled default
# partition is just another group (labels renamable, no special tag).
groups = []
for feats in labeled_groups.values():
groups.append(json.dumps(sorted(feats)))
if default_group:
groups.append(json.dumps(sorted(default_group)))
partition_sig = sorted(groups)
signature = {
"partition": partition_sig,
"opaque": sorted(opaque),
}
return json.dumps(signature, sort_keys=True)
def _hashable(value):
try:
hash(value)
return value
except TypeError:
return json.dumps(value, sort_keys=True)
# ---------------------------------------------------------------------------
# Driver
# ---------------------------------------------------------------------------
def solve_pair(pair):
sig_a = program_signature(pair.get("program_a", []))
sig_b = program_signature(pair.get("program_b", []))
return sig_a, sig_b
def main():
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = []
for pair in payload.get("pairs", []):
pid = pair.get("pair_id")
try:
sig_a, sig_b = solve_pair(pair)
equivalent = sig_a == sig_b
except Exception:
sig_a, sig_b = "", ""
equivalent = False
predictions.append(
{
"pair_id": pid,
"equivalent": bool(equivalent),
"signature_a": sig_a,
"signature_b": sig_b,
}
)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Run solver on public examples
python3 solve.py data/public_pairs.json predictions.json && cat predictions.json
{
"predictions": [
{
"equivalent": true,
"pair_id": "P001",
"signature_a": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 8.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"new\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [0.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [40.0, 20.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}",
"signature_b": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 8.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"new\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [0.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [40.0, 20.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}"
},
{
"equivalent": false,
"pair_id": "P002",
"signature_a": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 10.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"cut\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [8.0, 0.0], \\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\": 4.0}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}",
"signature_b": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 10.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"cut\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [8.0, 0.0], \\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\": 4.25}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}"
},
{
"equivalent": true,
"pair_id": "P003",
"signature_a": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 5.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"add\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [-6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\", \\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}",
"signature_b": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 5.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"add\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [-6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\", \\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}"
}
]
}
[stdout]
{
"predictions": [
{
"equivalent": true,
"pair_id": "P001",
"signature_a": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 8.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"new\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [0.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [40.0, 20.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}",
"signature_b": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 8.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"new\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [0.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [40.0, 20.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}"
},
{
"equivalent": false,
"pair_id": "P002",
"signature_a": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 10.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"cut\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [8.0, 0.0], \\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\": 4.0}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}",
"signature_b": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 10.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"cut\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [8.0, 0.0], \\\\\\\\\\\\\\\"radius\\\\\\\\\\\\\\\": 4.25}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"circle\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}"
},
{
"equivalent": true,
"pair_id": "P003",
"signature_a": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 5.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"add\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [-6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\", \\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}",
"signature_b": "{\"opaque\": [], \"partition\": [\"[\\\"{\\\\\\\"depth\\\\\\\": 5.0, \\\\\\\"direction\\\\\\\": \\\\\\\"normal\\\\\\\", \\\\\\\"extent\\\\\\\": \\\\\\\"one_side\\\\\\\", \\\\\\\"operation\\\\\\\": \\\\\\\"add\\\\\\\", \\\\\\\"profiles\\\\\\\": [\\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [-6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\", \\\\\\\"{\\\\\\\\\\\\\\\"fields\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"center\\\\\\\\\\\\\\\": [6.0, 0.0], \\\\\\\\\\\\\\\"size\\\\\\\\\\\\\\\": [2.0, 12.0]}, \\\\\\\\\\\\\\\"kind\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rect\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"plane\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"XY\\\\\\\\\\\\\\\"}\\\\\\\"]}\\\"]\"]}"
}
]
}No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_80cd6bf28a994b1e. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_80cd6bf28a994b1e · verifier authoritative; classifier explanatory.