tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.Read public pairs example
cat /workspace/data/public_pairs.json
{
"schema_version": "1.0",
"description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
"pairs": [
{
"pair_id": "P001",
"program_a": [
{"cmd": "param", "name": "w", "expr": "40"},
{"cmd": "param", "name": "h", "expr": "20"},
{"cmd": "param", "name": "d", "expr": "8"},
{"cmd": "sketch", "id": "s0", "plane": "XY"},
{"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
{"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
{"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "depth", "expr": "4 + 4"},
{"cmd": "param", "name": "height", "expr": "5 * 4"},
{"cmd": "param", "name": "width", "expr": "20 * 2"},
{"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
{"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
{"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
]
},
{
"pair_id": "P002",
"program_a": [
{"cmd": "param", "name": "r", "expr": "4"},
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
{"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "r2", "expr": "4.25"},
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
{"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
]
},
{
"pair_id": "P003",
"program_a": [
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
],
"program_b": [
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
]
}
]
}
[stdout]
{
"schema_version": "1.0",
"description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
"pairs": [
{
"pair_id": "P001",
"program_a": [
{"cmd": "param", "name": "w", "expr": "40"},
{"cmd": "param", "name": "h", "expr": "20"},
{"cmd": "param", "name": "d", "expr": "8"},
{"cmd": "sketch", "id": "s0", "plane": "XY"},
{"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
{"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
{"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "depth", "expr": "4 + 4"},
{"cmd": "param", "name": "height", "expr": "5 * 4"},
{"cmd": "param", "name": "width", "expr": "20 * 2"},
{"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
{"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
{"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
]
},
{
"pair_id": "P002",
"program_a": [
{"cmd": "param", "name": "r", "expr": "4"},
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
{"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
],
"program_b": [
{"cmd": "param", "name": "r2", "expr": "4.25"},
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
{"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
]
},
{
"pair_id": "P003",
"program_a": [
{"cmd": "sketch", "id": "s", "plane": "XY"},
{"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
],
"program_b": [
{"cmd": "sketch", "id": "renamed", "plane": "XY"},
{"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
{"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
{"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
]
}
]
}List workspace and check python
ls -la /workspace/ /workspace/data/ 2>/dev/null; echo "---python---"; python3 --version
/workspace/: total 24 drwxr-xr-x 1 root root 4096 Jul 6 02:01 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 299 Jul 4 05:02 README.md drwxr-xr-x 2 root root 4096 Jul 4 05:02 data drwxr-xr-x 2 root root 4096 Jul 6 02:01 output -rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py /workspace/data/: total 12 drwxr-xr-x 2 root root 4096 Jul 4 05:02 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json ---python--- Python 3.11.15 [stdout] /workspace/: total 24 drwxr-xr-x 1 root root 4096 Jul 6 02:01 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 299 Jul 4 05:02 README.md drwxr-xr-x 2 root root 4096 Jul 4 05:02 data drwxr-xr-x 2 root root 4096 Jul 6 02:01 output -rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py /workspace/data/: total 12 drwxr-xr-x 2 root root 4096 Jul 4 05:02 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json ---python--- Python 3.11.15
Read existing solve.py and README
cat /workspace/solve.py; echo "---README---"; cat /workspace/README.md
#!/usr/bin/env python3
"""Weak starter baseline for DeepCAD canonical-equivalence.
This intentionally predicts every pair as non-equivalent. It is useful only as
a schema example and should score reward 0.0 on the hidden verifier.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = [
{"pair_id": pair["pair_id"], "equivalent": False}
for pair in payload.get("pairs", [])
]
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
---README---
# DeepCAD Canonical Equivalence Starter
`solve.py` is an intentionally weak starter baseline. It reads an input JSON with program pairs and writes a prediction for each pair.
The public file `data/public_pairs.json` is unlabeled and only shows the command schema. Hidden labels are verifier-only.
[stdout]
#!/usr/bin/env python3
"""Weak starter baseline for DeepCAD canonical-equivalence.
This intentionally predicts every pair as non-equivalent. It is useful only as
a schema example and should score reward 0.0 on the hidden verifier.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = [
{"pair_id": pair["pair_id"], "equivalent": False}
for pair in payload.get("pairs", [])
]
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
---README---
# DeepCAD Canonical Equivalence Starter
`solve.py` is an intentionally weak starter baseline. It reads an input JSON with program pairs and writes a prediction for each pair.
The public file `data/public_pairs.json` is unlabeled and only shows the command schema. Hidden labels are verifier-only.#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads an input JSON with program pairs and decides, for each pair, whether both
programs describe the same canonical solid. See task spec for semantics.
Usage:
python3 solve.py <input_json> <output_json>
"""
from __future__ import annotations
import ast
import json
import sys
from pathlib import Path
# ---------------------------------------------------------------------------
# Command classification
# ---------------------------------------------------------------------------
SUPPORTED_KINDS = {"param", "sketch", "rect", "circle", "slot", "extrude"}
PROFILE_KINDS = {"rect", "circle", "slot"}
METADATA_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
TOL_DECIMALS = 6
def R(v):
"""Round a numeric value to the comparison tolerance, normalising -0.0."""
r = round(float(v) + 0.0, TOL_DECIMALS)
if r == 0:
return 0.0
return r
# ---------------------------------------------------------------------------
# Expression evaluation
# ---------------------------------------------------------------------------
class ExprError(Exception):
pass
def _eval_ast(node, env, cache, resolving):
if isinstance(node, ast.Expression):
return _eval_ast(node.body, env, cache, resolving)
if isinstance(node, ast.Constant):
if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
raise ExprError(f"non-numeric constant {node.value!r}")
return float(node.value)
# Python <3.8 compatibility not needed, but keep Num just in case.
if isinstance(node, ast.Num): # pragma: no cover
return float(node.n)
if isinstance(node, ast.Name):
return _resolve_param(node.id, env, cache, resolving)
if isinstance(node, ast.BinOp):
left = _eval_ast(node.left, env, cache, resolving)
right = _eval_ast(node.right, env, cache, resolving)
if isinstance(node.op, ast.Add):
return left + right
if isinstance(node.op, ast.Sub):
return left - right
if isinstance(node.op, ast.Mult):
return left * right
if isinstance(node.op, ast.Div):
return left / right
raise ExprError(f"unsupported operator {node.op!r}")
if isinstance(node, ast.UnaryOp):
operand = _eval_ast(node.operand, env, cache, resolving)
if isinstance(node.op, ast.UAdd):
return +operand
if isinstance(node.op, ast.USub):
return -operand
raise ExprError(f"unsupported unary op {node.op!r}")
if isinstance(node, ast.Paren): # pragma: no cover
return _eval_ast(node.value, env, cache, resolving)
raise ExprError(f"unsupported expression node {type(node).__name__}")
def _resolve_param(name, env, cache, resolving):
if name in cache:
return cache[name]
if name not in env:
raise ExprError(f"unknown parameter {name!r}")
if name in resolving:
raise ExprError(f"cyclic parameter {name!r}")
resolving.add(name)
try:
value = _eval_scalar(env[name], env, cache, resolving)
finally:
resolving.discard(name)
cache[name] = value
return value
def _eval_scalar(raw, env, cache, resolving):
if isinstance(raw, bool):
raise ExprError("boolean is not a scalar")
if isinstance(raw, (int, float)):
return float(raw)
if isinstance(raw, str):
tree = ast.parse(raw, mode="eval")
return _eval_ast(tree, env, cache, resolving)
raise ExprError(f"cannot evaluate scalar {raw!r}")
def eval_value(val, env, cache):
"""Evaluate a value which may be a scalar expression or a list of them."""
if isinstance(val, list):
return [eval_value(v, env, cache) for v in val]
try:
return R(_eval_scalar(val, env, cache, set()))
except ExprError:
# Not evaluable as a number -> keep as-is (literal token).
return val
# ---------------------------------------------------------------------------
# Program -> canonical signature
# ---------------------------------------------------------------------------
def canon_str(obj):
return json.dumps(obj, sort_keys=True, separators=(",", ":"))
def build_signature(program):
env = {}
for cmd in program:
if cmd.get("cmd") == "param":
name = cmd.get("name")
if name is not None:
env[name] = cmd.get("expr")
cache = {}
# Sketch plane table (case-insensitive token, axis order preserved).
planes = {}
for cmd in program:
if cmd.get("cmd") == "sketch":
sid = cmd.get("id")
plane = cmd.get("plane")
if isinstance(plane, str):
plane = plane.upper()
planes[sid] = plane
# Profile table (skip construction geometry).
profiles = {}
for cmd in program:
kind = cmd.get("cmd")
if kind in PROFILE_KINDS and not cmd.get("construction", False):
profiles[cmd.get("id")] = _canon_profile(cmd, planes, env, cache)
# Extrude features and opaque payload commands.
features = [] # list of (feature_struct, partition_label)
opaque = []
for cmd in program:
if cmd.get("construction", False):
continue
kind = cmd.get("cmd")
if kind == "extrude":
features.append(_canon_extrude(cmd, profiles, env, cache))
elif kind in SUPPORTED_KINDS or kind in METADATA_KINDS:
continue
else:
payload = {k: v for k, v in cmd.items() if k != "id"}
opaque.append(canon_str(payload))
# Group features into partitions (label-independent structure).
groups = {}
for feat_struct, label in features:
groups.setdefault(label, []).append(feat_struct)
group_structs = []
for label, feats in groups.items():
feats_sorted = sorted(feats, key=canon_str)
group_structs.append(feats_sorted)
group_structs.sort(key=canon_str)
opaque.sort()
signature = {"features": group_structs, "opaque": opaque}
return canon_str(signature)
def _canon_profile(cmd, planes, env, cache):
kind = str(cmd.get("cmd")).lower()
plane = planes.get(cmd.get("sketch"))
geom = {"_kind": kind, "_plane": plane}
for key, val in cmd.items():
if key in ("cmd", "id", "sketch", "construction"):
continue
ev = eval_value(val, env, cache)
if key.lower() == "angle" and isinstance(ev, (int, float)):
ev = R(float(ev) % 180.0)
geom[key] = ev
return geom
def _canon_direction(val, env, cache):
if isinstance(val, str):
return val.lower()
if isinstance(val, list):
return eval_value(val, env, cache)
return val
def _canon_extrude(cmd, profiles, env, cache):
operation = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = _canon_direction(cmd.get("direction", "normal"), env, cache)
depth = eval_value(cmd.get("depth", 0), env, cache)
# Resolve profile references (order not semantic).
ref = []
if "profiles" in cmd and isinstance(cmd["profiles"], list):
ref.extend(cmd["profiles"])
if "profile" in cmd:
p = cmd["profile"]
if isinstance(p, list):
ref.extend(p)
else:
ref.append(p)
geoms = []
for pid in ref:
if pid in profiles:
geoms.append(profiles[pid])
else:
geoms.append({"_kind": "unknown_profile", "_ref": pid})
geoms_sorted = sorted(geoms, key=canon_str)
feat = {
"operation": operation,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": geoms_sorted,
}
# Partition label: body or channel value; None => unlabeled default group.
if "body" in cmd:
label = ("L", cmd["body"])
elif "channel" in cmd:
label = ("L", cmd["channel"])
else:
label = ("DEFAULT",)
return feat, label
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main() -> int:
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = []
for pair in payload.get("pairs", []):
pid = pair.get("pair_id")
try:
sig_a = build_signature(pair.get("program_a", []))
sig_b = build_signature(pair.get("program_b", []))
equivalent = sig_a == sig_b
except Exception:
sig_a = sig_b = None
equivalent = False
predictions.append(
{
"pair_id": pid,
"equivalent": bool(equivalent),
"signature_a": sig_a,
"signature_b": sig_b,
}
)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Run solver on public pairs
python3 solve.py data/public_pairs.json /workspace/predictions.json && python3 -c "
import json
d=json.load(open('predictions.json'))
for p in d['predictions']:
print(p['pair_id'], p['equivalent'])
"P001 True P002 False P003 True [stdout] P001 True P002 False P003 True
Run edge case tests
cat > /tmp/edge.json <<'EOF'
{"pairs":[
{"pair_id":"E1_plane_order","program_a":[
{"cmd":"sketch","id":"s","plane":"XZ"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":2}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"ZX"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":2}]},
{"pair_id":"E2_default_op","program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":2}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":2,"operation":"new","extent":"one_side","direction":"normal"}]},
{"pair_id":"E3_neg_depth_vs_dir","program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":-2}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":2,"direction":"reverse"}]},
{"pair_id":"E4_slot_angle_mod180","program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":30},
{"cmd":"extrude","id":"e","profile":"sl","depth":2}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":210},
{"cmd":"extrude","id":"e","profile":"sl","depth":2}]},
{"pair_id":"E5_partition_diff","program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"rect","id":"r2","sketch":"s","center":[9,0],"size":[4,4]},
{"cmd":"extrude","id":"e1","profile":"r","depth":2,"body":"g"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":2,"body":"g"}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"rect","id":"r2","sketch":"s","center":[9,0],"size":[4,4]},
{"cmd":"extrude","id":"e1","profile":"r","depth":2,"body":"a"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":2,"body":"b"}]},
{"pair_id":"E6_body_channel_same","program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"rect","id":"r2","sketch":"s","center":[9,0],"size":[4,4]},
{"cmd":"extrude","id":"e1","profile":"r","depth":2,"body":"g"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":2,"body":"g"}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"rect","id":"r2","sketch":"s","center":[9,0],"size":[4,4]},
{"cmd":"extrude","id":"e1","profile":"r","depth":2,"channel":"z"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":2,"channel":"z"}]},
{"pair_id":"E7_opaque_literal","program_a":[
{"cmd":"fillet","id":"f1","edges":["e1","e2"],"radius":"2"}],
"program_b":[
{"cmd":"fillet","id":"fX","edges":["e1","e2"],"radius":"2"}]},
{"pair_id":"E8_opaque_diff","program_a":[
{"cmd":"fillet","id":"f1","edges":["e1","e2"],"radius":"2"}],
"program_b":[
{"cmd":"fillet","id":"f1","edges":["e1","e2"],"radius":"3"}]},
{"pair_id":"E9_add_vs_join","program_a":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":2,"operation":"add"}],
"program_b":[
{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[4,4]},
{"cmd":"extrude","id":"e","profile":"r","depth":2,"operation":"join"}]}
]}
EOF
python3 solve.py /tmp/edge.json /tmp/edge_out.json
python3 -c "
import json
for p in json.load(open('/tmp/edge_out.json'))['predictions']:
print(p['pair_id'], p['equivalent'])
"E1_plane_order False E2_default_op True E3_neg_depth_vs_dir False E4_slot_angle_mod180 True E5_partition_diff False E6_body_channel_same True E7_opaque_literal True E8_opaque_diff False E9_add_vs_join False [stdout] E1_plane_order False E2_default_op True E3_neg_depth_vs_dir False E4_slot_angle_mod180 True E5_partition_diff False E6_body_channel_same True E7_opaque_literal True E8_opaque_diff False E9_add_vs_join False
No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_5c11f2237606498a. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_5c11f2237606498a · verifier authoritative; classifier explanatory.