tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it./workspace/data/public_pairs.json
1 {
2 "schema_version": "1.0",
3 "description": "Unlabeled public format examples for the DeepCAD canonical-equivalence task.",
4 "pairs": [
5 {
6 "pair_id": "P001",
7 "program_a": [
8 {"cmd": "param", "name": "w", "expr": "40"},
9 {"cmd": "param", "name": "h", "expr": "20"},
10 {"cmd": "param", "name": "d", "expr": "8"},
11 {"cmd": "sketch", "id": "s0", "plane": "XY"},
12 {"cmd": "rect", "id": "base", "sketch": "s0", "center": [0, 0], "size": ["w", "h"]},
13 {"cmd": "line", "id": "guide", "sketch": "s0", "p1": [0, -10], "p2": [0, 10], "construction": true},
14 {"cmd": "extrude", "id": "pad", "profile": "base", "operation": "new", "depth": "d", "extent": "one_side"}
15 ],
16 "program_b": [
17 {"cmd": "param", "name": "depth", "expr": "4 + 4"},
18 {"cmd": "param", "name": "height", "expr": "5 * 4"},
19 {"cmd": "param", "name": "width", "expr": "20 * 2"},
20 {"cmd": "sketch", "id": "profile_sketch", "plane": "XY"},
21 {"cmd": "rect", "id": "outer", "sketch": "profile_sketch", "center": [0, 0], "size": ["width", "height"]},
22 {"cmd": "extrude", "id": "solid", "profile": "outer", "operation": "new", "depth": "depth", "extent": "one_side"}
23 ]
24 },
25 {
26 "pair_id": "P002",
27 "program_a": [
28 {"cmd": "param", "name": "r", "expr": "4"},
29 {"cmd": "sketch", "id": "s", "plane": "XY"},
30 {"cmd": "circle", "id": "hole", "sketch": "s", "center": [8, 0], "radius": "r"},
31 {"cmd": "extrude", "id": "cut", "profile": "hole", "operation": "cut", "depth": 10, "extent": "one_side"}
32 ],
33 "program_b": [
34 {"cmd": "param", "name": "r2", "expr": "4.25"},
35 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
36 {"cmd": "circle", "id": "hole2", "sketch": "renamed", "center": [8, 0], "radius": "r2"},
37 {"cmd": "extrude", "id": "cut2", "profile": "hole2", "operation": "cut", "depth": 10, "extent": "one_side"}
38 ]
39 },
40 {
41 "pair_id": "P003",
42 "program_a": [
43 {"cmd": "sketch", "id": "s", "plane": "XY"},
44 {"cmd": "rect", "id": "rib_a", "sketch": "s", "center": [-6, 0], "size": [2, 12]},
45 {"cmd": "rect", "id": "rib_b", "sketch": "s", "center": [6, 0], "size": [2, 12]},
46 {"cmd": "extrude", "id": "ribs", "profiles": ["rib_a", "rib_b"], "operation": "add", "depth": 5, "extent": "one_side"}
47 ],
48 "program_b": [
49 {"cmd": "sketch", "id": "renamed", "plane": "XY"},
50 {"cmd": "rect", "id": "right", "sketch": "renamed", "center": [6, 0], "size": [2, 12]},
51 {"cmd": "rect", "id": "left", "sketch": "renamed", "center": [-6, 0], "size": [2, 12]},
52 {"cmd": "extrude", "id": "ribs2", "profiles": ["right", "left"], "operation": "add", "depth": 5, "extent": "one_side"}
53 ]
54 }
55 ]
56 }
57
ls -la /workspace /workspace/data 2>/dev/null
/workspace: total 24 drwxr-xr-x 1 root root 4096 Jul 6 02:01 . drwxr-xr-x 1 root root 4096 Jul 6 02:33 .. -rw-r--r-- 1 root root 299 Jul 4 05:02 README.md drwxr-xr-x 2 root root 4096 Jul 4 05:02 data drwxr-xr-x 2 root root 4096 Jul 6 02:01 output -rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py /workspace/data: total 12 drwxr-xr-x 2 root root 4096 Jul 4 05:02 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json [stdout] /workspace: total 24 drwxr-xr-x 1 root root 4096 Jul 6 02:01 . drwxr-xr-x 1 root root 4096 Jul 6 02:33 .. -rw-r--r-- 1 root root 299 Jul 4 05:02 README.md drwxr-xr-x 2 root root 4096 Jul 4 05:02 data drwxr-xr-x 2 root root 4096 Jul 6 02:01 output -rwxr-xr-x 1 root root 1011 Jul 4 05:02 solve.py /workspace/data: total 12 drwxr-xr-x 2 root root 4096 Jul 4 05:02 . drwxr-xr-x 1 root root 4096 Jul 6 02:01 .. -rw-r--r-- 1 root root 2824 Jul 4 05:02 public_pairs.json
/workspace/solve.py
1 #!/usr/bin/env python3
2 """Weak starter baseline for DeepCAD canonical-equivalence.
3
4 This intentionally predicts every pair as non-equivalent. It is useful only as
5 a schema example and should score reward 0.0 on the hidden verifier.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import sys
12 from pathlib import Path
13
14
15 def main() -> int:
16 input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
17 output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
18
19 payload = json.loads(input_path.read_text(encoding="utf-8"))
20 predictions = [
21 {"pair_id": pair["pair_id"], "equivalent": False}
22 for pair in payload.get("pairs", [])
23 ]
24
25 output_path.parent.mkdir(parents=True, exist_ok=True)
26 output_path.write_text(
27 json.dumps({"predictions": predictions}, indent=2, sort_keys=True) + "\n",
28 encoding="utf-8",
29 )
30 return 0
31
32
33 if __name__ == "__main__":
34 raise SystemExit(main())
35
#!/usr/bin/env python3
"""DeepCAD canonical-equivalence solver.
Reads a JSON file of program pairs and decides, for each pair, whether the two
programs describe the same canonical solid, following the benchmark semantics.
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
# ---------------------------------------------------------------------------
# Expression evaluation (params + numeric fields)
# ---------------------------------------------------------------------------
_TOKEN_RE = re.compile(
r"\s*(?:(?P<num>\d+\.\d+|\.\d+|\d+\.?|\d+)|(?P<ident>[A-Za-z_][A-Za-z0-9_]*)|(?P<op>[()+\-*/]))"
)
def _tokenize(expr):
tokens = []
pos = 0
n = len(expr)
while pos < n:
if expr[pos].isspace():
pos += 1
continue
m = _TOKEN_RE.match(expr, pos)
if not m or m.start() != pos and m.start() > pos:
# fall back: try match at pos
m = _TOKEN_RE.match(expr, pos)
if not m:
raise ValueError("cannot tokenize expression: %r at %d" % (expr, pos))
if m.group("num") is not None:
tokens.append(("num", float(m.group("num"))))
elif m.group("ident") is not None:
tokens.append(("ident", m.group("ident")))
else:
tokens.append(("op", m.group("op")))
pos = m.end()
return tokens
class _Parser:
def __init__(self, tokens, resolve):
self.tokens = tokens
self.i = 0
self.resolve = resolve
def peek(self):
return self.tokens[self.i] if self.i < len(self.tokens) else (None, None)
def advance(self):
tok = self.tokens[self.i]
self.i += 1
return tok
def parse(self):
val = self.expr()
if self.i != len(self.tokens):
raise ValueError("trailing tokens in expression")
return val
def expr(self):
val = self.term()
while True:
typ, t = self.peek()
if typ == "op" and t in ("+", "-"):
self.advance()
rhs = self.term()
val = val + rhs if t == "+" else val - rhs
else:
break
return val
def term(self):
val = self.factor()
while True:
typ, t = self.peek()
if typ == "op" and t in ("*", "/"):
self.advance()
rhs = self.factor()
val = val * rhs if t == "*" else val / rhs
else:
break
return val
def factor(self):
typ, t = self.peek()
if typ == "op" and t in ("+", "-"):
self.advance()
v = self.factor()
return v if t == "+" else -v
if typ == "op" and t == "(":
self.advance()
v = self.expr()
typ2, t2 = self.peek()
if not (typ2 == "op" and t2 == ")"):
raise ValueError("expected )")
self.advance()
return v
if typ == "num":
self.advance()
return t
if typ == "ident":
self.advance()
return self.resolve(t)
raise ValueError("unexpected token in expression: %r" % (t,))
def make_evaluator(params):
"""params: dict name -> raw expression (str/number). Returns eval_expr(str)->float."""
cache = {}
in_progress = set()
def resolve(name):
if name in cache:
return cache[name]
if name in in_progress:
raise ValueError("cyclic param reference: %s" % name)
if name not in params:
raise ValueError("unknown param: %s" % name)
in_progress.add(name)
val = eval_expr(params[name])
in_progress.discard(name)
cache[name] = val
return val
def eval_expr(raw):
if isinstance(raw, bool):
return float(raw)
if isinstance(raw, (int, float)):
return float(raw)
tokens = _tokenize(str(raw))
return _Parser(tokens, resolve).parse()
return eval_expr
def eval_field(val, eval_expr):
"""Evaluate a numeric field: number, expression string, or (nested) list."""
if isinstance(val, bool):
return val
if isinstance(val, (int, float)):
return float(val)
if isinstance(val, str):
return eval_expr(val)
if isinstance(val, list):
return [eval_field(x, eval_expr) for x in val]
return val
def round_val(v, ndigits=6):
if isinstance(v, bool):
return v
if isinstance(v, float):
r = round(v, ndigits)
if r == 0.0:
r = 0.0 # normalise -0.0
return r
if isinstance(v, list):
return [round_val(x, ndigits) for x in v]
return v
# ---------------------------------------------------------------------------
# Program canonicalisation
# ---------------------------------------------------------------------------
METADATA_KINDS = {"constraint", "dimension", "note", "metadata", "view", "comment"}
SUPPORTED_KINDS = {"param", "sketch", "rect", "circle", "slot", "extrude"}
PROFILE_KINDS = {"rect", "circle", "slot"}
def is_construction(cmd):
return bool(cmd.get("construction"))
def profile_signature(cmd, plane, eval_expr):
kind = cmd.get("cmd")
fields = {}
for k, v in cmd.items():
if k in ("cmd", "id", "sketch", "construction"):
continue
fields[k] = eval_field(v, eval_expr)
if kind == "slot" and "angle" in fields:
try:
fields["angle"] = float(fields["angle"]) % 180.0
except (TypeError, ValueError):
pass
fields = {k: round_val(v) for k, v in fields.items()}
sig = {"kind": kind, "plane": plane, "fields": fields}
return json.dumps(sig, sort_keys=True)
def canon_direction(val, eval_expr):
if isinstance(val, str):
return val.lower()
return round_val(eval_field(val, eval_expr))
def canon_program(prog):
# Pass 1: params
params = {}
for c in prog:
if not isinstance(c, dict):
continue
if c.get("cmd") == "param" and "name" in c:
params[c["name"]] = c.get("expr", 0)
eval_expr = make_evaluator(params)
# Pass 2: sketches (id -> plane, uppercased literal token)
sketches = {}
for c in prog:
if isinstance(c, dict) and c.get("cmd") == "sketch":
plane = c.get("plane")
if isinstance(plane, str):
plane = plane.upper()
sketches[c.get("id")] = plane
# Pass 3: profiles (id -> canonical signature)
profiles = {}
for c in prog:
if not isinstance(c, dict):
continue
if c.get("cmd") in PROFILE_KINDS:
plane = sketches.get(c.get("sketch"))
try:
profiles[c.get("id")] = profile_signature(c, plane, eval_expr)
except Exception:
profiles[c.get("id")] = json.dumps({"kind": c.get("cmd"), "raw": True})
# Pass 4: extrude features + Pass 5: opaque commands
features = [] # list of (content_sig_str, partition_key)
opaque = []
for c in prog:
if not isinstance(c, dict):
continue
if is_construction(c):
continue
kind = c.get("cmd")
if kind in METADATA_KINDS:
continue
if kind == "param" or kind == "sketch" or kind in PROFILE_KINDS:
continue
if kind == "extrude":
features.append(feature_signature(c, profiles, eval_expr))
continue
# opaque solid-affecting payload: strip only top-level id
payload = {k: v for k, v in c.items() if k != "id"}
opaque.append(json.dumps(payload, sort_keys=True))
# Partition topology: group features by label (unlabeled -> shared default)
groups = {}
for content, label in features:
key = label if label is not None else ("__DEFAULT__",)
groups.setdefault(key, []).append(content)
blocks = [json.dumps(sorted(v)) for v in groups.values()]
blocks.sort()
opaque.sort()
signature = json.dumps({"features": blocks, "opaque": opaque}, sort_keys=True)
return signature
def feature_signature(cmd, profiles, eval_expr):
op = str(cmd.get("operation", "new")).lower()
extent = str(cmd.get("extent", "one_side")).lower()
direction = canon_direction(cmd.get("direction", "normal"), eval_expr)
depth_raw = cmd.get("depth", 0)
try:
depth = round_val(eval_field(depth_raw, eval_expr))
except Exception:
depth = depth_raw
prof_ids = []
if "profiles" in cmd and isinstance(cmd["profiles"], list):
prof_ids.extend(cmd["profiles"])
if "profile" in cmd:
p = cmd["profile"]
if isinstance(p, list):
prof_ids.extend(p)
else:
prof_ids.append(p)
prof_sigs = []
for pid in prof_ids:
prof_sigs.append(profiles.get(pid, json.dumps({"missing": pid})))
prof_sigs.sort()
content = json.dumps(
{
"operation": op,
"extent": extent,
"direction": direction,
"depth": depth,
"profiles": prof_sigs,
},
sort_keys=True,
)
label = None
if "body" in cmd:
label = ("label", str(cmd["body"]))
elif "channel" in cmd:
label = ("label", str(cmd["channel"]))
return content, label
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
input_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/workspace/data/public_pairs.json")
output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/workspace/predictions.json")
payload = json.loads(input_path.read_text(encoding="utf-8"))
predictions = []
for pair in payload.get("pairs", []):
pid = pair.get("pair_id")
try:
sig_a = canon_program(pair.get("program_a", []))
sig_b = canon_program(pair.get("program_b", []))
equivalent = sig_a == sig_b
except Exception:
sig_a = sig_b = None
equivalent = False
predictions.append(
{
"pair_id": pid,
"equivalent": bool(equivalent),
"signature_a": sig_a,
"signature_b": sig_b,
}
)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps({"predictions": predictions}, indent=2) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
/workspace/solve.py
_TOKEN_RE = re.compile(
r"\s*(?:(?P<num>\d+\.\d+|\.\d+|\d+\.?|\d+)|(?P<ident>[A-Za-z_][A-Za-z0-9_]*)|(?P<op>[()+\-*/]))"
)
def _tokenize(expr):
tokens = []
pos = 0
n = len(expr)
while pos < n:
if expr[pos].isspace():
pos += 1
continue
m = _TOKEN_RE.match(expr, pos)
if not m or m.start() != pos and m.start() > pos:
# fall back: try match at pos
m = _TOKEN_RE.match(expr, pos)
if not m:
raise ValueError("cannot tokenize expression: %r at %d" % (expr, pos))
if m.group("num") is not None:
tokens.append(("num", float(m.group("num"))))
elif m.group("ident") is not None:
tokens.append(("ident", m.group("ident")))
else:
tokens.append(("op", m.group("op")))
pos = m.end()
return tokens_TOKEN_RE = re.compile(
r"(?P<num>\d+\.\d+|\.\d+|\d+)|(?P<ident>[A-Za-z_][A-Za-z0-9_]*)|(?P<op>[()+\-*/])"
)
def _tokenize(expr):
tokens = []
pos = 0
n = len(expr)
while pos < n:
if expr[pos].isspace():
pos += 1
continue
m = _TOKEN_RE.match(expr, pos)
if not m:
raise ValueError("cannot tokenize expression: %r at %d" % (expr, pos))
if m.group("num") is not None:
tokens.append(("num", float(m.group("num"))))
elif m.group("ident") is not None:
tokens.append(("ident", m.group("ident")))
else:
tokens.append(("op", m.group("op")))
pos = m.end()
return tokenspython3 solve.py data/public_pairs.json /workspace/predictions.json && python3 -c "
import json
d=json.load(open('/workspace/predictions.json'))
for p in d['predictions']:
print(p['pair_id'], p['equivalent'])
"P001 True P002 False P003 True [stdout] P001 True P002 False P003 True
cat > /tmp/test.json << 'EOF'
{"pairs":[
{"pair_id":"T1_partition_body_vs_channel",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r1","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"r2","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r1","depth":3,"body":"A"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"body":"A"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r1","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"r2","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r1","depth":3,"channel":"Z"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"channel":"Z"}]},
{"pair_id":"T2_partition_shared_vs_split",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r1","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"r2","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r1","depth":3,"body":"A"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"body":"A"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r1","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"r2","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r1","depth":3,"body":"A"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"body":"B"}]},
{"pair_id":"T3_depth_sign",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"circle","id":"c","sketch":"s","center":[0,0],"radius":1},
{"cmd":"extrude","id":"e","profile":"c","depth":5}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"circle","id":"c","sketch":"s","center":[0,0],"radius":1},
{"cmd":"extrude","id":"e","profile":"c","depth":-5}]},
{"pair_id":"T4_slot_angle_mod180",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":30},
{"cmd":"extrude","id":"e","profile":"sl","depth":4}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"slot","id":"sl","sketch":"s","center":[0,0],"length":10,"width":2,"angle":210},
{"cmd":"extrude","id":"e","profile":"sl","depth":4}]},
{"pair_id":"T5_opaque_multiset_order",
"program_a":[{"cmd":"fillet","id":"f1","edges":["a","b"],"radius":2},{"cmd":"chamfer","id":"c1","edges":["x"],"dist":1}],
"program_b":[{"cmd":"chamfer","id":"c9","edges":["x"],"dist":1},{"cmd":"fillet","id":"f9","edges":["a","b"],"radius":2}]},
{"pair_id":"T6_opaque_ref_not_resolved",
"program_a":[{"cmd":"fillet","id":"f1","target":"e1","radius":2}],
"program_b":[{"cmd":"fillet","id":"f1","target":"e2","radius":2}]},
{"pair_id":"T7_plane_XZ_vs_ZX",
"program_a":[{"cmd":"sketch","id":"s","plane":"XZ"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,3]},
{"cmd":"extrude","id":"e","profile":"r","depth":1}],
"program_b":[{"cmd":"sketch","id":"s","plane":"ZX"},{"cmd":"rect","id":"r","sketch":"s","center":[0,0],"size":[2,3]},
{"cmd":"extrude","id":"e","profile":"r","depth":1}]},
{"pair_id":"T8_unlabeled_vs_shared_label",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r1","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"r2","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r1","depth":3},
{"cmd":"extrude","id":"e2","profile":"r2","depth":3}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},
{"cmd":"rect","id":"r1","sketch":"s","center":[0,0],"size":[2,2]},
{"cmd":"rect","id":"r2","sketch":"s","center":[5,0],"size":[2,2]},
{"cmd":"extrude","id":"e1","profile":"r1","depth":3,"body":"G"},
{"cmd":"extrude","id":"e2","profile":"r2","depth":3,"body":"G"}]},
{"pair_id":"T9_add_vs_new_distinct",
"program_a":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"circle","id":"c","sketch":"s","center":[0,0],"radius":1},
{"cmd":"extrude","id":"e","profile":"c","depth":5,"operation":"add"}],
"program_b":[{"cmd":"sketch","id":"s","plane":"XY"},{"cmd":"circle","id":"c","sketch":"s","center":[0,0],"radius":1},
{"cmd":"extrude","id":"e","profile":"c","depth":5,"operation":"new"}]}
]}
EOF
python3 solve.py /tmp/test.json /tmp/out.json && python3 -c "
import json
for p in json.load(open('/tmp/out.json'))['predictions']:
print(p['pair_id'], p['equivalent'])
"T1_partition_body_vs_channel True T2_partition_shared_vs_split False T3_depth_sign False T4_slot_angle_mod180 True T5_opaque_multiset_order True T6_opaque_ref_not_resolved False T7_plane_XZ_vs_ZX False T8_unlabeled_vs_shared_label True T9_add_vs_new_distinct False [stdout] T1_partition_body_vs_channel True T2_partition_shared_vs_split False T3_depth_sign False T4_slot_angle_mod180 True T5_opaque_multiset_order True T6_opaque_ref_not_resolved False T7_plane_XZ_vs_ZX False T8_unlabeled_vs_shared_label True T9_add_vs_new_distinct False
No file changes, the agent left the workspace untouched.
{
"metrics": {
"canonical_regime_min_accuracy": 0.934426,
"diagnostic_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"enum_literal": 1.0,
"explicit_defaults": 1.0,
"feature_multiset_multiplicity": 1.0,
"hard_decoy": 0.934426,
"numeric_tolerance": 1.0,
"ood": 0.952941,
"opaque_id_alias": 1.0,
"opaque_list_literal": 1.0,
"opaque_nested_literal": 1.0,
"opaque_numeric_literal": 1.0,
"opaque_reference_literal": 1.0,
"ordered_rect_dimensions": 1.0,
"partition_alias_true": 1.0,
"partition_default": 1.0,
"partition_topology": 0.0,
"plane_literal": 1.0,
"profile_order": 1.0,
"redundant": 1.0,
"signed_depth_direction": 1.0,
"slot_angle_modulo": 1.0,
"unconsumed_profile": 1.0
},
"diagnostic_macro_accuracy": 0.951625,
"equivalent_recall": 1.0,
"overall_accuracy": 0.958763,
"regime_accuracy": {
"alias": 1.0,
"commutation": 1.0,
"hard_decoy": 0.934426,
"ood": 0.952941,
"redundant": 1.0
}
},
"passed": false,
"reward": 0.441163,
"reward_policy": {
"anchors": {
"diagnostic_macro_accuracy": {
"baseline": 0.45,
"strong_target": 1.0,
"target": 0.985,
"threshold": 0.8
},
"equivalent_recall": {
"baseline": 0.3,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.8
},
"overall_accuracy": {
"baseline": 0.65,
"strong_target": 1.0,
"target": 0.99,
"threshold": 0.88
}
},
"meets_targets": false,
"metric_scores": {
"diagnostic_macro_accuracy": 0.474231,
"equivalent_recall": 1.0,
"overall_accuracy": 0.441163
},
"required_metrics": [
"overall_accuracy",
"equivalent_recall",
"diagnostic_macro_accuracy"
],
"target_reward": 0.5,
"threshold_reward": 0.2,
"type": "v3_50pass"
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_ece42ca05b7f44a6. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_ece42ca05b7f44a6 · verifier authoritative; classifier explanatory.