{ "meta": { "snapshotHash": "29b46d5ea53031efee082a7b9ff6f1c4535a37e21efce8924f9b05e6cdf2948f", "generatedAt": "2026-10-23T17:39:59+00:00", "sources": [ { "name": "1. Delivery cohort export", "asOf": "2026-09-11" }, { "name": "2. Website evaluation export", "asOf": "2026-08-23 (site build) / scraped 2026-09-11" }, { "name": "3. Independent review inventory", "asOf": "2026-09-11" }, { "name": "4. Independent review inventory", "asOf": "2026-09-12" }, { "name": "5. Repository change inventory", "asOf": "2026-09-11" }, { "name": "6. Delivery cohort export", "asOf": "2026-09-11" }, { "name": "7. Repository evaluation export", "asOf": "2026-09-11" }, { "name": "8. Repository evaluation export", "asOf": "2026-09-11" }, { "name": "9. Repository evaluation export", "asOf": "2026-10-16" }, { "name": "10. Console evaluation snapshot", "asOf": "2026-10-22" }, { "name": "11. Independent review inventory", "asOf": "2026-10-22" }, { "name": "12. Independent review inventory", "asOf": "2026-10-22" }, { "name": "13. Independent review inventory", "asOf": "2026-10-22" }, { "name": "14. Independent review inventory", "asOf": "2026-10-22" }, { "name": "15. Repository evaluation export", "asOf": "2026-10-21" }, { "name": "16. Independent review inventory", "asOf": "2026-10-21" }, { "name": "17. Independent review inventory", "asOf": "2026-10-21" }, { "name": "18. Delivery cohort export", "asOf": "2026-10-16" }, { "name": "19. Independent review inventory", "asOf": "2026-10-14" }, { "name": "20. Domain taxonomy inventory", "asOf": "2026-10-14" }, { "name": "21. Independent review inventory", "asOf": "2026-10-20" }, { "name": "22. Repository change inventory", "asOf": "2026-10-23" }, { "name": "23. Delivery cohort export", "asOf": "2026-10-22" } ], "methodsVersion": "Syncvals analysis v1.2 ยท 2026-10-24", "bootstrap": { "replicates": 2000, "seed": 20261023, "method": "Paired task resampling, then binomial run resampling; percentile 95% interval" }, "limitations": [ "This is an observational catalog of recorded agent systems, not a balanced experiment or a representative sample of coding work.", "Current and legacy status comes from the frozen snapshot. No external release date or model identity was inferred.", "Aggregate rates use one latest nondefective version per task ID and at least three valid runs per task. Rows with fewer than three eligible tasks show counts only.", "Task names and display keys do not establish identity. Distinct task IDs remain distinct even if their names match.", "Confidence intervals describe sampling uncertainty under the stated assumptions. They do not correct selection bias, missing results, correlated task families, harness changes, or optional stopping.", "Two-stage empirical bootstrap intervals are approximate. Boundary cells cannot generate unobserved outcomes and may produce zero-width intervals; no certainty claim follows.", "The archive has no complete task-selection field. Delivery source marks a known curated proxy, while other tasks have unknown selection status.", "Frozen-classifier provenance is declared by the snapshot; classifier model IDs and prompt hashes are absent, so this build cannot independently attest them.", "Most historical effort modes, execution dates and CLI versions are missing. Unknown values remain unknown.", "One snapshot-valid negative reward is excluded as invalid_reward_range in analysis. The historical inventory retains original snapshot validity and trial-domain assignments for reproducibility.", "Classifiers explain verifier grades. HARNESS_ERROR is excluded under the inherited policy, including classifier inconsistencies; this policy can introduce outcome-dependent missingness.", "Duplicate trial IDs use console > site > delivery > repo precedence. Shared timestamps do not identify duplicate executions; timestamp collisions are retained.", "Pooled-run Wilson intervals are descriptive and assume independent Bernoulli trials. Headline comparisons use task macro averages and paired task resampling.", "Sign-test values are conditional sign diagnostics, not validated tests of equal model capability; unequal repetition counts can violate the fair-sign null. No superiority inference is issued.", "Reviews are already represented in the normalized trial labels. The site hashes the review inventory but does not adjudicate or independently establish reviewer independence.", "Legacy showcase trial evidence URLs have been retired because those routes return 404. Missing public evidence is shown explicitly, not replaced with a proof claim." ] }, "tasks": [ { "id": "task_2c50bc34eab24ea6-v1", "taskId": "task_2c50bc34eab24ea6", "version": "1", "name": "rolling-expand-contract-deploy-v3", "domain": "Cloud operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: author READY: D1 crash-stub -1 + nop 0; oracle ro", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_594e02dd3c414d77-v1", "taskId": "task_594e02dd3c414d77", "version": "1", "name": "pg-timeline-fork-recovery-v3", "domain": "Cloud operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: author v3 READY; independent QA-D re-verification RUNN", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_5d20041e15ae46ae-v1", "taskId": "task_5d20041e15ae46ae", "version": "1", "name": "pg-timeline-fork-recovery-v2", "domain": "Cloud operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (independent QA-D 10-13 03:11): v1 NEE", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_62d135487c854564-v1", "taskId": "task_62d135487c854564", "version": "1", "name": "tcl-rsync-snapshot-integrity-audit", "domain": "Cloud operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed + SEE gate 1 PASS with notes (01:37)", "knownDefects": [ "Recorded concern about the empty-solution control." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_9d305f2f98544017-v1", "taskId": "task_9d305f2f98544017", "version": "1", "name": "redis-failover-v3", "domain": "Cloud operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: Local QA passed (QA-A v3)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_a1e8fd3f24804ff0-v1", "taskId": "task_a1e8fd3f24804ff0", "version": "1", "name": "external-secrets-perf-degradation-47720efc-lyra-e4d505ea", "domain": "Cloud operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_a822cbe2fcfa498f-v1", "taskId": "task_a822cbe2fcfa498f", "version": "1", "name": "tcl-rsync-snapshot-integrity-audit", "domain": "Cloud operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-H sweep (budget fairness)", "knownDefects": [ "Recorded concern about the empty-solution control." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 4, "classes": { "HARNESS_ERROR": 6, "GOOD_FAILURE": 0, "BAD_FAILURE": 2, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 6, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_ea1b7bd1586f4682-v1", "taskId": "task_ea1b7bd1586f4682", "version": "1", "name": "pg-timeline-fork-recovery-v4", "domain": "Cloud operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: Local QA passed: author v4 + INDEPENDENT QA-D REA", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_f63654ffbe9d442b-v1", "taskId": "task_f63654ffbe9d442b", "version": "1", "name": "rolling-expand-contract-deploy-v2", "domain": "Cloud operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: author READY: oracle root 1 + uid1000 1; alt (bash/psq", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_265a85be81b34483-v1", "taskId": "task_265a85be81b34483", "version": "1", "name": "bikeshare-daily-leakage-regression-v4", "domain": "Data processing / data science", "source": "console", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Qualified; QA: QA-H sweep: budget fairness (>= 50% of 75-min estimate,", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 10, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.5, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@", "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_2d7c192fb46741bf-v1", "taskId": "task_2d7c192fb46741bf", "version": "1", "name": "skglm-perspective-certificates", "domain": "Data processing / data science", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2dce90edaa3348bb-v1", "taskId": "task_2dce90edaa3348bb", "version": "1", "name": "weak-moment-location-scale", "domain": "Data processing / data science", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@", "defectReasons": [] }, { "id": "task_379d1c597e6a4e8f-v1", "taskId": "task_379d1c597e6a4e8f", "version": "1", "name": "compact-noise-reflected-divergence", "domain": "Data processing / data science", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_462d3c6816b7413b-v1", "taskId": "task_462d3c6816b7413b", "version": "1", "name": "pyclustering-selective-joint", "domain": "Data processing / data science", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 0, "n": 7, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.3543304350666875, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.3543304350666875, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.6456695649333125, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@", "defectReasons": [] }, { "id": "task_497082259544425d-v1", "taskId": "task_497082259544425d", "version": "1", "name": "esda-mh-diffusion-distance", "domain": "Data processing / data science", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_52ac6d0d5679465f-v1", "taskId": "task_52ac6d0d5679465f", "version": "1", "name": "bikeshare-daily-leakage-regression-v2", "domain": "Data processing / data science", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Rejected; QA: SEE gate-1 REJECT (00:18 IST 10-13): undisclosed 360 s p", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_56ae95087cab4b3a-v1", "taskId": "task_56ae95087cab4b3a", "version": "1", "name": "pyfixest-picchetti-ri", "domain": "Data processing / data science", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7270225c699741c2-v1", "taskId": "task_7270225c699741c2", "version": "1", "name": "augmented-gmm-covariance", "domain": "Data processing / data science", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 1, "n": 7, "p": 0.14285714285714285, "lo": 0.02567962434474355, "hi": 0.5131278292743189, "tasks": 1, "macro": 0.14285714285714285, "macroLo": 0.02567962434474355, "macroHi": 0.5131278292743189, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.8571428571428572, "difficultyLo": 0.4868721707256811, "difficultyHi": 0.9743203756552564, "strongestRow": "openai/gpt-6-astra@@", "defectReasons": [] }, { "id": "task_764dea880c534b90-v1", "taskId": "task_764dea880c534b90", "version": "1", "name": "nonconformist-kernel-quantile-path", "domain": "Data processing / data science", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 0, "n": 7, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.3543304350666875, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.3543304350666875, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.6456695649333125, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@", "defectReasons": [] }, { "id": "task_83d33200dea04aa4-v1", "taskId": "task_83d33200dea04aa4", "version": "1", "name": "lifelines-dr-cosarc", "domain": "Data processing / data science", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@", "defectReasons": [] }, { "id": "task_997db6b233e7428a-v1", "taskId": "task_997db6b233e7428a", "version": "1", "name": "bikeshare-daily-leakage-regression", "domain": "Data processing / data science", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.6, "difficultyLo": 0.23072428127601297, "difficultyHi": 0.8823792257673522, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_b064031828f2474f-v1", "taskId": "task_b064031828f2474f", "version": "1", "name": "generative-process-two-sample", "domain": "Data processing / data science", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b0cd41c467f24ad4-v1", "taskId": "task_b0cd41c467f24ad4", "version": "1", "name": "ruptures_grace", "domain": "Data processing / data science", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e15a53cd0ba74b66-v1", "taskId": "task_e15a53cd0ba74b66", "version": "1", "name": "store-remodel-incrementality-v1", "domain": "Data processing / data science", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-D2 independent READY (truth recomputed to 1e-1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_e40788e9529d472e-v1", "taskId": "task_e40788e9529d472e", "version": "1", "name": "dcor-taustar-ties", "domain": "Data processing / data science", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f14cf7ad30bd49f3-v1", "taskId": "task_f14cf7ad30bd49f3", "version": "1", "name": "bikeshare-daily-leakage-regression-v3", "domain": "Data processing / data science", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: SEE gate 1 PASS (v3 text)", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_03f5f9f4f811412b-v1", "taskId": "task_03f5f9f4f811412b", "version": "1", "name": "syncrope/crossterm-vt-interpreter", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_07f9db62401542be-v1", "taskId": "task_07f9db62401542be", "version": "1", "name": "syncrope/semver-pubgrub-resolver", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_0d0eb0d05b024025-v1", "taskId": "task_0d0eb0d05b024025", "version": "1", "name": "timescaledb-fix-invalidation-and-batchin", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0d0eb0d05b024025-v2", "taskId": "task_0d0eb0d05b024025", "version": "2", "name": "timescaledb-fix-invalidation-and-batchin", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_0d0eb0d05b024025-v3", "taskId": "task_0d0eb0d05b024025", "version": "3", "name": "timescaledb-fix-invalidation-and-batchin", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_0d0eb0d05b024025-v4", "taskId": "task_0d0eb0d05b024025", "version": "4", "name": "timescaledb-fix-invalidation-and-batchin", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "deepswe review: EXCLUDED from delivery (coordinator, 2026-10-22 ~02:50Z); runs c", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_109a82d7da884780-v1", "taskId": "task_109a82d7da884780", "version": "1", "name": "pydub-dsp-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_11657b566bd84269-v1", "taskId": "task_11657b566bd84269", "version": "1", "name": "qdrant-fix-gridstore-has-pointer-lookup", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 2, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.6666666666666666, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Task fault on at least half of labelled current-model failures" ] }, { "id": "task_12173d76aea547e8-v1", "taskId": "task_12173d76aea547e8", "version": "1", "name": "syncrope/etree-xpath10-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/etree-xpath10-engine-qa.md)", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_14752158278742f6-v1", "taskId": "task_14752158278742f6", "version": "1", "name": "qdrant-fix-gridstore-has-pointer-lookup", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_14a0c1160d2b418f-v1", "taskId": "task_14a0c1160d2b418f", "version": "1", "name": "faiss-scan-exactly-max-codes-elements-26", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_14e103e2e9454d6b-v1", "taskId": "task_14e103e2e9454d6b", "version": "1", "name": "syncrope/marked-commonmark-strict", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 1 }, "harnessErrors": 1, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_18a5ba5874ae458c-v1", "taskId": "task_18a5ba5874ae458c", "version": "1", "name": "oxigraph-write-policy-ec68e3d", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_1abf61aae49a429b-v1", "taskId": "task_1abf61aae49a429b", "version": "1", "name": "path-to-regexp-route-fragments-bd12a33", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.6-sol@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_1b302d95ecf84eeb-v1", "taskId": "task_1b302d95ecf84eeb", "version": "1", "name": "timescaledb-fix-segmentby-crash-in-cagg", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_23152389f9124dcb-v1", "taskId": "task_23152389f9124dcb", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.6, "difficultyLo": 0.23072428127601297, "difficultyHi": 0.8823792257673522, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_28206617bb4d469a-v1", "taskId": "task_28206617bb4d469a", "version": "1", "name": "syncrope/cascadia-selectors-level4", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2ad9f7dfda1e4f8c-v1", "taskId": "task_2ad9f7dfda1e4f8c", "version": "1", "name": "sqlite-distinct-agg-arity", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_2ad9f7dfda1e4f8c-v2", "taskId": "task_2ad9f7dfda1e4f8c", "version": "2", "name": "sqlite-distinct-agg-arity", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "deepswe review: EXCLUDED from delivery (coordinator, 2026-10-22 ~00:25Z)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "tencent/hy4-preview@@think", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_2f8a69fe4cfd42a0-v1", "taskId": "task_2f8a69fe4cfd42a0", "version": "1", "name": "syncrope/regexptree-exec-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.19999999999999996, "difficultyLo": 0.036224108632430196, "difficultyHi": 0.6244653702374747, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_306b20d05cc54ae1-v1", "taskId": "task_306b20d05cc54ae1", "version": "1", "name": "jsbarcode-qr-model2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_32393fcd1ac640e3-v1", "taskId": "task_32393fcd1ac640e3", "version": "1", "name": "etcd-fix-race-hint-validation", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3677102caba14694-v1", "taskId": "task_3677102caba14694", "version": "1", "name": "syncrope/quick-xml-c14n", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-J READY: oracle root/uid1000/host-patch 1; nop", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3b8bbd5b547043af-v1", "taskId": "task_3b8bbd5b547043af", "version": "1", "name": "syncrope/echo-http-signatures", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3fbbd1e5aaef4a84-v1", "taskId": "task_3fbbd1e5aaef4a84", "version": "1", "name": "nltk-subword-tokenization", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4063b4ecff23444b-v1", "taskId": "task_4063b4ecff23444b", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_40df7b0ce1654f8f-v1", "taskId": "task_40df7b0ce1654f8f", "version": "1", "name": "syncrope/ammonia-sanitizer-api", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_414b118303d24680-v1", "taskId": "task_414b118303d24680", "version": "1", "name": "syncrope/fasthttp-websocket-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_42603212f8434552-v1", "taskId": "task_42603212f8434552", "version": "1", "name": "syncrope/cheerio-aria-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 7, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.3543304350666875, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.3543304350666875, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.6456695649333125, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_435cb8db6e274ba5-v1", "taskId": "task_435cb8db6e274ba5", "version": "1", "name": "qdrant-multivectors-hint-validation", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_48e33847ac1244bc-v1", "taskId": "task_48e33847ac1244bc", "version": "1", "name": "syncrope/networkx-graphml-full", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 9, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.29914504841954404, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.29914504841954404, "labelled": 9, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 9, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.8, "difficultyLo": 0.3755346297625253, "difficultyHi": 0.9637758913675699, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_48e33847ac1244bc-v2", "taskId": "task_48e33847ac1244bc", "version": "2", "name": "syncrope/networkx-graphml-full", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4ad2a01eff464d32-v1", "taskId": "task_4ad2a01eff464d32", "version": "1", "name": "syncrope/gopher-lua-54-semantics", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-A wave 2): oracle 1x3 root + uid10", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_55f55e872cbc4753-v1", "taskId": "task_55f55e872cbc4753", "version": "1", "name": "qdrant-fix-wrong-multivectors-count-in-s", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5716741dd58d4ee5-v1", "taskId": "task_5716741dd58d4ee5", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5b2abc5cb94d4a75-v1", "taskId": "task_5b2abc5cb94d4a75", "version": "1", "name": "gopher-lua-exact-integers-75f4976", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.6-sol@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_5c156a1be6db47fc-v1", "taskId": "task_5c156a1be6db47fc", "version": "1", "name": "syncrope/uniseg-bidi-uax9", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 1, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.19999999999999996, "difficultyLo": 0.036224108632430196, "difficultyHi": 0.6244653702374747, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_6672f1a44a2d4a3f-v1", "taskId": "task_6672f1a44a2d4a3f", "version": "1", "name": "crossterm-vt-interpreter", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6df2c2ca80cb42d4-v1", "taskId": "task_6df2c2ca80cb42d4", "version": "1", "name": "syncrope/jsbarcode-qr-model2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 4, "p": 0, "lo": 0, "hi": 0.4898908364545973, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.4898908364545973, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_7034c6eb7b39483d-v1", "taskId": "task_7034c6eb7b39483d", "version": "1", "name": "syncrope/linkedom-range-selection", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-J READY", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_752da4e1649540f2-v1", "taskId": "task_752da4e1649540f2", "version": "1", "name": "syncrope/paramiko-knownhosts-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 9, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.29914504841954404, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.29914504841954404, "labelled": 9, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 9, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.8, "difficultyLo": 0.3755346297625253, "difficultyHi": 0.9637758913675699, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_752da4e1649540f2-v2", "taskId": "task_752da4e1649540f2", "version": "2", "name": "syncrope/paramiko-knownhosts-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.2, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_76ef8bbffb424b20-v1", "taskId": "task_76ef8bbffb424b20", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": null, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_79559260b7e3427d-v1", "taskId": "task_79559260b7e3427d", "version": "1", "name": "syncrope/nltk-subword-tokenization", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_7b18be5807394275-v1", "taskId": "task_7b18be5807394275", "version": "1", "name": "semver-pubgrub-resolver", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8166eb6ec6f14fb4-v1", "taskId": "task_8166eb6ec6f14fb4", "version": "1", "name": "syncrope/etree-xpath10-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed + SEE gate 1 PASS with notes (both tex", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_84e4bb52932545db-v1", "taskId": "task_84e4bb52932545db", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (budget)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_87a05faca03f4898-v1", "taskId": "task_87a05faca03f4898", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": null, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_8934246571b14c03-v1", "taskId": "task_8934246571b14c03", "version": "1", "name": "syncrope/sqliteutils-btree-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_8f19a886eb7c4005-v1", "taskId": "task_8f19a886eb7c4005", "version": "1", "name": "sqlite-distinct-agg-arity", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9dd118dd4f614701-v1", "taskId": "task_9dd118dd4f614701", "version": "1", "name": "syncrope/bibtexparser-biblatex-model", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_a002f3acdd5d49f9-v1", "taskId": "task_a002f3acdd5d49f9", "version": "1", "name": "syncrope/spdx-matching-guidelines", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 1, "n": 3, "p": 0.3333333333333333, "lo": 0.06149194472039615, "hi": 0.7923403991979523, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.06149194472039615, "macroHi": 0.7923403991979523, "labelled": 3, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 1, "BAD_FAILURE": 1, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0.3333333333333333, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 4, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@", "defectReasons": [ "Task fault on at least half of labelled current-model failures" ] }, { "id": "task_a320d02d5d2042d2-v1", "taskId": "task_a320d02d5d2042d2", "version": "1", "name": "syncrope/quick-xml-c14n", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_a81d43af84a848ce-v1", "taskId": "task_a81d43af84a848ce", "version": "1", "name": "orb-projection-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b09ffd88dd38438e-v1", "taskId": "task_b09ffd88dd38438e", "version": "1", "name": "syncrope/orb-projection-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_b4cd94ae9b6d459c-v1", "taskId": "task_b4cd94ae9b6d459c", "version": "1", "name": "regexptree-exec-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ba73c622bbe24dc6-v1", "taskId": "task_ba73c622bbe24dc6", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_bb2e200d1de14882-v1", "taskId": "task_bb2e200d1de14882", "version": "1", "name": "syncrope/oxigraph-shacl-core", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.19999999999999996, "difficultyLo": 0.036224108632430196, "difficultyHi": 0.6244653702374747, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_bb2e200d1de14882-v2", "taskId": "task_bb2e200d1de14882", "version": "2", "name": "syncrope/oxigraph-shacl-core", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.6, "difficultyLo": 0.23072428127601297, "difficultyHi": 0.8823792257673522, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_be13c5ae46354493-v1", "taskId": "task_be13c5ae46354493", "version": "1", "name": "syncrope/iconv-lite-whatwg-mode", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 1, "n": 3, "p": 0.3333333333333333, "lo": 0.06149194472039615, "hi": 0.7923403991979523, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.06149194472039615, "macroHi": 0.7923403991979523, "labelled": 3, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0.6666666666666666, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.19999999999999996, "difficultyLo": 0.036224108632430196, "difficultyHi": 0.6244653702374747, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_c35aa2775044472f-v1", "taskId": "task_c35aa2775044472f", "version": "1", "name": "syncrope/gopher-lua-54-semantics", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (budget)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_c4f682a6d8d6463e-v1", "taskId": "task_c4f682a6d8d6463e", "version": "1", "name": "faiss-scan-exactly-max-codes-elements-26", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c4f682a6d8d6463e-v2", "taskId": "task_c4f682a6d8d6463e", "version": "2", "name": "faiss-scan-exactly-max-codes-elements-26", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c4f682a6d8d6463e-v3", "taskId": "task_c4f682a6d8d6463e", "version": "3", "name": "faiss-scan-exactly-max-codes-elements-26", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "deepswe review: ON HOLD, excluded from tonight's delivery (coordinator, 2026-10-", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_cb67d085c2e3483b-v1", "taskId": "task_cb67d085c2e3483b", "version": "1", "name": "timescaledb-fix-segmentby-crash-in-cagg", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_cfa8761a3a5e4e4c-v1", "taskId": "task_cfa8761a3a5e4e4c", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d073861da1174cfc-v1", "taskId": "task_d073861da1174cfc", "version": "1", "name": "paramiko-knownhosts-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.6-sol@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_d3646a9a5a9e4ec0-v1", "taskId": "task_d3646a9a5a9e4ec0", "version": "1", "name": "cheerio-aria-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d483c707635f46fd-v1", "taskId": "task_d483c707635f46fd", "version": "1", "name": "syncrope/n3js-rdf-canonicalization", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d8b20176b252466e-v1", "taskId": "task_d8b20176b252466e", "version": "1", "name": "timescaledb-fix-invalidation-and-batchin", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e11a661d1cf24cdd-v1", "taskId": "task_e11a661d1cf24cdd", "version": "1", "name": "sqliteutils-btree-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e4fdd21719434f58-v1", "taskId": "task_e4fdd21719434f58", "version": "1", "name": "syncrope/etree-xpath10-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (budget)", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_e6929d3809664fe0-v1", "taskId": "task_e6929d3809664fe0", "version": "1", "name": "fasthttp-websocket-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ea263a329eda4a29-v1", "taskId": "task_ea263a329eda4a29", "version": "1", "name": "syncrope/jimp-exif-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 3, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ed834aff171c4f06-v1", "taskId": "task_ed834aff171c4f06", "version": "1", "name": "syncrope/pint-ucum-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_efac1df7688e4a98-v1", "taskId": "task_efac1df7688e4a98", "version": "1", "name": "qdrant-fix-wrong-multivectors-count-in-s", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 1, "n": 3, "p": 0.3333333333333333, "lo": 0.06149194472039615, "hi": 0.7923403991979523, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.06149194472039615, "macroHi": 0.7923403991979523, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.6666666666666666, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_efac1df7688e4a98-v2", "taskId": "task_efac1df7688e4a98", "version": "2", "name": "qdrant-fix-wrong-multivectors-count-in-s", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_efac1df7688e4a98-v3", "taskId": "task_efac1df7688e4a98", "version": "3", "name": "qdrant-fix-wrong-multivectors-count-in-s", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_efac1df7688e4a98-v4", "taskId": "task_efac1df7688e4a98", "version": "4", "name": "qdrant-fix-wrong-multivectors-count-in-s", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "deepswe review: OUT of delivery (coordinator, 2026-10-22 ~03:00Z): fails D2; rem", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 3, "n": 8, "p": 0.375, "lo": 0.13684428582359737, "hi": 0.6942576053973728, "tasks": 1, "macro": 0.375, "macroLo": 0.13684428582359737, "macroHi": 0.6942576053973728, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.625, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@high", "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_f0d5c3954cf64995-v1", "taskId": "task_f0d5c3954cf64995", "version": "1", "name": "syncrope/linkedom-range-selection", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f1883a39d2ee49b1-v1", "taskId": "task_f1883a39d2ee49b1", "version": "1", "name": "syncrope/goldmark-cmark-sourcepos", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f23a5f1298c94cb9-v1", "taskId": "task_f23a5f1298c94cb9", "version": "1", "name": "syncrope/gopher-lua-54-semantics", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f23a5f1298c94cb9-v2", "taskId": "task_f23a5f1298c94cb9", "version": "2", "name": "syncrope/gopher-lua-54-semantics", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4, "difficultyLo": 0.11762077423264805, "difficultyHi": 0.7692757187239871, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_f690cc4d68e94d74-v1", "taskId": "task_f690cc4d68e94d74", "version": "1", "name": "syncrope/path-to-regexp-urlpattern", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gemini-flash-latest@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0.5, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4, "difficultyLo": 0.11762077423264805, "difficultyHi": 0.7692757187239871, "strongestRow": "gpt-5.6-sol@@", "defectReasons": [] }, { "id": "task_f690cc4d68e94d74-v2", "taskId": "task_f690cc4d68e94d74", "version": "2", "name": "syncrope/path-to-regexp-urlpattern", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 0.8, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f8508f3d38fa462b-v1", "taskId": "task_f8508f3d38fa462b", "version": "1", "name": "syncrope/babel-messageformat2", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-A wave 2): oracle 1x3 root + uid10", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_fcb17ab842554cdb-v1", "taskId": "task_fcb17ab842554cdb", "version": "1", "name": "syncrope/pydub-dsp-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_113e8f32cb144742-v1", "taskId": "task_113e8f32cb144742", "version": "1", "name": "page-program-suspend", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_125106d14ad9412a-v1", "taskId": "task_125106d14ad9412a", "version": "1", "name": "debug-halt-step-fsm", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_173001e9d6804261-v1", "taskId": "task_173001e9d6804261", "version": "1", "name": "apb-lite-slave-regfile", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_1bc80656db8540d5-v1", "taskId": "task_1bc80656db8540d5", "version": "1", "name": "noc-vc-wormhole-router-v3", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Rejected: too-easy (screen 3/3 genuine) -> v4 TODO; QA: QA-D: local la", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_25efe679d95844e1-v1", "taskId": "task_25efe679d95844e1", "version": "1", "name": "terminal-bench/netlist-rtl-recovery", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Rejected; QA: QA-P1 READY", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_2822c50d730e43fe-v1", "taskId": "task_2822c50d730e43fe", "version": "1", "name": "coprocessor-dispatcher-classmix", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_28fa60a6a50546ba-v1", "taskId": "task_28fa60a6a50546ba", "version": "1", "name": "stream-frame-hold", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": null, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_3189f881254944b5-v1", "taskId": "task_3189f881254944b5", "version": "1", "name": "bus-slave-abort-ack", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3961573458f54ac1-v1", "taskId": "task_3961573458f54ac1", "version": "1", "name": "nonblocking-wb-cache-mshr-v1", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Author READY (QA-C); independent QA-D pending", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_3ed58d284f734fdd-v1", "taskId": "task_3ed58d284f734fdd", "version": "1", "name": "axi-interconnect-ordering-v2", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Rejected; QA: QA-D (shadow-design hole fixed; 2 alternates pass, fabri", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_4a4cfca10af34f53-v1", "taskId": "task_4a4cfca10af34f53", "version": "1", "name": "rv32i-pipeline-precise-traps-v6", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Rejected: too-easy (screen 3/3 genuine) -> v7 TODO; QA: QA-D READY (or", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4c4afdcd3aef4219-v1", "taskId": "task_4c4afdcd3aef4219", "version": "1", "name": "rv32i-pipeline-precise-traps-v3", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (author QA-C + independent QA-D, 10-13", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_5763254b63af4339-v1", "taskId": "task_5763254b63af4339", "version": "1", "name": "bcd-tens-rollover", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5bc1e5f3583947db-v1", "taskId": "task_5bc1e5f3583947db", "version": "1", "name": "axi-interconnect-ordering-v1", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Author READY (QA-C); independent QA-D pending", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_5e76a96d838b406f-v1", "taskId": "task_5e76a96d838b406f", "version": "1", "name": "noc-vc-wormhole-router-v2", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-A v2 port; QA-D independent on v1)", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6b8f42231cb940ae-v1", "taskId": "task_6b8f42231cb940ae", "version": "1", "name": "rtl-axi-event-snapshot-bank", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@xhigh", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6f4be42c66204984-v1", "taskId": "task_6f4be42c66204984", "version": "1", "name": "nonblocking-wb-cache-mshr-v3", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Independent QA-D READY (oracle root+uid1000 1, c_m4 1,", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_778ee0bfa5474eb3-v1", "taskId": "task_778ee0bfa5474eb3", "version": "1", "name": "serial-receiver-framed", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_784f195a701f427f-v1", "taskId": "task_784f195a701f427f", "version": "1", "name": "deflate-inflate-streaming-v1", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: author QA-C READY; independent QA-D RUNNING", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_7eb184590bc44793-v1", "taskId": "task_7eb184590bc44793", "version": "1", "name": "noc-vc-wormhole-router-v1", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed: author QA-C + INDEPENDENT QA-D READY ", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_828beca732214d72-v1", "taskId": "task_828beca732214d72", "version": "1", "name": "serial-break-resync", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_84c8c181919447da-v1", "taskId": "task_84c8c181919447da", "version": "1", "name": "round-robin-arbiter-grant", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b128ec9a19b3410e-v1", "taskId": "task_b128ec9a19b3410e", "version": "1", "name": "rv32i-pipeline-precise-traps-v4", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed: author QA-C v4 + INDEPENDENT QA-D REA", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_b5802ccdd9e440a4-v1", "taskId": "task_b5802ccdd9e440a4", "version": "1", "name": "deflate-inflate-streaming-v3", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-D READY (oracle root/uid1000 1, nop 0, alt_combaddr", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_d2e841f3824d4584-v1", "taskId": "task_d2e841f3824d4584", "version": "1", "name": "byte-serial-round-scheduler", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d5eeb6548dc94dd8-v1", "taskId": "task_d5eeb6548dc94dd8", "version": "1", "name": "rv32i-pipeline-precise-traps-v5", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-A v5 port)", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e2137dbcb9f847d7-v1", "taskId": "task_e2137dbcb9f847d7", "version": "1", "name": "deflate-inflate-streaming-v2", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Superseded; QA: Independent QA-D READY (oracle root+uid1000=1, nop 0, ", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e61401e857ba4be5-v1", "taskId": "task_e61401e857ba4be5", "version": "1", "name": "dualmaster-membridge", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_eec1126c625e471f-v1", "taskId": "task_eec1126c625e471f", "version": "1", "name": "axi-interconnect-ordering-v3", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-D: oracle root/uid1000 1 (45/45); 2 alternates", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_fbd7ec2b60e94fe8-v1", "taskId": "task_fbd7ec2b60e94fe8", "version": "1", "name": "hash-message-padder", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fddd74714b2a4980-v1", "taskId": "task_fddd74714b2a4980", "version": "1", "name": "rtl-dma-tag-credit-tracker", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ffabad2ecc46495e-v1", "taskId": "task_ffabad2ecc46495e", "version": "1", "name": "nonblocking-wb-cache-mshr-v4", "domain": "Electrical engineering (RTL)", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-D (.sfunc/SIGPIPE fix; 6000-line probe: v4 rej", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_007db1b630754fd7-v1", "taskId": "task_007db1b630754fd7", "version": "1", "name": "modded-campaign-save-rebase", "domain": "Games", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_06f30bf9b3d342af-v1", "taskId": "task_06f30bf9b3d342af", "version": "1", "name": "mine-sweep-run", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_074849bc26ee4c41-v1", "taskId": "task_074849bc26ee4c41", "version": "1", "name": "pathfinding-memory-optimize-grid", "domain": "Games", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0aaf7fc7cc7b4cf0-v1", "taskId": "task_0aaf7fc7cc7b4cf0", "version": "1", "name": "tower-siege", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_0e3bdc358aaa48eb-v1", "taskId": "task_0e3bdc358aaa48eb", "version": "1", "name": "negotiator-table", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_11dc705631944c69-v1", "taskId": "task_11dc705631944c69", "version": "1", "name": "alchemy-cauldron", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_184768488f684128-v1", "taskId": "task_184768488f684128", "version": "1", "name": "glass-harbor-time-loop-v1", "domain": "Games", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-F): all 8 canonical journals re-de", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_240b68d6609c4998-v1", "taskId": "task_240b68d6609c4998", "version": "1", "name": "kelp-ring-auction", "domain": "Games", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_265c3826c2ae4786-v1", "taskId": "task_265c3826c2ae4786", "version": "1", "name": "glass-harbor-time-loop-v2", "domain": "Games", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-F): all 8 canonical journals re-de", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_28d272c3abf54d9f-v1", "taskId": "task_28d272c3abf54d9f", "version": "1", "name": "glass-harbor-time-loop-v3", "domain": "Games", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (write-early; v2 never in trials)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_30b1e1c8ca5d45a0-v1", "taskId": "task_30b1e1c8ca5d45a0", "version": "1", "name": "crusader-sprite-assembly", "domain": "Games", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3fd47b5383b6460b-v1", "taskId": "task_3fd47b5383b6460b", "version": "1", "name": "orbit-ferry", "domain": "Games", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.19999999999999996, "difficultyLo": 0.036224108632430196, "difficultyHi": 0.6244653702374747, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_460b309aec69444d-v1", "taskId": "task_460b309aec69444d", "version": "1", "name": "tbrain-slantway-move-legality", "domain": "Games", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4868dff5988142f9-v1", "taskId": "task_4868dff5988142f9", "version": "1", "name": "outpost-arena", "domain": "Games", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.19999999999999996, "difficultyLo": 0.036224108632430196, "difficultyHi": 0.6244653702374747, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_5ee788f6e4ed4445-v1", "taskId": "task_5ee788f6e4ed4445", "version": "1", "name": "lattice-signals", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_719494943d8949ff-v1", "taskId": "task_719494943d8949ff", "version": "1", "name": "lines-of-action-move-legality-v3", "domain": "Games", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: Local QA passed (lead, 10-13 02:25): diff-only test.sh", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_7c15117244d043e3-v1", "taskId": "task_7c15117244d043e3", "version": "1", "name": "lines-of-action-move-legality-v2", "domain": "Games", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/lines-of-action-move-legality-", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_83de5532825d437f-v1", "taskId": "task_83de5532825d437f", "version": "1", "name": "triage-ward", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_9403da6e6d1f48a2-v1", "taskId": "task_9403da6e6d1f48a2", "version": "1", "name": "arb-desk", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_ad25a1a289c74016-v1", "taskId": "task_ad25a1a289c74016", "version": "1", "name": "go-stone-arena", "domain": "Games", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e0fb33c55ed44307-v1", "taskId": "task_e0fb33c55ed44307", "version": "1", "name": "reef-graze", "domain": "Games", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_10275360bb104705-v1", "taskId": "task_10275360bb104705", "version": "1", "name": "bank-additional-macro-duration-leakage", "domain": "Machine learning", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 8, "p": 0.625, "lo": 0.3057423946026273, "hi": 0.8631557141764027, "tasks": 1, "macro": 0.625, "macroLo": 0.3057423946026273, "macroHi": 0.8631557141764027, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.375, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.375, "difficultyLo": 0.1368442858235973, "difficultyHi": 0.6942576053973727, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_16ce94f5b938473e-v1", "taskId": "task_16ce94f5b938473e", "version": "1", "name": "squeeze-qwen3.5-0.8b-300mb", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 9, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 9, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 11, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 11, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_17155ef80fdd40ed-v1", "taskId": "task_17155ef80fdd40ed", "version": "1", "name": "cdsandvinyl-imbalance-sentmismatch-text-vocabdrift", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_1dd8dd076c474c45-v1", "taskId": "task_1dd8dd076c474c45", "version": "1", "name": "lalonde-doubly-robust-ate", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_1f787a0df1884007-v1", "taskId": "task_1f787a0df1884007", "version": "1", "name": "mbias-pretreatment-collider-ate", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_24ae66ce9a0a41c3-v1", "taskId": "task_24ae66ce9a0a41c3", "version": "1", "name": "healthandhousehold-helpfulvotes-heteroscedastic-missing-monotone", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_24c0f2181d9d4287-v1", "taskId": "task_24c0f2181d9d4287", "version": "1", "name": "loan-default-cost-imbalance", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.6, "difficultyLo": 0.23072428127601297, "difficultyHi": 0.8823792257673522, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_25583b5ca1fa4cc3-v1", "taskId": "task_25583b5ca1fa4cc3", "version": "1", "name": "movies-drama-imbalance-sentmismatch-text", "domain": "Machine learning", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4, "difficultyLo": 0.11762077423264805, "difficultyHi": 0.7692757187239871, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_2a71367699ea4400-v1", "taskId": "task_2a71367699ea4400", "version": "1", "name": "worst-group-spurious-dfr", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2b294665791c4d8a-v1", "taskId": "task_2b294665791c4d8a", "version": "1", "name": "spotify-songs-modeling", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2cb05e1584b8469b-v1", "taskId": "task_2cb05e1584b8469b", "version": "1", "name": "critical-care-survival-censoring-calibration", "domain": "Machine learning", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_2f4da329af9e40be-v1", "taskId": "task_2f4da329af9e40be", "version": "1", "name": "multisequence-identity-scorecard", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_34c1cb48e1c24aec-v1", "taskId": "task_34c1cb48e1c24aec", "version": "1", "name": "acsincome-shift-calibration", "domain": "Machine learning", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 1 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4, "difficultyLo": 0.11762077423264805, "difficultyHi": 0.7692757187239871, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_384495d8bf274cd6-v1", "taskId": "task_384495d8bf274cd6", "version": "1", "name": "health-baseline-helpfulvotes-monotone-v2", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3a1df3c73d3049e3-v1", "taskId": "task_3a1df3c73d3049e3", "version": "1", "name": "maintenance-therapy-competing-risks-cif", "domain": "Machine learning", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_3c7280b2b77845fa-v1", "taskId": "task_3c7280b2b77845fa", "version": "1", "name": "hausman-taylor-time-invariant-endogenous-effect", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3f34010b10804509-v1", "taskId": "task_3f34010b10804509", "version": "1", "name": "adult-calib-imbalance", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3f59a2136b2b4832-v1", "taskId": "task_3f59a2136b2b4832", "version": "1", "name": "clothing-imbalance-leakage-metafusion-text", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_49910916742b4868-v1", "taskId": "task_49910916742b4868", "version": "1", "name": "nvss-conceptdrift-heteroscedastic-monotone-multiout", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_49c52a02207540f8-v1", "taskId": "task_49c52a02207540f8", "version": "1", "name": "loan-default-cost-imbalance-v2", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4f57e739c00a4f44-v1", "taskId": "task_4f57e739c00a4f44", "version": "1", "name": "delay-regression-group-holdout", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_533aa5622794420d-v1", "taskId": "task_533aa5622794420d", "version": "1", "name": "immortal-time-bias-tvc-survival-hr-v2", "domain": "Machine learning", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_54200b8fd93b4b0d-v1", "taskId": "task_54200b8fd93b4b0d", "version": "1", "name": "forest-fires-skewed-regression", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_58621eb0f98a4892-v1", "taskId": "task_58621eb0f98a4892", "version": "1", "name": "meeting-multitrack-reconcile", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.5, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.7224672001371109, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_58ad2319a3c64b9a-v1", "taskId": "task_58ad2319a3c64b9a", "version": "1", "name": "reverse-engineer-objectives-v3", "domain": "Machine learning", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: awaiting SEE gate-1 check on v3; note: host lane ", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_59eb74c10b6e41d7-v1", "taskId": "task_59eb74c10b6e41d7", "version": "1", "name": "grok-modular-arithmetic", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5ca57348af7843da-v1", "taskId": "task_5ca57348af7843da", "version": "1", "name": "bike-sharing-regression-timeaware-v2", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_606ea0833b2d4633-v1", "taskId": "task_606ea0833b2d4633", "version": "1", "name": "weighted-ranking-audit", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 8, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 8, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_64cb16d0c2c542b0-v1", "taskId": "task_64cb16d0c2c542b0", "version": "1", "name": "qat-int2-cifar", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_64d5125dda834c57-v1", "taskId": "task_64d5125dda834c57", "version": "1", "name": "openfda-drug-route-calib-imbalance-temporal", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6557e1bec8414a9c-v1", "taskId": "task_6557e1bec8414a9c", "version": "1", "name": "hotel-bookings-modeling", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6f25464ae86a4a25-v1", "taskId": "task_6f25464ae86a4a25", "version": "1", "name": "label-noise-robust-loss", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_76275fa8f81544c6-v1", "taskId": "task_76275fa8f81544c6", "version": "1", "name": "kindle-imbalance-sentmismatch-text-vocabdrift", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_773848f5343846d8-v1", "taskId": "task_773848f5343846d8", "version": "1", "name": "kd-param-budget", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7aa7b78357d74823-v1", "taskId": "task_7aa7b78357d74823", "version": "1", "name": "fewlabel-ssl-fixmatch", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_817beaea022844f9-v1", "taskId": "task_817beaea022844f9", "version": "1", "name": "length-gen-addition", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8505ce85fff14d50-v1", "taskId": "task_8505ce85fff14d50", "version": "1", "name": "sensor-placement-shift-rare-activity", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 4, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0.25, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8901d2a86bbc44c2-v1", "taskId": "task_8901d2a86bbc44c2", "version": "1", "name": "airquality-benzene-sensor-leakage", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8ffa4841e82946c1-v1", "taskId": "task_8ffa4841e82946c1", "version": "1", "name": "openfda-imbalance-multilabel-text", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9090bc32da5f41ae-v1", "taskId": "task_9090bc32da5f41ae", "version": "1", "name": "scratch-autograd-lm", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9a7318b537894c8c-v1", "taskId": "task_9a7318b537894c8c", "version": "1", "name": "terminal-bench/benchmark-gold-forensics", "domain": "Machine learning", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-P2 (v1 local evidence applies: oracle root/uid1000 ", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_9b632e321032481e-v1", "taskId": "task_9b632e321032481e", "version": "1", "name": "petsupplies-coldstartuser-groupcv-metafusion-seqrec-v2", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9eee270ffa52448e-v1", "taskId": "task_9eee270ffa52448e", "version": "1", "name": "bike-sharing-regression-timeaware", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_a8c0928f1e7f4a2f-v1", "taskId": "task_a8c0928f1e7f4a2f", "version": "1", "name": "prune-sparsity-budget", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_a93ea238c640451c-v1", "taskId": "task_a93ea238c640451c", "version": "1", "name": "interval-event-evaluation", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 8, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 8, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_aeaf9cad3f3e46ff-v1", "taskId": "task_aeaf9cad3f3e46ff", "version": "1", "name": "brfss-calib-imbalance-groupcv-svywt", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_aeb8875faf944628-v1", "taskId": "task_aeb8875faf944628", "version": "1", "name": "multisequence-identity-scorecard", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 8, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 8, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_aedca5b87cab4746-v1", "taskId": "task_aedca5b87cab4746", "version": "1", "name": "aps-failure-cost-imbalance", "domain": "Machine learning", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_b3149ffa2eda4e8c-v1", "taskId": "task_b3149ffa2eda4e8c", "version": "1", "name": "office-calib-catencode-cost-leakage", "domain": "Machine learning", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_b40ac17c5a9d4af9-v1", "taskId": "task_b40ac17c5a9d4af9", "version": "1", "name": "landmark-competing-risk-scorecard", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 8, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 8, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b48d044b6fcc43c7-v1", "taskId": "task_b48d044b6fcc43c7", "version": "1", "name": "landmark-competing-risk-scorecard", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@xhigh", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b6b7ff00f321461e-v1", "taskId": "task_b6b7ff00f321461e", "version": "1", "name": "refresh/rlhf-decoding-primitives", "domain": "Machine learning", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_b6c40fb1fe08495b-v1", "taskId": "task_b6c40fb1fe08495b", "version": "1", "name": "paged-kvcache-inference-engine-v1", "domain": "Machine learning", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: Local QA passed (author incl. 2 blind solves); SE", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_c351ea16f6e147bd-v1", "taskId": "task_c351ea16f6e147bd", "version": "1", "name": "reverse-engineer-objectives-v2", "domain": "Machine learning", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Rejected; QA: SEE gate-1 REJECT (00:55 IST): static-scan limits (32 KB", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_c6d3374309dd42c0-v1", "taskId": "task_c6d3374309dd42c0", "version": "1", "name": "fars-leakage-multiclass", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.6, "difficultyLo": 0.31267376973365835, "difficultyHi": 0.8318196702937639, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_c90fee5238af4b7d-v1", "taskId": "task_c90fee5238af4b7d", "version": "1", "name": "nhis-adult-calib-pooling-selbias-svywt", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d525db2f015c4333-v1", "taskId": "task_d525db2f015c4333", "version": "1", "name": "ai4i-maintenance-leakage-imbalance-cost", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 7, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.3543304350666875, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.3543304350666875, "labelled": 7, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": 0.8571428571428571, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.6456695649333125, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_d84258911faf49c3-v1", "taskId": "task_d84258911faf49c3", "version": "1", "name": "adv-robust-pgd", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_dcd41368cf4c4fe8-v1", "taskId": "task_dcd41368cf4c4fe8", "version": "1", "name": "appliances-helpfulvotes-heteroscedastic-multiout", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e79433d2c065466f-v1", "taskId": "task_e79433d2c065466f", "version": "1", "name": "brfss-agestd-groupcv-stratifiedeval-svywt", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ec929f9e12bf47bb-v1", "taskId": "task_ec929f9e12bf47bb", "version": "1", "name": "immortal-time-bias-tvc-survival-hr", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 9, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.9, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 0.9, "difficultyLo": 0.5958499732047615, "difficultyHi": 0.9821237869049271, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_ef9de343f4354e22-v1", "taskId": "task_ef9de343f4354e22", "version": "1", "name": "petsupplies-coldstartuser-groupcv-metafusion-seqrec", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f13252be591241dd-v1", "taskId": "task_f13252be591241dd", "version": "1", "name": "nhanes-brfss-adult-conceptdrift-xcat-selbias-svywt", "domain": "Machine learning", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f1aa48615d644c7f-v1", "taskId": "task_f1aa48615d644c7f", "version": "1", "name": "ensemble-missingness-scorecard", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 8, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 8, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f42ec79c59944cc4-v1", "taskId": "task_f42ec79c59944cc4", "version": "1", "name": "health-baseline-helpfulvotes-monotone", "domain": "Machine learning", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 9, "p": 0.3333333333333333, "lo": 0.1205838183869109, "hi": 0.6457978644196036, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.1205838183869109, "macroHi": 0.6457978644196036, "labelled": 9, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.6666666666666666, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.6666666666666667, "difficultyLo": 0.35420213558039637, "difficultyHi": 0.8794161816130891, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f7586e597c254079-v1", "taskId": "task_f7586e597c254079", "version": "1", "name": "clinical-trial-survival-censoring", "domain": "Machine learning", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f83f3913b9ad4d64-v1", "taskId": "task_f83f3913b9ad4d64", "version": "1", "name": "openfda-metafusion-quality-text-vocabdrift", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fae2e8c033894147-v1", "taskId": "task_fae2e8c033894147", "version": "1", "name": "coil2000-imbalance-metric", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fec6fb892b424ee7-v1", "taskId": "task_fec6fb892b424ee7", "version": "1", "name": "cellphonesandaccessories-imbalance-quality-text-vocabdrift", "domain": "Machine learning", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_064d00bafb3f4cd0-v1", "taskId": "task_064d00bafb3f4cd0", "version": "1", "name": "mechanical-engineering/pipe-rack-truss-sizing", "domain": "Mechanical engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/pipe-rack-truss-sizing-qa.md);", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_3a0208a1efc74069-v1", "taskId": "task_3a0208a1efc74069", "version": "1", "name": "mechanical-engineering/driveline-torsional-detune", "domain": "Mechanical engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-F): eigensolver re-derived 3 ways ", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_8807493621084574-v1", "taskId": "task_8807493621084574", "version": "1", "name": "mechanical-engineering/driveline-torsional-detune", "domain": "Mechanical engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-F): eigensolver re-derived 3 ways ", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_ac699379f60e4a07-v1", "taskId": "task_ac699379f60e4a07", "version": "1", "name": "cam-modal-toolpath-envelope", "domain": "Mechanical engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@xhigh", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c870fc659280435f-v1", "taskId": "task_c870fc659280435f", "version": "1", "name": "mechanical-engineering/pipe-rack-truss-sizing", "domain": "Mechanical engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (budget)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_ca5c53a3e9e64871-v1", "taskId": "task_ca5c53a3e9e64871", "version": "1", "name": "mechanical-engineering/driveline-torsional-detune", "domain": "Mechanical engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (budget)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_e08681b82c6c40ef-v1", "taskId": "task_e08681b82c6c40ef", "version": "1", "name": "sheetmetal-flat-pattern-v1", "domain": "Mechanical engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Author READY; independent QA-D pending", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_f48ea8010149425d-v1", "taskId": "task_f48ea8010149425d", "version": "1", "name": "dxf-nested-assembly-schedule", "domain": "Mechanical engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@xhigh", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 4, "p": 0, "lo": 0, "hi": 0.4898908364545973, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.4898908364545973, "labelled": 3, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.75, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fa130f9a671047cb-v1", "taskId": "task_fa130f9a671047cb", "version": "1", "name": "sheetmetal-flat-pattern-v2", "domain": "Mechanical engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: author READY w/ CALIBRATION WARNING: Opus-class b", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_21d949912d4e426c-v1", "taskId": "task_21d949912d4e426c", "version": "1", "name": "podcast-conform-edl-v2", "domain": "Media", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Author READY (v1 full local evidence applies: same gra", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_480dcb0133ab4166-v1", "taskId": "task_480dcb0133ab4166", "version": "1", "name": "syncrope/ab-glyph-render", "domain": "Media", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": true, "qaVerdict": "taxonomy: Evaluating; QA: QA-I READY", "knownDefects": [ "Recorded concern about the empty-solution control." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Empty solution passes" ] }, { "id": "task_5497956b4eff49a3-v1", "taskId": "task_5497956b4eff49a3", "version": "1", "name": "podcast-conform-edl-v3", "domain": "Media", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-D2 independent READY: oracle root/uid1000 1; n", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_a78ea9a3be2c41d6-v1", "taskId": "task_a78ea9a3be2c41d6", "version": "1", "name": "syncrope/tone-soundfont-engine", "domain": "Media", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (write-early + seal)", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_b031f6a77feb43b6-v1", "taskId": "task_b031f6a77feb43b6", "version": "1", "name": "syncrope/tone-soundfont-engine", "domain": "Media", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/tone-soundfont-engine-qa.md); ", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_d9d893f549e64959-v1", "taskId": "task_d9d893f549e64959", "version": "1", "name": "syncrope/ttfparser-tt-hinting", "domain": "Media", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-I READY: oracle root/uid1000 1 (420/420); nop 0; wr", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ec2abd5ff8634a4e-v1", "taskId": "task_ec2abd5ff8634a4e", "version": "1", "name": "syncrope/tone-soundfont-engine", "domain": "Media", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (lead, 10-13 02:25): diff-only test.sh", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_fe29ee614394433c-v1", "taskId": "task_fe29ee614394433c", "version": "1", "name": "podcast-conform-edl-v1", "domain": "Media", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Author READY (report saved by lead: audits/podcast-con", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_227dd058c9a14571-v1", "taskId": "task_227dd058c9a14571", "version": "1", "name": "cargo-route-load-control", "domain": "Operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_451a7c2c9c834af6-v1", "taskId": "task_451a7c2c9c834af6", "version": "1", "name": "supplier-cumulative-release-reconciliation", "domain": "Operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_465a571df99f4539-v1", "taskId": "task_465a571df99f4539", "version": "1", "name": "family-benefit-accumulator-reprocessing", "domain": "Operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@xhigh", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 2, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Task fault on at least half of labelled current-model failures" ] }, { "id": "task_4e3b020aafeb4d8e-v1", "taskId": "task_4e3b020aafeb4d8e", "version": "1", "name": "occurrence-reinsurance-recovery-bordereau", "domain": "Operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@xhigh", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6eabd517a9244f93-v1", "taskId": "task_6eabd517a9244f93", "version": "1", "name": "home-health-routing-skills-breaks-v3", "domain": "Operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (write-early; v2 never in trials)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_7407cd8a16a746e3-v1", "taskId": "task_7407cd8a16a746e3", "version": "1", "name": "home-health-routing-skills-breaks-v2", "domain": "Operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-F): both optima proved OPTIMAL ind", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_958606903dec4656-v1", "taskId": "task_958606903dec4656", "version": "1", "name": "landed-cost-retrospective-allocation", "domain": "Operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_984a8630c20f46b4-v1", "taskId": "task_984a8630c20f46b4", "version": "1", "name": "claimsplit-v3", "domain": "Operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: Local QA passed (lead, 10-13 02:25): diff-only test.sh", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_aca68139392347ca-v1", "taskId": "task_aca68139392347ca", "version": "1", "name": "claimsplit-v2", "domain": "Operations", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/claimsplit-qa.md); band 20-50%", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_f732fa3e4e264ff6-v1", "taskId": "task_f732fa3e4e264ff6", "version": "1", "name": "container-free-time-invoice-audit", "domain": "Operations", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_01377f892b1a4200-v1", "taskId": "task_01377f892b1a4200", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_21d6d6ecb3d54708-v1", "taskId": "task_21d6d6ecb3d54708", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_35fae6f5aecd4bf1-v1", "taskId": "task_35fae6f5aecd4bf1", "version": "1", "name": "lru-cache", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_405f3d7e01c54e48-v1", "taskId": "task_405f3d7e01c54e48", "version": "1", "name": "google-certificate-authority-api-clone", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_43c564a2a4ff41a6-v1", "taskId": "task_43c564a2a4ff41a6", "version": "1", "name": "orb-projection-engine", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_523845da91604387-v1", "taskId": "task_523845da91604387", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_57c4802194db475b-v1", "taskId": "task_57c4802194db475b", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5905242d46c54cfd-v1", "taskId": "task_5905242d46c54cfd", "version": "1", "name": "hello-world-ui-smoke-1781489413", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_765f2b5330904ef0-v1", "taskId": "task_765f2b5330904ef0", "version": "1", "name": "buggy-adder", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8ab5a5c46dc14af0-v1", "taskId": "task_8ab5a5c46dc14af0", "version": "1", "name": "e2e-exp-task", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_903380a29a404c50-v1", "taskId": "task_903380a29a404c50", "version": "1", "name": "nth-fibonacci", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b1e806392e7a48c0-v1", "taskId": "task_b1e806392e7a48c0", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c98c788505034c5c-v1", "taskId": "task_c98c788505034c5c", "version": "1", "name": "buggy-adder", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_cb83b52d5a164eb4-v1", "taskId": "task_cb83b52d5a164eb4", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d296728a24ce4eb8-v1", "taskId": "task_d296728a24ce4eb8", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ef5f2a22dd46414d-v1", "taskId": "task_ef5f2a22dd46414d", "version": "1", "name": "hello-world", "domain": "Other / uncategorised", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gemini-flash-latest@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-sonnet-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_028a271156564aa4-v1", "taskId": "task_028a271156564aa4", "version": "1", "name": "sqlite-file-engine-rs-v2", "domain": "SWE-Marathon", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Rejected: too-easy (screen 3/3 genuine) -> v3 TODO; QA: QA-D3 independ", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_1fe1d1db989440e4-v1", "taskId": "task_1fe1d1db989440e4", "version": "1", "name": "archives-mig-rs-v2", "domain": "SWE-Marathon", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Rejected; QA: SEE REJECT (01:41): lane privscan refuses ANY agent-writ", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_2e1fdb07de4241b9-v1", "taskId": "task_2e1fdb07de4241b9", "version": "1", "name": "archives-mig-rs-v4", "domain": "SWE-Marathon", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Rejected: too-easy (screen 3/3, SEE-confirmed) -> v5 TODO; QA: Local Q", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_330f0c0c4a6d4ed8-v1", "taskId": "task_330f0c0c4a6d4ed8", "version": "1", "name": "archives-mig-rs", "domain": "SWE-Marathon", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3bb8d4f019464904-v1", "taskId": "task_3bb8d4f019464904", "version": "1", "name": "jsonnet-interpreter", "domain": "SWE-Marathon", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_58f02322e3be45a3-v1", "taskId": "task_58f02322e3be45a3", "version": "1", "name": "archives-mig-rs-v3", "domain": "SWE-Marathon", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (v2 report) + lane fix; not locally re", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_a37a1ff7ca924c0f-v1", "taskId": "task_a37a1ff7ca924c0f", "version": "1", "name": "archives-mig-rs", "domain": "SWE-Marathon", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ad9bf846cf814e6f-v1", "taskId": "task_ad9bf846cf814e6f", "version": "1", "name": "implement-yaml2json", "domain": "SWE-Marathon", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_095ae651036742f2-v1", "taskId": "task_095ae651036742f2", "version": "1", "name": "syncrope/ecod-leave-family-out-homology-group-assignment-alpha-and-sandwich-domains", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0b009b8ef9494b69-v1", "taskId": "task_0b009b8ef9494b69", "version": "1", "name": "syncrope/pdb-unusual-growth-environment-crystal", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_155a17920cce4aa7-v1", "taskId": "task_155a17920cce4aa7", "version": "1", "name": "umi-duplicate-collapse", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_25897415ce834cba-v1", "taskId": "task_25897415ce834cba", "version": "1", "name": "syncrope/real-albert-canal-heldout-tow-track-unsynchronised-toa", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_30409182f3484b4d-v1", "taskId": "task_30409182f3484b4d", "version": "1", "name": "syncrope/gnss-coseismic-offset", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_548fb9445d2b47e7-v1", "taskId": "task_548fb9445d2b47e7", "version": "1", "name": "real-zenodo-ch2nh2-heldout-inversion-tunnelling-coriolis-lines", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5535b119edba4e9a-v1", "taskId": "task_5535b119edba4e9a", "version": "1", "name": "cf-hybrid-pressure-columns", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5acdd105892640af-v1", "taskId": "task_5acdd105892640af", "version": "1", "name": "terminal-bench/nmr-spin-inversion", "domain": "Science / STEM", "source": "console", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-P2 READY", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 5, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@", "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_5ba674df3143403e-v1", "taskId": "task_5ba674df3143403e", "version": "1", "name": "syncrope/pdb-legionella-coxiella-rickettsia-intracellular-pathogen", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5f59681db2ce40ab-v1", "taskId": "task_5f59681db2ce40ab", "version": "1", "name": "hrms-formula-fine-structure", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_632ad43219ad4be2-v1", "taskId": "task_632ad43219ad4be2", "version": "1", "name": "lsw-coarsening-truncation", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_654ebf40c2d44457-v1", "taskId": "task_654ebf40c2d44457", "version": "1", "name": "syncrope/real-tandf-33so2-nu1nu3-coriolis-heldout-highka-lines", "domain": "Science / STEM", "source": "console", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: Local QA passed + SEE gate 1 PASS (00:55 IST)", "knownDefects": [ "Recorded concern about the empty-solution control." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 7, "p": 0.2857142857142857, "lo": 0.08221892400405678, "hi": 0.6410655481673808, "tasks": 1, "macro": 0.2857142857142857, "macroLo": 0.08221892400405678, "macroHi": 0.6410655481673808, "labelled": 7, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": 0.7142857142857143, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@", "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_6d75ea96f6a3417e-v1", "taskId": "task_6d75ea96f6a3417e", "version": "1", "name": "syncrope/pdb-clostridioides-difficile-peptostreptococcaceae-crystal", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8416fb7bdc134735-v1", "taskId": "task_8416fb7bdc134735", "version": "1", "name": "syncrope/real-butpdb-heldout-av-dissociation-pwave-positions", "domain": "Science / STEM", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: Local QA passed (QA-E; thinner false-failure evid", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_845db66395934c55-v1", "taskId": "task_845db66395934c55", "version": "1", "name": "syncrope/nomad-comgo-nh3-heldout-cooling-branch-exsolution-activation", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_85366d7577b74b1d-v1", "taskId": "task_85366d7577b74b1d", "version": "1", "name": "syncrope/real-butpdb-heldout-av-dissociation-pwave-positions", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8abfda2c4191486f-v1", "taskId": "task_8abfda2c4191486f", "version": "1", "name": "track-state-diffusion", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9d72cfdc05184561-v1", "taskId": "task_9d72cfdc05184561", "version": "1", "name": "gamma-nuclide-assay", "domain": "Science / STEM", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 6, "n": 6, "p": 1, "lo": 0.6096657120978346, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.6096657120978346, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 6, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 0.8333333333333334, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.3903342879021654, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_a5501ab48b9148f8-v1", "taskId": "task_a5501ab48b9148f8", "version": "1", "name": "syncrope/real-triple-pendulum-heldout-two-hidden-arms-from-base-encoder", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_a8c4bfa1a92d41c7-v1", "taskId": "task_a8c4bfa1a92d41c7", "version": "1", "name": "syncrope/pdb-gatase-gapdh-aph-kelch-sh2-family", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ac015bf24406411d-v1", "taskId": "task_ac015bf24406411d", "version": "1", "name": "syncrope/real-memristor-jerk-circuit-intersample-dealiasing", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b5574e0b37094fe7-v1", "taskId": "task_b5574e0b37094fe7", "version": "1", "name": "syncrope/real-mast-tess-year4-s47-s55-north-heldout-minor-planet-sap-excursions", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c0ac702b56bd4a07-v1", "taskId": "task_c0ac702b56bd4a07", "version": "1", "name": "chromatography-inverse-calibration", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c4808f49e3c14c4d-v1", "taskId": "task_c4808f49e3c14c4d", "version": "1", "name": "stepped-wedge-secular-trend", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c58ca78cde44458a-v1", "taskId": "task_c58ca78cde44458a", "version": "1", "name": "real-pdb-bfactor-flexibility", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d2af0dfdbdac482a-v1", "taskId": "task_d2af0dfdbdac482a", "version": "1", "name": "bandpass-radiometer-inversion", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d7324342a8ba4eeb-v1", "taskId": "task_d7324342a8ba4eeb", "version": "1", "name": "syncrope/real-sbn-deep-impact-tempel1-encounter-doppler-heldout-pass", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_da5ddd1fa2484d07-v1", "taskId": "task_da5ddd1fa2484d07", "version": "1", "name": "aquifer-test-diagnosis", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f12bc011ea044d0c-v1", "taskId": "task_f12bc011ea044d0c", "version": "1", "name": "syncrope/real-zenodo-kerr-optomech-heldout-bistable-branch-backaction", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f404773f4ad447ba-v1", "taskId": "task_f404773f4ad447ba", "version": "1", "name": "circuit-transient-companion-models", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f461c901daa54249-v1", "taskId": "task_f461c901daa54249", "version": "1", "name": "syncrope/real-hprc-heldout-2q37-bok-locus-interval-haplotype-span-partition", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f63c17dcfb6e4797-v1", "taskId": "task_f63c17dcfb6e4797", "version": "1", "name": "syncrope/real-tandf-33so2-nu1nu3-coriolis-heldout-highka-lines", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fa239900d31747ff-v1", "taskId": "task_fa239900d31747ff", "version": "1", "name": "aqueous-speciation-activity", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@medium", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fd7cae7e4a044646-v1", "taskId": "task_fd7cae7e4a044646", "version": "1", "name": "syncrope/real-ssl-buscomp-heldout-auto-release-gain-trajectories", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fddcdab7d981491a-v1", "taskId": "task_fddcdab7d981491a", "version": "1", "name": "syncrope/real-eries-aeroburst-heldout-downburst-section-model-response", "domain": "Science / STEM", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_258b922c739d44a9-v1", "taskId": "task_258b922c739d44a9", "version": "1", "name": "syncrope/real-cxidb-pt-sapphire-450c-bcdi-heldout-002-fringe-streak-blocks", "domain": "Scientific computing", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3e17a0cc7a1240e4-v1", "taskId": "task_3e17a0cc7a1240e4", "version": "1", "name": "syncrope/real-cxidb-pt-sapphire-450c-bcdi-heldout-002-fringe-streak-blocks", "domain": "Scientific computing", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: Local QA passed (QA-E)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_4d182f199a314f63-v1", "taskId": "task_4d182f199a314f63", "version": "1", "name": "grid-inertia-audit-v2", "domain": "Scientific computing", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed + SEE gate 1 PASS (00:27 IST; graded f", "knownDefects": [ "Recorded concern about the empty-solution control." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_561745095aed43d9-v1", "taskId": "task_561745095aed43d9", "version": "1", "name": "syncrope/treewidth-pace2017-084-width-gap", "domain": "Scientific computing", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5bf47d1535284a34-v1", "taskId": "task_5bf47d1535284a34", "version": "1", "name": "grid-inertia-audit-v3", "domain": "Scientific computing", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Rejected; QA: QA-H sweep: budget fairness at the 8 h cap (lead chose v", "knownDefects": [ "Recorded concern about the empty-solution control." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_69da196c60b14e69-v1", "taskId": "task_69da196c60b14e69", "version": "1", "name": "grid-inertia-audit", "domain": "Scientific computing", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7d85da2e70394057-v1", "taskId": "task_7d85da2e70394057", "version": "1", "name": "grid-inertia-audit-v1", "domain": "Scientific computing", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/grid-inertia-audit-qa.md)", "knownDefects": [ "Recorded concern about the empty-solution control." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_9f25f55005fd4625-v1", "taskId": "task_9f25f55005fd4625", "version": "1", "name": "contour-cutting-plasticity", "domain": "Scientific computing", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b8979d89b1fa47cb-v1", "taskId": "task_b8979d89b1fa47cb", "version": "1", "name": "syncrope/real-uhdrppic-heldout-low-bias-collection-efficiency", "domain": "Scientific computing", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_feb592c3c0a24f14-v1", "taskId": "task_feb592c3c0a24f14", "version": "1", "name": "syncrope/treewidth-pace2017-084-width-gap", "domain": "Scientific computing", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0ef28063018e4947-v1", "taskId": "task_0ef28063018e4947", "version": "1", "name": "syncrope/pypdf-revision-forensics", "domain": "Security / cyber", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed + SEE gate 1 PASS with notes (both tex", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_144c7844470e4a8f-v1", "taskId": "task_144c7844470e4a8f", "version": "1", "name": "spreadsheet-iterative-circular-convergence", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_161f4569470f4aa1-v1", "taskId": "task_161f4569470f4aa1", "version": "1", "name": "webauthn-assertion-transaction-gate", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@xhigh", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_26f14effae254b98-v1", "taskId": "task_26f14effae254b98", "version": "1", "name": "hpke-authenticated-message-contexts", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_28d7c8031fdb4324-v1", "taskId": "task_28d7c8031fdb4324", "version": "1", "name": "zopfli-deflate-compressor", "domain": "Security / cyber", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 6, "p": 0.6666666666666666, "lo": 0.299993315138392, "hi": 0.9032285888942195, "tasks": 1, "macro": 0.6666666666666666, "macroLo": 0.299993315138392, "macroHi": 0.9032285888942195, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 1, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 6, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.6666666666666666, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.33333333333333337, "difficultyLo": 0.09677141110578047, "difficultyHi": 0.700006684861608, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_2db3ee04627e4cda-v1", "taskId": "task_2db3ee04627e4cda", "version": "1", "name": "implement-ed", "domain": "Security / cyber", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_2e167e8a905e4e8d-v1", "taskId": "task_2e167e8a905e4e8d", "version": "1", "name": "reverse-engineer-listed-substance-timeline-registry", "domain": "Security / cyber", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.19999999999999996, "difficultyLo": 0.036224108632430196, "difficultyHi": 0.6244653702374747, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_3ee471276e3c4dee-v1", "taskId": "task_3ee471276e3c4dee", "version": "1", "name": "text-sender-heap-overlap", "domain": "Security / cyber", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_4ecee91716c74f10-v1", "taskId": "task_4ecee91716c74f10", "version": "1", "name": "reverse-engineer-bitmap-index", "domain": "Security / cyber", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_7a15a7e74cee401c-v1", "taskId": "task_7a15a7e74cee401c", "version": "1", "name": "reverse-engineer-layout-policy-selector", "domain": "Security / cyber", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_88f9aca85c1c474d-v1", "taskId": "task_88f9aca85c1c474d", "version": "1", "name": "tuf-offline-update-chain", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@xhigh", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8e0c1e8950c84b8c-v1", "taskId": "task_8e0c1e8950c84b8c", "version": "1", "name": "seccompward-go-bpf-filter-reachability-v2", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Superseded; QA: QA-K READY", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9df54a4187274623-v1", "taskId": "task_9df54a4187274623", "version": "1", "name": "sigparse-der-memory-safety-audit-v2", "domain": "Security / cyber", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (disclosure + budget)", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_af7650ce24df4abb-v1", "taskId": "task_af7650ce24df4abb", "version": "1", "name": "archive-extract-repair", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 3, "p": 0.6666666666666666, "lo": 0.20765960080204765, "hi": 0.9385080552796037, "tasks": 1, "macro": 0.6666666666666666, "macroLo": 0.20765960080204765, "macroHi": 0.9385080552796037, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 1, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b7f114ed3305443c-v1", "taskId": "task_b7f114ed3305443c", "version": "1", "name": "syncrope/pypdf-revision-forensics", "domain": "Security / cyber", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-H sweep (budget)", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_b94e8deddab54e04-v1", "taskId": "task_b94e8deddab54e04", "version": "1", "name": "deconstruct-header-compressor", "domain": "Security / cyber", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_bb2c2a0827ac4c5d-v1", "taskId": "task_bb2c2a0827ac4c5d", "version": "1", "name": "reverse-engineer-oscillator-discipline-governor", "domain": "Security / cyber", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_c1fec78f2b0b49e0-v1", "taskId": "task_c1fec78f2b0b49e0", "version": "1", "name": "http-smuggle-repair", "domain": "Security / cyber", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.5, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.7224672001371109, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_c79d06c5f0ac454b-v1", "taskId": "task_c79d06c5f0ac454b", "version": "1", "name": "sigparse-der-memory-safety-audit-v1", "domain": "Security / cyber", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-F): 11 defects confirmed in source", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_d353cd4a8ff747d8-v1", "taskId": "task_d353cd4a8ff747d8", "version": "1", "name": "archive-extract-repair", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_db1402fc696e4530-v1", "taskId": "task_db1402fc696e4530", "version": "1", "name": "oci-layer-admission-audit", "domain": "Security / cyber", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@xhigh", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@xhigh", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 1, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4, "difficultyLo": 0.11762077423264805, "difficultyHi": 0.7692757187239871, "strongestRow": "tencent/hy4-preview@@think", "defectReasons": [] }, { "id": "task_dea4ffcded2a474f-v1", "taskId": "task_dea4ffcded2a474f", "version": "1", "name": "seccompward-go-bpf-filter-reachability-v3", "domain": "Security / cyber", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-K READY (stub builds + solve.sh exit 0 as agent; or", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.6666666666666666, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_eb165d4186e84dec-v1", "taskId": "task_eb165d4186e84dec", "version": "1", "name": "deconstruct-sat-solver", "domain": "Security / cyber", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f001a4acb02b458a-v1", "taskId": "task_f001a4acb02b458a", "version": "1", "name": "reverse-engineer-divergent-weave-amendment-dossier", "domain": "Security / cyber", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4, "difficultyLo": 0.11762077423264805, "difficultyHi": 0.7692757187239871, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_f71ddbde38c24515-v1", "taskId": "task_f71ddbde38c24515", "version": "1", "name": "reverse-vm-keygen-v2", "domain": "Security / cyber", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-K READY", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_ff52b9d8e07e4592-v1", "taskId": "task_ff52b9d8e07e4592", "version": "1", "name": "syncrope/pypdf-revision-forensics", "domain": "Security / cyber", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/pypdf-revision-forensics-qa.md", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_07e305189bf74f32-v1", "taskId": "task_07e305189bf74f32", "version": "1", "name": "image-loader", "domain": "Software engineering", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 6, "p": 0.5, "lo": 0.18761630648265054, "hi": 0.8123836935173494, "tasks": 1, "macro": 0.5, "macroLo": 0.18761630648265054, "macroHi": 0.8123836935173494, "labelled": 6, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 1, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 6, "measuredTasks": 1, "coverage": 1, "gfr": 0.3333333333333333, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.5, "difficultyLo": 0.1876163064826506, "difficultyHi": 0.8123836935173494, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_131d1f93966e4354-v1", "taskId": "task_131d1f93966e4354", "version": "1", "name": "qdrant-fix-flusher-data-race-in-gridstor", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_164a99c5cebc4aff-v1", "taskId": "task_164a99c5cebc4aff", "version": "1", "name": "qdrant-fix-flusher-data-race-in-gridstor", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 2, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.6666666666666666, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_17499976e3a34d09-v1", "taskId": "task_17499976e3a34d09", "version": "1", "name": "etcd-fix-race-berween-read-index-and-lea", "domain": "Software engineering", "source": "console", "tier": "high-fail (unreviewed)", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_17499976e3a34d09-v2", "taskId": "task_17499976e3a34d09", "version": "2", "name": "etcd-fix-race-berween-read-index-and-lea", "domain": "Software engineering", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 8, "p": 0, "lo": 0, "hi": 0.32440756488388023, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.32440756488388023, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 8, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.6755924351161198, "difficultyHi": 1, "strongestRow": "tencent/hy4-preview@@think", "defectReasons": [] }, { "id": "task_1a25802d0b6a403d-v1", "taskId": "task_1a25802d0b6a403d", "version": "1", "name": "syncrope/keycloak-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2547b6a32e6145ce-v1", "taskId": "task_2547b6a32e6145ce", "version": "1", "name": "qdrant-fix-flusher-data-race-in-gridstor-v2", "domain": "Software engineering", "source": "console", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: QA-J READY: oracle root + agent uid1001 1 (22/22); nop", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 7, "p": 0.5714285714285714, "lo": 0.2504583645276572, "hi": 0.8417801447485302, "tasks": 1, "macro": 0.5714285714285714, "macroLo": 0.2504583645276572, "macroHi": 0.8417801447485302, "labelled": 5, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 2, "BAD_FAILURE": 1, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.7142857142857143, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4285714285714286, "difficultyLo": 0.15821985525146975, "difficultyHi": 0.7495416354723428, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_3b5837a41a034f35-v1", "taskId": "task_3b5837a41a034f35", "version": "1", "name": "rocksdb-fix-compaction-abort-reschedulin-v2", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-A wave 2): oracle 1x3, nop 0, cras", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_40942ad7907f4a0c-v1", "taskId": "task_40942ad7907f4a0c", "version": "1", "name": "evanw-esbuild-4417", "domain": "Software engineering", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_450af4d51db04832-v1", "taskId": "task_450af4d51db04832", "version": "1", "name": "timescaledb-fix-set-returning-functions", "domain": "Software engineering", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.75, "gfrCoverage": 0.8, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.7, "difficultyLo": 0.39677814746114537, "difficultyHi": 0.892208732593699, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_4a09fc48a1104c6f-v1", "taskId": "task_4a09fc48a1104c6f", "version": "1", "name": "etcd-fix-race-berween-read-index-and-lea-v2", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-A v2)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_4a7cd61ab8124610-v1", "taskId": "task_4a7cd61ab8124610", "version": "1", "name": "syncrope/airflow-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_540eef91fb3045b3-v1", "taskId": "task_540eef91fb3045b3", "version": "1", "name": "rocksdb-fix-compaction-abort-reschedulin-v4", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Rejected: too-easy (screen 3/3, SEE-confirmed) -> v5 TODO; QA: Local Q", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_576984f677554810-v1", "taskId": "task_576984f677554810", "version": "1", "name": "syncrope/asana-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_591888c9e8304fbf-v1", "taskId": "task_591888c9e8304fbf", "version": "1", "name": "audio-metadata-rust-port", "domain": "Software engineering", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_5ab0af843ae24f9e-v1", "taskId": "task_5ab0af843ae24f9e", "version": "1", "name": "terminal-bench/ssa-miscompile-hunt", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Local QA passed; QA: QA-P1 READY: oracle root/uid1000 1; nop 0; empty/", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_5fde620f793f4898-v1", "taskId": "task_5fde620f793f4898", "version": "1", "name": "syncrope/rocketchat-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_654664472be14f24-v1", "taskId": "task_654664472be14f24", "version": "1", "name": "timescaledb-fix-set-returning-functions-v2", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (audits/timescaledb-fix-set-returning-", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_69bbae703b224f62-v1", "taskId": "task_69bbae703b224f62", "version": "1", "name": "etcd-fix-race-berween-read-index-and-lea-v3", "domain": "Software engineering", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: Local QA passed (QA-A v3; 4 redirect probes 0)", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 7, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.3543304350666875, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.3543304350666875, "labelled": 7, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.6456695649333125, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_6a7b3eb045bb434f-v1", "taskId": "task_6a7b3eb045bb434f", "version": "1", "name": "syncrope/datadog-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6ccfe02bc1814822-v1", "taskId": "task_6ccfe02bc1814822", "version": "1", "name": "klauspost-compress-1115", "domain": "Software engineering", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7140c07984dd4e8c-v1", "taskId": "task_7140c07984dd4e8c", "version": "1", "name": "timescaledb-fix-set-returning-functions-v4", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "taxonomy: Evaluating; QA: Local QA passed (lead, 10-13 02:25): diff-only test.sh", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_758d7c719570445b-v1", "taskId": "task_758d7c719570445b", "version": "1", "name": "rocksdb-fix-compaction-abort-reschedulin", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 3, "p": 0.6666666666666666, "lo": 0.20765960080204765, "hi": 0.9385080552796037, "tasks": 1, "macro": 0.6666666666666666, "macroLo": 0.20765960080204765, "macroHi": 0.9385080552796037, "labelled": 1, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.3333333333333333, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_758d7c719570445b-v2", "taskId": "task_758d7c719570445b", "version": "2", "name": "rocksdb-fix-compaction-abort-reschedulin", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "deepswe review: OUT of delivery (coordinator, 2026-10-22 ~00:55Z)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "tencent/hy4-preview@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@high", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_810901df76e9486f-v1", "taskId": "task_810901df76e9486f", "version": "1", "name": "tar-rust-rewrite", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": null, "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@", "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "task_83b0ddb6c43443ae-v1", "taskId": "task_83b0ddb6c43443ae", "version": "1", "name": "rocksdb-fix-compaction-abort-reschedulin-v3", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed (QA-A v3)", "knownDefects": [ "Recorded concern about task budget or timeout behavior." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_84331f6b34d74400-v1", "taskId": "task_84331f6b34d74400", "version": "1", "name": "syncrope/mailchimp-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_92ef1ca56a8e4e1f-v1", "taskId": "task_92ef1ca56a8e4e1f", "version": "1", "name": "uax14-linebreak", "domain": "Software engineering", "source": "console", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 7, "n": 7, "p": 1, "lo": 0.6456695649333126, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.6456695649333126, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0.7142857142857143, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.3543304350666874, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_97443cf7210843d2-v1", "taskId": "task_97443cf7210843d2", "version": "1", "name": "etcd-fix-race-berween-read-index-and-lea", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b929faf5299c4047-v1", "taskId": "task_b929faf5299c4047", "version": "1", "name": "float-to-string", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b94b66248bc44678-v1", "taskId": "task_b94b66248bc44678", "version": "1", "name": "syncrope/cloudflare-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c42ad9d8b535441a-v1", "taskId": "task_c42ad9d8b535441a", "version": "1", "name": "minpack-solvers", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d12a5d74a23a4125-v1", "taskId": "task_d12a5d74a23a4125", "version": "1", "name": "syncrope/terraform-cloud-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d6a60c4029734f27-v1", "taskId": "task_d6a60c4029734f27", "version": "1", "name": "bignum-arithmetic", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 4, "p": 0, "lo": 0, "hi": 0.4898908364545973, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.4898908364545973, "labelled": 4, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0.75, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d9fc716ee5cd4825-v1", "taskId": "task_d9fc716ee5cd4825", "version": "1", "name": "xmllint-rust-rewrite", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_db45dba828f34ded-v1", "taskId": "task_db45dba828f34ded", "version": "1", "name": "expr-printf-rust-rewrite", "domain": "Software engineering", "source": "console", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_e03f150b221445eb-v1", "taskId": "task_e03f150b221445eb", "version": "1", "name": "timescaledb-fix-set-returning-functions-v3", "domain": "Software engineering", "source": "console", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "taxonomy: Superseded; QA: Local QA passed + SEE gate 1 PASS on v2 text; v3 chang", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_e087cad33d2d4461-v1", "taskId": "task_e087cad33d2d4461", "version": "1", "name": "rocksdb-fix-compaction-abort-reschedulin", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e99bb7b59dbf427e-v1", "taskId": "task_e99bb7b59dbf427e", "version": "1", "name": "syncrope/temporal-clone", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_eeea4d5ab5974e81-v1", "taskId": "task_eeea4d5ab5974e81", "version": "1", "name": "rust-lang-semver-305", "domain": "Software engineering", "source": "console", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fd960530713b4792-v1", "taskId": "task_fd960530713b4792", "version": "1", "name": "wireformats-rs", "domain": "Software engineering", "source": "console", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-augmented-gmm-covariance-v1", "taskId": "sfdata2/augmented-gmm-covariance", "version": "1", "name": "datasciresearch/augmented-gmm-covariance", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "superseded by v2 (h34 clean cohort)", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 1, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/609" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-augmented-gmm-covariance-v2-v2", "taskId": "sfdata2/augmented-gmm-covariance-v2", "version": "2", "name": "datasciresearch/augmented-gmm-covariance", "domain": "Data processing / data science", "source": "delivery", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "ACCEPT-WITH-DOCUMENTED-GAPS (sfdata2-pr609v2-final-signoff.md; D26-W waiver sign", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/609" }, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@medium", "defectReasons": [] }, { "id": "sfdata2-augmented-gmm-covariance-v3-v3", "taskId": "sfdata2/augmented-gmm-covariance-v3", "version": "3", "name": "datasciresearch/augmented-gmm-covariance", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "vendor redesign; solved 2/2 (sfdata2-pr609-v3-gates.md)", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/609" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-compact-noise-reflected-divergence-v1", "taskId": "sfdata2/compact-noise-reflected-divergence", "version": "1", "name": "compact-noise-reflected-divergence", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house design (no vendor PR); see docs/reports/datasci-research-v5", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-dcor-taustar-ties-v1", "taskId": "sfdata2/dcor-taustar-ties", "version": "1", "name": "datasciresearch/dcor-taustar-ties", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house design (no vendor PR); see docs/reports/datasci-research-v5", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-esda-mh-diffusion-distance-v1", "taskId": "sfdata2/esda-mh-diffusion-distance", "version": "1", "name": "datasciresearch/esda-mh-diffusion-distance", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house design (no vendor PR); see docs/reports/datasci-research-v5", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-generative-process-two-sample-v1", "taskId": "sfdata2/generative-process-two-sample", "version": "1", "name": "datasciresearch/generative-process-two-sample", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "vendor version; r3 blind read UNCLEAR (sfdata2-605-r3-blind-tie-read.md)", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/605" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-generative-two-sample-r3-v1", "taskId": "sfdata2/generative-two-sample-r3", "version": "1", "name": "datasciresearch/generative-process-two-sample", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house r3 revision; blind read UNCLEAR (sfdata2-605-r3-blind-tie-read.md)", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/605" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-gmm-misspecified-covariance-v1", "taskId": "sfdata2/gmm-misspecified-covariance", "version": "1", "name": "datasciresearch/gmm-misspecified-covariance", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house design (no vendor PR); see docs/reports/datasci-research-v5", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-lifelines-dr-cosarc-v1", "taskId": "sfdata2/lifelines-dr-cosarc", "version": "1", "name": "syncrope/lifelines-dr-cosarc", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "ACCEPT-WITH-DOCUMENTED-GAPS (sfdata2-pr603-final-signoff.md: 61 PASS / 17 GAP)", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/603" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-nonconformist-kernel-quantile-path-v1", "taskId": "sfdata2/nonconformist-kernel-quantile-path", "version": "1", "name": "syncrope/nonconformist-kernel-quantile-path", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "superseded by v2", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/604" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-nonconformist-kernel-quantile-path-v2-v2", "taskId": "sfdata2/nonconformist-kernel-quantile-path-v2", "version": "2", "name": "syncrope/nonconformist-kernel-quantile-path", "domain": "Data processing / data science", "source": "delivery", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "REVISE (#604 v2 ledger: 24 FAIL incl. S04/S16/S18/S22)", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/604" }, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@medium", "defectReasons": [] }, { "id": "sfdata2-pyclustering-selective-joint-v1", "taskId": "sfdata2/pyclustering-selective-joint", "version": "1", "name": "syncrope/pyclustering-selective-joint", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "superseded by v2", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/606" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-pyclustering-selective-joint-v2-v2", "taskId": "sfdata2/pyclustering-selective-joint-v2", "version": "2", "name": "syncrope/pyclustering-selective-joint", "domain": "Data processing / data science", "source": "delivery", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "ACCEPT-WITH-DOCUMENTED-GAPS (sfdata2-pr606v2-final-signoff.md; FULL S04 waiver)", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/606" }, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@medium", "defectReasons": [] }, { "id": "sfdata2-pyfixest-picchetti-ri-v1", "taskId": "sfdata2/pyfixest-picchetti-ri", "version": "1", "name": "datasciresearch/pyfixest-picchetti-ri", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house design (no vendor PR); see docs/reports/datasci-research-v5", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-rlm-sandwich-covariance-v1", "taskId": "sfdata2/rlm-sandwich-covariance", "version": "1", "name": "datasciresearch/rlm-sandwich-covariance", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house design (no vendor PR); see docs/reports/datasci-research-v5", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-ruptures-selective-mean-v1", "taskId": "sfdata2/ruptures-selective-mean", "version": "1", "name": "syncrope/ruptures-selective-mean", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "REJECT (#607)", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/607" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-ruptures_grace_v6-v1", "taskId": "sfdata2/ruptures_grace_v6", "version": "1", "name": "ruptures_grace_v6", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "in-house design (no vendor PR); see docs/reports/datasci-research-v5", "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-skglm-perspective-certificates-v1", "taskId": "sfdata2/skglm-perspective-certificates", "version": "1", "name": "syncrope/skglm-perspective-certificates", "domain": "Data processing / data science", "source": "delivery", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "REJECT (#608)", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/608" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfdata2-weak-moment-location-scale-v1", "taskId": "sfdata2/weak-moment-location-scale", "version": "1", "name": "datasciresearch/weak-moment-location-scale", "domain": "Data processing / data science", "source": "delivery", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "ACCEPT-WITH-DOCUMENTED-GAPS (sfdata2-pr610-final-signoff.md: 70 PASS / 4 FAIL / ", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/610" }, "confirmed": true, "difficulty": 1, "difficultyLo": 0.5655175352168252, "difficultyHi": 1, "strongestRow": "openai/gpt-6-astra@@medium", "defectReasons": [] }, { "id": "sfdata2-weak-moment-location-scale-v2-v2", "taskId": "sfdata2/weak-moment-location-scale-v2", "version": "2", "name": "datasciresearch/weak-moment-location-scale", "domain": "Data processing / data science", "source": "delivery", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "vendor redesign; solved 5/5", "knownDefects": [] }, "rows": [ { "rowId": "openai/gpt-6-astra@@medium", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/610" }, "confirmed": false, "difficulty": 0, "difficultyLo": 0, "difficultyHi": 0.43448246478317487, "strongestRow": "openai/gpt-6-astra@@medium", "defectReasons": [] }, { "id": "gopher-lua-exact-integers-75f4976-v3", "taskId": "gopher-lua-exact-integers-75f4976", "version": "3", "name": "gopher-lua-exact-integers-75f4976", "domain": "DeepSWE (long-horizon SWE)", "source": "delivery", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": true, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "review audit 2026-09-16: 8/8 gemini fails BAD_FAILURE (probes derived from Gemin", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gemini-3.7-flash@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_6be3a5abe6ca439e-v1", "taskId": "task_6be3a5abe6ca439e", "version": "1", "name": "gatt-database-serialize-and-diff", "domain": "DeepSWE (long-horizon SWE)", "source": "delivery", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "DELIVER-WITH-DISCLOSURE", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@high", "k": 0, "n": 8, "p": 0, "lo": 0, "hi": 0.32440756488388023, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.32440756488388023, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 8, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.4, "difficultyLo": 0.11762077423264805, "difficultyHi": 0.7692757187239871, "strongestRow": "claude-opus-5@@high", "defectReasons": [] }, { "id": "task_e4e29638eb354f33-v1", "taskId": "task_e4e29638eb354f33", "version": "1", "name": "slack-block-kit-validator", "domain": "DeepSWE (long-horizon SWE)", "source": "delivery", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": true, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "EXCLUDE", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@high", "k": 0, "n": 8, "p": 0, "lo": 0, "hi": 0.32440756488388023, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.32440756488388023, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 8, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": 0.8, "difficultyLo": 0.3755346297625253, "difficultyHi": 0.9637758913675699, "strongestRow": "claude-opus-5@@high", "defectReasons": [] }, { "id": "task_6137c113d11144a3-v4", "taskId": "task_6137c113d11144a3", "version": "4", "name": "landmark-competing-risk-scorecard", "domain": "Machine learning", "source": "delivery", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": true, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "EXCLUDE as packaged (reviewer recommendation; owner decision pending)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 8, "p": 0, "lo": 0, "hi": 0.32440756488388023, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.32440756488388023, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 8, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@high", "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_11c1e7d970804362-v4", "taskId": "task_11c1e7d970804362", "version": "4", "name": "occurrence-reinsurance-recovery-bordereau", "domain": "Operations", "source": "delivery", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": true, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "EXCLUDE (reviewer recommendation; owner decision pending)", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 3, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 8, "p": 0, "lo": 0, "hi": 0.32440756488388023, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.32440756488388023, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 5, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.375, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@high", "defectReasons": [ "Known task defect recorded", "Task fault on at least half of labelled current-model failures" ] }, { "id": "task_19e0016c752b4253-v1", "taskId": "task_19e0016c752b4253", "version": "1", "name": "oci-layer-admission-audit", "domain": "Security / cyber", "source": "delivery", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": true, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "EXCLUDE", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 0, "n": 7, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.3543304350666875, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.3543304350666875, "labelled": 7, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@high", "defectReasons": [ "Known task defect recorded" ] }, { "id": "task_4faf88d8421f4bf7-v1", "taskId": "task_4faf88d8421f4bf7", "version": "1", "name": "openssh-certificate-authorization", "domain": "Security / cyber", "source": "delivery", "tier": "contested/defective", "calibrated": true, "benchVersion": false, "curated": true, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "EXCLUDE", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-5@@high", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 1, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "tencent/hy4-preview@@think", "k": 3, "n": 8, "p": 0.375, "lo": 0.13684428582359737, "hi": 0.6942576053973728, "tasks": 1, "macro": 0.375, "macroLo": 0.13684428582359737, "macroHi": 0.6942576053973728, "labelled": 8, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 3, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.25, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": "claude-opus-5@@high", "defectReasons": [ "Known task defect recorded", "Task fault on at least half of labelled current-model failures" ] }, { "id": "admission-gateway-v1", "taskId": "admission-gateway", "version": "1", "name": "admission-gateway", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/admission-gateway" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "allotment-engine-v1", "taskId": "allotment-engine", "version": "1", "name": "allotment-engine", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/allotment-engine" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "animal-complaints-etl-v1", "taskId": "animal-complaints-etl", "version": "1", "name": "animal-complaints-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/animal-complaints-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "animal-crossing-items-etl-v1", "taskId": "animal-crossing-items-etl", "version": "1", "name": "animal-crossing-items-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/animal-crossing-items-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "animal-crossing-items-joins-v1", "taskId": "animal-crossing-items-joins", "version": "1", "name": "animal-crossing-items-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/animal-crossing-items-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "apple-bplist-v0-reader-v1", "taskId": "apple-bplist-v0-reader", "version": "1", "name": "apple-bplist-v0-reader", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/systems-spec/apple-bplist-v0-reader" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "audio-features-cleaning-v1", "taskId": "audio-features-cleaning", "version": "1", "name": "audio-features-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/audio-features-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "australia-fires-etl-v1", "taskId": "australia-fires-etl", "version": "1", "name": "australia-fires-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/australia-fires-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "australia-fires-joins-v1", "taskId": "australia-fires-joins", "version": "1", "name": "australia-fires-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/australia-fires-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bee-colonies-etl-v1", "taskId": "bee-colonies-etl", "version": "1", "name": "bee-colonies-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/bee-colonies-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bee-colonies-joins-v1", "taskId": "bee-colonies-joins", "version": "1", "name": "bee-colonies-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/bee-colonies-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bee-colonies-missing-data-v1", "taskId": "bee-colonies-missing-data", "version": "1", "name": "bee-colonies-missing-data", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/bee-colonies-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "beer-production-joins-v1", "taskId": "beer-production-joins", "version": "1", "name": "beer-production-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/beer-production-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bidi-isolates-weak-neutral-resolution-v1", "taskId": "bidi-isolates-weak-neutral-resolution", "version": "1", "name": "bidi-isolates-weak-neutral-resolution", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/bidi-isolates-weak-neutral-resolution" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bidi-n0-bracket-pairing-v1", "taskId": "bidi-n0-bracket-pairing", "version": "1", "name": "bidi-n0-bracket-pairing", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/bidi-n0-bracket-pairing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bike-traffic-outliers-v1", "taskId": "bike-traffic-outliers", "version": "1", "name": "bike-traffic-outliers", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/bike-traffic-outliers" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "billboard-etl-v1", "taskId": "billboard-etl", "version": "1", "name": "billboard-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/billboard-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bird-collisions-modeling-v1", "taskId": "bird-collisions-modeling", "version": "1", "name": "bird-collisions-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/bird-collisions-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bom-explosion-recursive-quantity-rollup-v1", "taskId": "bom-explosion-recursive-quantity-rollup", "version": "1", "name": "bom-explosion-recursive-quantity-rollup", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/entsql/bom-explosion-recursive-quantity-rollup" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "boston-cocktails-cleaning-v1", "taskId": "boston-cocktails-cleaning", "version": "1", "name": "boston-cocktails-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/boston-cocktails-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "case-folding-turkic-specialcasing-v1", "taskId": "case-folding-turkic-specialcasing", "version": "1", "name": "case-folding-turkic-specialcasing", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/case-folding-turkic-specialcasing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chess-modeling-v1", "taskId": "chess-modeling", "version": "1", "name": "chess-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/chess-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "childcare-costs-etl-v1", "taskId": "childcare-costs-etl", "version": "1", "name": "childcare-costs-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/childcare-costs-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "childcare-costs-reshape-v1", "taskId": "childcare-costs-reshape", "version": "1", "name": "childcare-costs-reshape", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/childcare-costs-reshape" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chocolate-cleaning-v1", "taskId": "chocolate-cleaning", "version": "1", "name": "chocolate-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/chocolate-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cidrforge-v1", "taskId": "cidrforge", "version": "1", "name": "cidrforge", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/cidrforge" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "citation-transitive-influence-reachability-v1", "taskId": "citation-transitive-influence-reachability", "version": "1", "name": "citation-transitive-influence-reachability", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lattice/citation-transitive-influence-reachability" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "claim-severity-weighted-percentile-v1", "taskId": "claim-severity-weighted-percentile", "version": "1", "name": "claim-severity-weighted-percentile", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/entsql/claim-severity-weighted-percentile" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "claims-zero-paid-null-semantics-v1", "taskId": "claims-zero-paid-null-semantics", "version": "1", "name": "claims-zero-paid-null-semantics", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/claims-zero-paid-null-semantics" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "claimsplit-v1", "taskId": "claimsplit", "version": "1", "name": "claimsplit", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/claimsplit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "clinic-local-day-no-show-null-v1", "taskId": "clinic-local-day-no-show-null", "version": "1", "name": "clinic-local-day-no-show-null", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/clinic-local-day-no-show-null" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cobol-packed-zoned-decimal-v1", "taskId": "cobol-packed-zoned-decimal", "version": "1", "name": "cobol-packed-zoned-decimal", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/systems-spec/cobol-packed-zoned-decimal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "coffee-ratings-cleaning-v1", "taskId": "coffee-ratings-cleaning", "version": "1", "name": "coffee-ratings-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/coffee-ratings-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "coffee-ratings-hypothesis-testing-v1", "taskId": "coffee-ratings-hypothesis-testing", "version": "1", "name": "coffee-ratings-hypothesis-testing", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/coffee-ratings-hypothesis-testing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "coffee-survey-outliers-v1", "taskId": "coffee-survey-outliers", "version": "1", "name": "coffee-survey-outliers", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/coffee-survey-outliers" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "crop-yield-per-acre-by-commodity-v1", "taskId": "crop-yield-per-acre-by-commodity", "version": "1", "name": "crop-yield-per-acre-by-commodity", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/datalake/crop-yield-per-acre-by-commodity" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "customer-golden-record-field-survivorship-v1", "taskId": "customer-golden-record-field-survivorship", "version": "1", "name": "customer-golden-record-field-survivorship", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/entsql/customer-golden-record-field-survivorship" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "doctor-who-joins-v1", "taskId": "doctor-who-joins", "version": "1", "name": "doctor-who-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/doctor-who-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "drought-fips-modeling-v1", "taskId": "drought-fips-modeling", "version": "1", "name": "drought-fips-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/drought-fips-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "eclipse-annular-2023-joins-v1", "taskId": "eclipse-annular-2023-joins", "version": "1", "name": "eclipse-annular-2023-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/eclipse-annular-2023-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ecommerce-net-revenue-by-category-v1", "taskId": "ecommerce-net-revenue-by-category", "version": "1", "name": "ecommerce-net-revenue-by-category", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/datalake/ecommerce-net-revenue-by-category" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "electricity-holtwinters-prediction-intervals-v1", "taskId": "electricity-holtwinters-prediction-intervals", "version": "1", "name": "electricity-holtwinters-prediction-intervals", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/timeseries/electricity-holtwinters-prediction-intervals" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "erasmus-cleaning-v1", "taskId": "erasmus-cleaning", "version": "1", "name": "erasmus-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/erasmus-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "european-flights-etl-v1", "taskId": "european-flights-etl", "version": "1", "name": "european-flights-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/european-flights-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "f1-racing-cleaning-v1", "taskId": "f1-racing-cleaning", "version": "1", "name": "f1-racing-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/f1-racing-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "f1-racing-joins-v1", "taskId": "f1-racing-joins", "version": "1", "name": "f1-racing-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/f1-racing-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "f1-racing-missing-data-v1", "taskId": "f1-racing-missing-data", "version": "1", "name": "f1-racing-missing-data", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/f1-racing-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "feederwatch-modeling-v1", "taskId": "feederwatch-modeling", "version": "1", "name": "feederwatch-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/feederwatch-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "freight-carrier-linehaul-fanout-v1", "taskId": "freight-carrier-linehaul-fanout", "version": "1", "name": "freight-carrier-linehaul-fanout", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/freight-carrier-linehaul-fanout" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "generalized-assignment-skill-budgets-v1", "taskId": "generalized-assignment-skill-budgets", "version": "1", "name": "generalized-assignment-skill-budgets", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/generalized-assignment-skill-budgets" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-hexfloat-p-exponent-v1", "taskId": "go-hexfloat-p-exponent", "version": "1", "name": "go-hexfloat-p-exponent", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/chomsky/go-hexfloat-p-exponent" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "great-lakes-fishing-joins-v1", "taskId": "great-lakes-fishing-joins", "version": "1", "name": "great-lakes-fishing-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/great-lakes-fishing-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "himalayan-expeditions-hypothesis-testing-v1", "taskId": "himalayan-expeditions-hypothesis-testing", "version": "1", "name": "himalayan-expeditions-hypothesis-testing", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/himalayan-expeditions-hypothesis-testing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "himalayan-expeditions-joins-v1", "taskId": "himalayan-expeditions-joins", "version": "1", "name": "himalayan-expeditions-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/himalayan-expeditions-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "himalayan-expeditions-missing-data-v1", "taskId": "himalayan-expeditions-missing-data", "version": "1", "name": "himalayan-expeditions-missing-data", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/himalayan-expeditions-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "horror-movies-cleaning-v1", "taskId": "horror-movies-cleaning", "version": "1", "name": "horror-movies-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/horror-movies-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "horror-movies-outliers-v1", "taskId": "horror-movies-outliers", "version": "1", "name": "horror-movies-outliers", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/horror-movies-outliers" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hotel-bookings-missing-data-v1", "taskId": "hotel-bookings-missing-data", "version": "1", "name": "hotel-bookings-missing-data", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/hotel-bookings-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hotels-cleaning-v1", "taskId": "hotels-cleaning", "version": "1", "name": "hotels-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/hotels-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "idna-uts46-punycode-roundtrip-v1", "taskId": "idna-uts46-punycode-roundtrip", "version": "1", "name": "idna-uts46-punycode-roundtrip", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/idna-uts46-punycode-roundtrip" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "incarceration-trends-etl-v1", "taskId": "incarceration-trends-etl", "version": "1", "name": "incarceration-trends-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/incarceration-trends-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "incarceration-trends-modeling-v1", "taskId": "incarceration-trends-modeling", "version": "1", "name": "incarceration-trends-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/incarceration-trends-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "incarceration-trends-reshape-v1", "taskId": "incarceration-trends-reshape", "version": "1", "name": "incarceration-trends-reshape", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/incarceration-trends-reshape" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ipl-toss-impact-analysis-r-v1", "taskId": "ipl-toss-impact-analysis-r", "version": "1", "name": "ipl-toss-impact-analysis-r", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/99" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "job-postings-skill-demand-delimited-unnest-v1", "taskId": "job-postings-skill-demand-delimited-unnest", "version": "1", "name": "job-postings-skill-demand-delimited-unnest", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/job-postings-skill-demand-delimited-unnest" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "job-scheduler-core-v1", "taskId": "job-scheduler-core", "version": "1", "name": "job-scheduler-core", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/job-scheduler-core" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lemur-data-etl-v1", "taskId": "lemur-data-etl", "version": "1", "name": "lemur-data-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/lemur-data-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lemur-data-missing-data-v1", "taskId": "lemur-data-missing-data", "version": "1", "name": "lemur-data-missing-data", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/lemur-data-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "line-break-uax14-ri-emoji-lb30a-lb30b-v1", "taskId": "line-break-uax14-ri-emoji-lb30a-lb30b", "version": "1", "name": "line-break-uax14-ri-emoji-lb30a-lb30b", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/line-break-uax14-ri-emoji-lb30a-lb30b" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "locations-cleaning-v1", "taskId": "locations-cleaning", "version": "1", "name": "locations-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/locations-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lumenyard-v1", "taskId": "lumenyard", "version": "1", "name": "lumenyard", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/lumenyard" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "medical-device-cold-chain-exposure-v1", "taskId": "medical-device-cold-chain-exposure", "version": "1", "name": "medical-device-cold-chain-exposure", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/datalake/medical-device-cold-chain-exposure" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "min-cost-flow-arc-lower-bounds-v1", "taskId": "min-cost-flow-arc-lower-bounds", "version": "1", "name": "min-cost-flow-arc-lower-bounds", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/min-cost-flow-arc-lower-bounds" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mroz-heckman-sample-selection-v1", "taskId": "mroz-heckman-sample-selection", "version": "1", "name": "mroz-heckman-sample-selection", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/mroz-heckman-sample-selection" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "multicurrency-net-revenue-asof-fx-v1", "taskId": "multicurrency-net-revenue-asof-fx", "version": "1", "name": "multicurrency-net-revenue-asof-fx", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/multicurrency-net-revenue-asof-fx" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nfd-recursive-decompose-hangul-v1", "taskId": "nfd-recursive-decompose-hangul", "version": "1", "name": "nfd-recursive-decompose-hangul", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/nfd-recursive-decompose-hangul" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nfl-attendance-modeling-v1", "taskId": "nfl-attendance-modeling", "version": "1", "name": "nfl-attendance-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/nfl-attendance-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "noaa-tornados-cleaning-v1", "taskId": "noaa-tornados-cleaning", "version": "1", "name": "noaa-tornados-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/noaa-tornados-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "noaa-tornados-outliers-v1", "taskId": "noaa-tornados-outliers", "version": "1", "name": "noaa-tornados-outliers", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/noaa-tornados-outliers" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nurse-shift-rest-period-v1", "taskId": "nurse-shift-rest-period", "version": "1", "name": "nurse-shift-rest-period", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/nurse-shift-rest-period" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "olympics-etl-v1", "taskId": "olympics-etl", "version": "1", "name": "olympics-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/olympics-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "olympics-hypothesis-testing-v1", "taskId": "olympics-hypothesis-testing", "version": "1", "name": "olympics-hypothesis-testing", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/olympics-hypothesis-testing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "olympics-joins-v1", "taskId": "olympics-joins", "version": "1", "name": "olympics-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/olympics-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "orgchart-manager-chain-reachability-v1", "taskId": "orgchart-manager-chain-reachability", "version": "1", "name": "orgchart-manager-chain-reachability", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lattice/orgchart-manager-chain-reachability" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "owid-energy-reshape-v1", "taskId": "owid-energy-reshape", "version": "1", "name": "owid-energy-reshape", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/owid-energy-reshape" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "packwire-v1", "taskId": "packwire", "version": "1", "name": "packwire", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/packwire" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "parallel-machine-maintenance-makespan-v1", "taskId": "parallel-machine-maintenance-makespan", "version": "1", "name": "parallel-machine-maintenance-makespan", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/parallel-machine-maintenance-makespan" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "paygap-modeling-v1", "taskId": "paygap-modeling", "version": "1", "name": "paygap-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/paygap-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pell-etl-v1", "taskId": "pell-etl", "version": "1", "name": "pell-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/pell-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "plastic-pollution-audits-joins-v1", "taskId": "plastic-pollution-audits-joins", "version": "1", "name": "plastic-pollution-audits-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/plastic-pollution-audits-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "plastic-pollution-etl-v1", "taskId": "plastic-pollution-etl", "version": "1", "name": "plastic-pollution-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/plastic-pollution-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "polling-places-etl-v1", "taskId": "polling-places-etl", "version": "1", "name": "polling-places-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/polling-places-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "post-offices-cleaning-v1", "taskId": "post-offices-cleaning", "version": "1", "name": "post-offices-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/post-offices-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "post-offices-etl-v1", "taskId": "post-offices-etl", "version": "1", "name": "post-offices-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/post-offices-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pretrial-population-joins-v1", "taskId": "pretrial-population-joins", "version": "1", "name": "pretrial-population-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/pretrial-population-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pricing-engine-v1", "taskId": "pricing-engine", "version": "1", "name": "pricing-engine", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/pricing-engine" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "protocol-state-machine-aarch64-v1", "taskId": "protocol-state-machine-aarch64", "version": "1", "name": "protocol-state-machine-aarch64", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/protocol-state-machine-aarch64" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pulsefold-v1", "taskId": "pulsefold", "version": "1", "name": "pulsefold", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/pulsefold" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rate-governor-v1", "taskId": "rate-governor", "version": "1", "name": "rate-governor", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/rate-governor" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "regex-engine-v1", "taskId": "regex-engine", "version": "1", "name": "regex-engine", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/regex-engine" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "restaurant-shift-waste-costs-v1", "taskId": "restaurant-shift-waste-costs", "version": "1", "name": "restaurant-shift-waste-costs", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/datalake/restaurant-shift-waste-costs" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-008ad3a45371bee1-v1", "taskId": "review-008ad3a45371bee1", "version": "1", "name": "Snorkel-reviewed task review-008ad3a45371bee1", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/152" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0152a1c96b9d8103-v1", "taskId": "review-0152a1c96b9d8103", "version": "1", "name": "Snorkel-reviewed task review-0152a1c96b9d8103", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-032352cd7db8f953-v1", "taskId": "review-032352cd7db8f953", "version": "1", "name": "Snorkel-reviewed task review-032352cd7db8f953", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/258" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-05fc149892e70110-v1", "taskId": "review-05fc149892e70110", "version": "1", "name": "Snorkel-reviewed task review-05fc149892e70110", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/262" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-07ca06f6e0e06abb-v1", "taskId": "review-07ca06f6e0e06abb", "version": "1", "name": "Snorkel-reviewed task review-07ca06f6e0e06abb", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0830d6f93658dc71-v1", "taskId": "review-0830d6f93658dc71", "version": "1", "name": "Snorkel-reviewed task review-0830d6f93658dc71", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/209" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0a6d6c1681fd4c91-v1", "taskId": "review-0a6d6c1681fd4c91", "version": "1", "name": "Snorkel-reviewed task review-0a6d6c1681fd4c91", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/211" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0e64f73269406cdb-v1", "taskId": "review-0e64f73269406cdb", "version": "1", "name": "Snorkel-reviewed task review-0e64f73269406cdb", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/218" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-10daef0770206047-v1", "taskId": "review-10daef0770206047", "version": "1", "name": "Snorkel-reviewed task review-10daef0770206047", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/185" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-11c6f402ab5f36a9-v1", "taskId": "review-11c6f402ab5f36a9", "version": "1", "name": "Snorkel-reviewed task review-11c6f402ab5f36a9", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/234" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-13ed8da0fe78942b-v1", "taskId": "review-13ed8da0fe78942b", "version": "1", "name": "Snorkel-reviewed task review-13ed8da0fe78942b", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/178" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-15dc6c3847427416-v1", "taskId": "review-15dc6c3847427416", "version": "1", "name": "Snorkel-reviewed task review-15dc6c3847427416", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/292" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-203f3c27ffc3b145-v1", "taskId": "review-203f3c27ffc3b145", "version": "1", "name": "Snorkel-reviewed task review-203f3c27ffc3b145", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/231" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-205fc9589670c791-v1", "taskId": "review-205fc9589670c791", "version": "1", "name": "Snorkel-reviewed task review-205fc9589670c791", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T042215Z__9c03e10c__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-21fe28c4186b8e9b-v1", "taskId": "review-21fe28c4186b8e9b", "version": "1", "name": "Snorkel-reviewed task review-21fe28c4186b8e9b", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/194" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-22f5aea227d28e50-v1", "taskId": "review-22f5aea227d28e50", "version": "1", "name": "Snorkel-reviewed task review-22f5aea227d28e50", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/232" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-23a79779fea08dd9-v1", "taskId": "review-23a79779fea08dd9", "version": "1", "name": "Snorkel-reviewed task review-23a79779fea08dd9", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-27657be69c4edb49-v1", "taskId": "review-27657be69c4edb49", "version": "1", "name": "Snorkel-reviewed task review-27657be69c4edb49", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 4, "p": 0.25, "lo": 0.04558726080970055, "hi": 0.6993581574175981, "tasks": 1, "macro": 0.25, "macroLo": 0.04558726080970055, "macroHi": 0.6993581574175981, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/220" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-286ce4d3586d9214-v1", "taskId": "review-286ce4d3586d9214", "version": "1", "name": "rust-vcf-haplotype-block-consistency-auditor", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2ae2f31b-v1", "taskId": "review-2ae2f31b", "version": "1", "name": "Snorkel-reviewed task review-2ae2f31b", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2bbc7dc7-v1", "taskId": "review-2bbc7dc7", "version": "1", "name": "Snorkel-reviewed task review-2bbc7dc7", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2f214a8890a339ec-v1", "taskId": "review-2f214a8890a339ec", "version": "1", "name": "Snorkel-reviewed task review-2f214a8890a339ec", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2f88566672ab0b75-v1", "taskId": "review-2f88566672ab0b75", "version": "1", "name": "Snorkel-reviewed task review-2f88566672ab0b75", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/241" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2f8afb67c99a6ca3-v1", "taskId": "review-2f8afb67c99a6ca3", "version": "1", "name": "Snorkel-reviewed task review-2f8afb67c99a6ca3", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-2f8e114e7da7a62b-v1", "taskId": "review-2f8e114e7da7a62b", "version": "1", "name": "parallel-machine-single-server-setup", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-319983f594885111-v1", "taskId": "review-319983f594885111", "version": "1", "name": "Snorkel-reviewed task review-319983f594885111", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-31a2e48327930691-v1", "taskId": "review-31a2e48327930691", "version": "1", "name": "Snorkel-reviewed task review-31a2e48327930691", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/311" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-35400c11fe8d83b5-v1", "taskId": "review-35400c11fe8d83b5", "version": "1", "name": "Snorkel-reviewed task review-35400c11fe8d83b5", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T052317Z__6d8ee432__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-373909373d46cce1-v1", "taskId": "review-373909373d46cce1", "version": "1", "name": "rust-vcf-haplotype-block-consistency-auditor", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-375ddd4b4d0e36f2-v1", "taskId": "review-375ddd4b4d0e36f2", "version": "1", "name": "Snorkel-reviewed task review-375ddd4b4d0e36f2", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T060125Z__0a715fb1__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-38136370-v1", "taskId": "review-38136370", "version": "1", "name": "Snorkel-reviewed task review-38136370", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-38299731ba828d36-v1", "taskId": "review-38299731ba828d36", "version": "1", "name": "Snorkel-reviewed task review-38299731ba828d36", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3d0510d0-v1", "taskId": "review-3d0510d0", "version": "1", "name": "Snorkel-reviewed task review-3d0510d0", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-474e08262d899fc1-v1", "taskId": "review-474e08262d899fc1", "version": "1", "name": "Snorkel-reviewed task review-474e08262d899fc1", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/256" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4b27f5f12b676ea8-v1", "taskId": "review-4b27f5f12b676ea8", "version": "1", "name": "Snorkel-reviewed task review-4b27f5f12b676ea8", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/425" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4be40120934f9b09-v1", "taskId": "review-4be40120934f9b09", "version": "1", "name": "Snorkel-reviewed task review-4be40120934f9b09", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/261" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-50880b1aeec0b1df-v1", "taskId": "review-50880b1aeec0b1df", "version": "1", "name": "Snorkel-reviewed task review-50880b1aeec0b1df", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/199" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5204b7b144f08a1a-v1", "taskId": "review-5204b7b144f08a1a", "version": "1", "name": "Snorkel-reviewed task review-5204b7b144f08a1a", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/159" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-535c56e3ee975cee-v1", "taskId": "review-535c56e3ee975cee", "version": "1", "name": "Snorkel-reviewed task review-535c56e3ee975cee", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 8, "n": 10, "p": 0.8, "lo": 0.4901624715366418, "hi": 0.9433178485456248, "tasks": 1, "macro": 0.8, "macroLo": 0.4901624715366418, "macroHi": 0.9433178485456248, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/249" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-54f4819d9bd34180-v1", "taskId": "review-54f4819d9bd34180", "version": "1", "name": "Snorkel-reviewed task review-54f4819d9bd34180", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/182" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5781c01e47126df9-v1", "taskId": "review-5781c01e47126df9", "version": "1", "name": "Snorkel-reviewed task review-5781c01e47126df9", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-58c18cd7-v1", "taskId": "review-58c18cd7", "version": "1", "name": "Snorkel-reviewed task review-58c18cd7", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6d30e8417ea73fa4-v1", "taskId": "review-6d30e8417ea73fa4", "version": "1", "name": "Snorkel-reviewed task review-6d30e8417ea73fa4", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-711947c83ac6640b-v1", "taskId": "review-711947c83ac6640b", "version": "1", "name": "Snorkel-reviewed task review-711947c83ac6640b", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/195" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7236c8abe5b40afa-v1", "taskId": "review-7236c8abe5b40afa", "version": "1", "name": "Snorkel-reviewed task review-7236c8abe5b40afa", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-734a8a0d121dcb2d-v1", "taskId": "review-734a8a0d121dcb2d", "version": "1", "name": "Snorkel-reviewed task review-734a8a0d121dcb2d", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/207" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7402e41286eb7cfc-v1", "taskId": "review-7402e41286eb7cfc", "version": "1", "name": "Snorkel-reviewed task review-7402e41286eb7cfc", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 3, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/191" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-76569403dfa221f4-v1", "taskId": "review-76569403dfa221f4", "version": "1", "name": "Snorkel-reviewed task review-76569403dfa221f4", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-29T143440Z__d3fcef1b__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7688a880a0c820a2-v1", "taskId": "review-7688a880a0c820a2", "version": "1", "name": "Snorkel-reviewed task review-7688a880a0c820a2", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/305" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-79fdb8aee3e9b29f-v1", "taskId": "review-79fdb8aee3e9b29f", "version": "1", "name": "Snorkel-reviewed task review-79fdb8aee3e9b29f", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7babdd01-v1", "taskId": "review-7babdd01", "version": "1", "name": "Snorkel-reviewed task review-7babdd01", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8016f33a45d3e463-v1", "taskId": "review-8016f33a45d3e463", "version": "1", "name": "Snorkel-reviewed task review-8016f33a45d3e463", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-80470c9d711756ae-v1", "taskId": "review-80470c9d711756ae", "version": "1", "name": "Snorkel-reviewed task review-80470c9d711756ae", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 3, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/137" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8121e26ecf94ace5-v1", "taskId": "review-8121e26ecf94ace5", "version": "1", "name": "Snorkel-reviewed task review-8121e26ecf94ace5", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/273" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-83499b65c829dc20-v1", "taskId": "review-83499b65c829dc20", "version": "1", "name": "Snorkel-reviewed task review-83499b65c829dc20", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T051548Z__8e2d084f__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8367d115cc36c5ea-v1", "taskId": "review-8367d115cc36c5ea", "version": "1", "name": "Snorkel-reviewed task review-8367d115cc36c5ea", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8a0305cc9dd6fd40-v1", "taskId": "review-8a0305cc9dd6fd40", "version": "1", "name": "Snorkel-reviewed task review-8a0305cc9dd6fd40", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/260" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8b1b942a321c83d6-v1", "taskId": "review-8b1b942a321c83d6", "version": "1", "name": "Snorkel-reviewed task review-8b1b942a321c83d6", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/198" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8e684b7adb9cbb82-v1", "taskId": "review-8e684b7adb9cbb82", "version": "1", "name": "Snorkel-reviewed task review-8e684b7adb9cbb82", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/20260702T134113Z__1ad6c9e6__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8f290d83-v1", "taskId": "review-8f290d83", "version": "1", "name": "Snorkel-reviewed task review-8f290d83", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8f8e94e3de72ad68-v1", "taskId": "review-8f8e94e3de72ad68", "version": "1", "name": "Snorkel-reviewed task review-8f8e94e3de72ad68", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 3, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/148" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8fa97351-v1", "taskId": "review-8fa97351", "version": "1", "name": "Snorkel-reviewed task review-8fa97351", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9116c038-v1", "taskId": "review-9116c038", "version": "1", "name": "Snorkel-reviewed task review-9116c038", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-91b9576b01592414-v1", "taskId": "review-91b9576b01592414", "version": "1", "name": "Snorkel-reviewed task review-91b9576b01592414", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-9206f9f198f440ed-v1", "taskId": "review-9206f9f198f440ed", "version": "1", "name": "Snorkel-reviewed task review-9206f9f198f440ed", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9528cda2f1e12500-v1", "taskId": "review-9528cda2f1e12500", "version": "1", "name": "Snorkel-reviewed task review-9528cda2f1e12500", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-96f21e2c-v1", "taskId": "review-96f21e2c", "version": "1", "name": "Snorkel-reviewed task review-96f21e2c", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/219" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-98f43e43-v1", "taskId": "review-98f43e43", "version": "1", "name": "Snorkel-reviewed task review-98f43e43", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9ba5a91d7ce33e0b-v1", "taskId": "review-9ba5a91d7ce33e0b", "version": "1", "name": "Snorkel-reviewed task review-9ba5a91d7ce33e0b", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/163" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9be89ed8e69def86-v1", "taskId": "review-9be89ed8e69def86", "version": "1", "name": "Snorkel-reviewed task review-9be89ed8e69def86", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/304" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a1a2cdaaf525c38a-v1", "taskId": "review-a1a2cdaaf525c38a", "version": "1", "name": "Snorkel-reviewed task review-a1a2cdaaf525c38a", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a1e1a80f39f053bc-v1", "taskId": "review-a1e1a80f39f053bc", "version": "1", "name": "Snorkel-reviewed task review-a1e1a80f39f053bc", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/248" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a77f124e37f3ac9b-v1", "taskId": "review-a77f124e37f3ac9b", "version": "1", "name": "Snorkel-reviewed task review-a77f124e37f3ac9b", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a9e671f0fa608f9d-v1", "taskId": "review-a9e671f0fa608f9d", "version": "1", "name": "Snorkel-reviewed task review-a9e671f0fa608f9d", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/158" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-aaef3e3a549f1422-v1", "taskId": "review-aaef3e3a549f1422", "version": "1", "name": "Snorkel-reviewed task review-aaef3e3a549f1422", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/349" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-adc4fc6b-v1", "taskId": "review-adc4fc6b", "version": "1", "name": "Snorkel-reviewed task review-adc4fc6b", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-aea4fa357faa5976-v1", "taskId": "review-aea4fa357faa5976", "version": "1", "name": "Snorkel-reviewed task review-aea4fa357faa5976", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-af4a3f2d2fd450ab-v1", "taskId": "review-af4a3f2d2fd450ab", "version": "1", "name": "Snorkel-reviewed task review-af4a3f2d2fd450ab", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/145" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b0e3721bf746b9a6-v1", "taskId": "review-b0e3721bf746b9a6", "version": "1", "name": "Snorkel-reviewed task review-b0e3721bf746b9a6", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/296" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b2267b7f5a55574a-v1", "taskId": "review-b2267b7f5a55574a", "version": "1", "name": "Snorkel-reviewed task review-b2267b7f5a55574a", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b23fd4f3bbf1905b-v1", "taskId": "review-b23fd4f3bbf1905b", "version": "1", "name": "Snorkel-reviewed task review-b23fd4f3bbf1905b", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b7c3795f60b85a6e-v1", "taskId": "review-b7c3795f60b85a6e", "version": "1", "name": "Snorkel-reviewed task review-b7c3795f60b85a6e", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/200" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ba411a1299dde36a-v1", "taskId": "review-ba411a1299dde36a", "version": "1", "name": "Snorkel-reviewed task review-ba411a1299dde36a", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/397" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-bbea5f6b27ca907f-v1", "taskId": "review-bbea5f6b27ca907f", "version": "1", "name": "Snorkel-reviewed task review-bbea5f6b27ca907f", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c116d963-v1", "taskId": "review-c116d963", "version": "1", "name": "Snorkel-reviewed task review-c116d963", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c21e7240e37f24f1-v1", "taskId": "review-c21e7240e37f24f1", "version": "1", "name": "Snorkel-reviewed task review-c21e7240e37f24f1", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c4939b39c3a37010-v1", "taskId": "review-c4939b39c3a37010", "version": "1", "name": "Snorkel-reviewed task review-c4939b39c3a37010", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/180" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c69b7fc2c3d92ce4-v1", "taskId": "review-c69b7fc2c3d92ce4", "version": "1", "name": "Snorkel-reviewed task review-c69b7fc2c3d92ce4", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/297" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c81c23c38bc0ef40-v1", "taskId": "review-c81c23c38bc0ef40", "version": "1", "name": "Snorkel-reviewed task review-c81c23c38bc0ef40", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/214" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-cb60ba39c33fd0da-v1", "taskId": "review-cb60ba39c33fd0da", "version": "1", "name": "Snorkel-reviewed task review-cb60ba39c33fd0da", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/140" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cc3a68b1b2a552d3-v1", "taskId": "review-cc3a68b1b2a552d3", "version": "1", "name": "Snorkel-reviewed task review-cc3a68b1b2a552d3", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/280" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cd4f47217ea47f7c-v1", "taskId": "review-cd4f47217ea47f7c", "version": "1", "name": "Snorkel-reviewed task review-cd4f47217ea47f7c", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-cdc910d345082988-v1", "taskId": "review-cdc910d345082988", "version": "1", "name": "Snorkel-reviewed task review-cdc910d345082988", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/165" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d9423bb69741de07-v1", "taskId": "review-d9423bb69741de07", "version": "1", "name": "Snorkel-reviewed task review-d9423bb69741de07", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 4, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/175" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-de1063f9ec38f973-v1", "taskId": "review-de1063f9ec38f973", "version": "1", "name": "Snorkel-reviewed task review-de1063f9ec38f973", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 3, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/227" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e27e99f3c9d8884a-v1", "taskId": "review-e27e99f3c9d8884a", "version": "1", "name": "tidelab-mode-aware-allocation", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ead5bb42d26f0110-v1", "taskId": "review-ead5bb42d26f0110", "version": "1", "name": "bom-demand-exploder", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-eb9f9f325e65fad4-v1", "taskId": "review-eb9f9f325e65fad4", "version": "1", "name": "Snorkel-reviewed task review-eb9f9f325e65fad4", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/445" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ec2cadf0-v1", "taskId": "review-ec2cadf0", "version": "1", "name": "Snorkel-reviewed task review-ec2cadf0", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ec855015-v1", "taskId": "review-ec855015", "version": "1", "name": "Snorkel-reviewed task review-ec855015", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ecf643acf808427f-v1", "taskId": "review-ecf643acf808427f", "version": "1", "name": "Snorkel-reviewed task review-ecf643acf808427f", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-effd801de2d36566-v1", "taskId": "review-effd801de2d36566", "version": "1", "name": "Snorkel-reviewed task review-effd801de2d36566", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/20260702T141812Z__4c436f92__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-f14babaf078831ac-v1", "taskId": "review-f14babaf078831ac", "version": "1", "name": "Snorkel-reviewed task review-f14babaf078831ac", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/226" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-f210ced397d4dc6e-v1", "taskId": "review-f210ced397d4dc6e", "version": "1", "name": "Snorkel-reviewed task review-f210ced397d4dc6e", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/150" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f2cf0c232898ce52-v1", "taskId": "review-f2cf0c232898ce52", "version": "1", "name": "Snorkel-reviewed task review-f2cf0c232898ce52", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/263" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f5af3d2b-v1", "taskId": "review-f5af3d2b", "version": "1", "name": "Snorkel-reviewed task review-f5af3d2b", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f8b01b4eec225cd1-v1", "taskId": "review-f8b01b4eec225cd1", "version": "1", "name": "pallet-rebatcher", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fa9f6434b1fef55f-v1", "taskId": "review-fa9f6434b1fef55f", "version": "1", "name": "Snorkel-reviewed task review-fa9f6434b1fef55f", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/271" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-fc072cd1bf41b028-v1", "taskId": "review-fc072cd1bf41b028", "version": "1", "name": "Snorkel-reviewed task review-fc072cd1bf41b028", "domain": "Data processing / data science", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/151" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fd90fba599b56e65-v1", "taskId": "review-fd90fba599b56e65", "version": "1", "name": "Snorkel-reviewed task review-fd90fba599b56e65", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-kiln-shelf-registry-revised-v14-v1", "taskId": "review-slug-kiln-shelf-registry-revised-v14", "version": "1", "name": "kiln-shelf-registry-revised-v14", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-lua-busybox-aho-corasick-v1", "taskId": "review-slug-lua-busybox-aho-corasick", "version": "1", "name": "lua-busybox-aho-corasick", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-opentype-glyph-mapper-v1", "taskId": "review-slug-opentype-glyph-mapper", "version": "1", "name": "opentype-glyph-mapper", "domain": "Data processing / data science", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "review-slug-procurement-license-review-v1", "taskId": "review-slug-procurement-license-review", "version": "1", "name": "procurement-license-review", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rstats-tweets-etl-v1", "taskId": "rstats-tweets-etl", "version": "1", "name": "rstats-tweets-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/rstats-tweets-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "salary-potential-modeling-v1", "taskId": "salary-potential-modeling", "version": "1", "name": "salary-potential-modeling", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/salary-potential-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "scooby-doo-hypothesis-testing-v1", "taskId": "scooby-doo-hypothesis-testing", "version": "1", "name": "scooby-doo-hypothesis-testing", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/scooby-doo-hypothesis-testing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "seabird-sightings-joins-v1", "taskId": "seabird-sightings-joins", "version": "1", "name": "seabird-sightings-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/seabird-sightings-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "segmentation-uax29-suite-v1", "taskId": "segmentation-uax29-suite", "version": "1", "name": "segmentation-uax29-suite", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/segmentation-uax29-suite" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sensor-rollup-v1", "taskId": "sensor-rollup", "version": "1", "name": "sensor-rollup", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/sensor-rollup" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sentence-break-uax29-aterm-sterm-v1", "taskId": "sentence-break-uax29-aterm-sterm", "version": "1", "name": "sentence-break-uax29-aterm-sterm", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/sentence-break-uax29-aterm-sterm" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "settlement-fail-tplus2-business-day-calendar-v1", "taskId": "settlement-fail-tplus2-business-day-calendar", "version": "1", "name": "settlement-fail-tplus2-business-day-calendar", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/settlement-fail-tplus2-business-day-calendar" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sf-permits-cleaning-v1", "taskId": "sf-permits-cleaning", "version": "1", "name": "sf-permits-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/sf-permits-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sf-permits-joins-v1", "taskId": "sf-permits-joins", "version": "1", "name": "sf-permits-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/sf-permits-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "snapdepot-v1", "taskId": "snapdepot", "version": "1", "name": "snapdepot", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/snapdepot" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "softwaredep-directed-cycle-diamond-detection-v1", "taskId": "softwaredep-directed-cycle-diamond-detection", "version": "1", "name": "softwaredep-directed-cycle-diamond-detection", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lattice/softwaredep-directed-cycle-diamond-detection" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spandrel-v1", "taskId": "spandrel", "version": "1", "name": "spandrel", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/spandrel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spotify-songs-outliers-v1", "taskId": "spotify-songs-outliers", "version": "1", "name": "spotify-songs-outliers", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/spotify-songs-outliers" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "starscyg-robust-regression-suite-v1", "taskId": "starscyg-robust-regression-suite", "version": "1", "name": "starscyg-robust-regression-suite", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/starscyg-robust-regression-suite" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "state-milk-production-reshape-v1", "taskId": "state-milk-production-reshape", "version": "1", "name": "state-milk-production-reshape", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/state-milk-production-reshape" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "supplychain-embargo-upstream-reachability-v1", "taskId": "supplychain-embargo-upstream-reachability", "version": "1", "name": "supplychain-embargo-upstream-reachability", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lattice/supplychain-embargo-upstream-reachability" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "survivor-joins-v1", "taskId": "survivor-joins", "version": "1", "name": "survivor-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/survivor-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "survivor-missing-data-v1", "taskId": "survivor-missing-data", "version": "1", "name": "survivor-missing-data", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/survivor-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tate-artwork-joins-v1", "taskId": "tate-artwork-joins", "version": "1", "name": "tate-artwork-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/tate-artwork-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "taylor-swift-joins-v1", "taskId": "taylor-swift-joins", "version": "1", "name": "taylor-swift-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/taylor-swift-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "telecom-plan-revenue-by-tier-v1", "taskId": "telecom-plan-revenue-by-tier", "version": "1", "name": "telecom-plan-revenue-by-tier", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/datalake/telecom-plan-revenue-by-tier" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "telemetry-padding-layout-x86-v1", "taskId": "telemetry-padding-layout-x86", "version": "1", "name": "telemetry-padding-layout-x86", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/telemetry-padding-layout-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "thanksgiving-meals-etl-v1", "taskId": "thanksgiving-meals-etl", "version": "1", "name": "thanksgiving-meals-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/thanksgiving-meals-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tidemark-v1", "taskId": "tidemark", "version": "1", "name": "tidemark", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/tidemark" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tuition-income-cleaning-v1", "taskId": "tuition-income-cleaning", "version": "1", "name": "tuition-income-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/tuition-income-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uax14-line-break-cm-zwj-v1", "taskId": "uax14-line-break-cm-zwj", "version": "1", "name": "uax14-line-break-cm-zwj", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/uax14-line-break-cm-zwj" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ufo-sightings-etl-v1", "taskId": "ufo-sightings-etl", "version": "1", "name": "ufo-sightings-etl", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/ufo-sightings-etl" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ufo-sightings-joins-v1", "taskId": "ufo-sightings-joins", "version": "1", "name": "ufo-sightings-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/ufo-sightings-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ultra-rankings-cleaning-v1", "taskId": "ultra-rankings-cleaning", "version": "1", "name": "ultra-rankings-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/ultra-rankings-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "un-votes-joins-v1", "taskId": "un-votes-joins", "version": "1", "name": "un-votes-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/un-votes-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "us-place-names-reshape-v1", "taskId": "us-place-names-reshape", "version": "1", "name": "us-place-names-reshape", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/us-place-names-reshape" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vb-matches-joins-v1", "taskId": "vb-matches-joins", "version": "1", "name": "vb-matches-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/vb-matches-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vendor-invoice-receipt-variance-fanout-v1", "taskId": "vendor-invoice-receipt-variance-fanout", "version": "1", "name": "vendor-invoice-receipt-variance-fanout", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/entsql/vendor-invoice-receipt-variance-fanout" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "volcanoes-cleaning-v1", "taskId": "volcanoes-cleaning", "version": "1", "name": "volcanoes-cleaning", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/volcanoes-cleaning" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "warehouse-labor-pick-pack-fanout-v1", "taskId": "warehouse-labor-pick-pack-fanout", "version": "1", "name": "warehouse-labor-pick-pack-fanout", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/warehouse-labor-pick-pack-fanout" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "warehouse-picklist-unit-conversion-v1", "taskId": "warehouse-picklist-unit-conversion", "version": "1", "name": "warehouse-picklist-unit-conversion", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/warehouse-picklist-unit-conversion" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "warranty-reserve-adjustments-dedup-v1", "taskId": "warranty-reserve-adjustments-dedup", "version": "1", "name": "warranty-reserve-adjustments-dedup", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/entsql/warranty-reserve-adjustments-dedup" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "water-missing-data-v1", "taskId": "water-missing-data", "version": "1", "name": "water-missing-data", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/water-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wealth-income-joins-v1", "taskId": "wealth-income-joins", "version": "1", "name": "wealth-income-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/wealth-income-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wwbi-joins-v1", "taskId": "wwbi-joins", "version": "1", "name": "wwbi-joins", "domain": "Data processing / data science", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/wwbi-joins" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "alu-flags-v1", "taskId": "alu-flags", "version": "1", "name": "alu-flags", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/11" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "apb-lite-slave-regfile-v1", "taskId": "apb-lite-slave-regfile", "version": "1", "name": "apb-lite-slave-regfile", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/apb-lite-slave-regfile" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "apb-wait-slave-v1", "taskId": "apb-wait-slave", "version": "1", "name": "apb-wait-slave", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/67" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bcd-tens-rollover-v1", "taskId": "bcd-tens-rollover", "version": "1", "name": "bcd-tens-rollover", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/bcd-tens-rollover" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bin2bcd-v1", "taskId": "bin2bcd", "version": "1", "name": "bin2bcd", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/12" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bit-destuff-framer-v1", "taskId": "bit-destuff-framer", "version": "1", "name": "bit-destuff-framer", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/65" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "booth-multiplier-v1", "taskId": "booth-multiplier", "version": "1", "name": "booth-multiplier", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/13" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bus-slave-abort-ack-v1", "taskId": "bus-slave-abort-ack", "version": "1", "name": "bus-slave-abort-ack", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/bus-slave-abort-ack" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "byte-serial-round-scheduler-v1", "taskId": "byte-serial-round-scheduler", "version": "1", "name": "byte-serial-round-scheduler", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/byte-serial-round-scheduler" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "coprocessor-dispatcher-classmix-v1", "taskId": "coprocessor-dispatcher-classmix", "version": "1", "name": "coprocessor-dispatcher-classmix", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/coprocessor-dispatcher-classmix" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "crc-gen-v1", "taskId": "crc-gen", "version": "1", "name": "crc-gen", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/14" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "credit-flow-ctrl-v1", "taskId": "credit-flow-ctrl", "version": "1", "name": "credit-flow-ctrl", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/78" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "debug-halt-step-fsm-v1", "taskId": "debug-halt-step-fsm", "version": "1", "name": "debug-halt-step-fsm", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/debug-halt-step-fsm" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dualmaster-membridge-v1", "taskId": "dualmaster-membridge", "version": "1", "name": "dualmaster-membridge", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/dualmaster-membridge" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gated-fold-stage-v1", "taskId": "gated-fold-stage", "version": "1", "name": "gated-fold-stage", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/63" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "grant-lock-arbiter-v1", "taskId": "grant-lock-arbiter", "version": "1", "name": "grant-lock-arbiter", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/76" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hamming-ecc-v1", "taskId": "hamming-ecc", "version": "1", "name": "hamming-ecc", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/15" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hash-message-padder-v1", "taskId": "hash-message-padder", "version": "1", "name": "hash-message-padder", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/hash-message-padder" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hazard-forward-unit-v1", "taskId": "hazard-forward-unit", "version": "1", "name": "hazard-forward-unit", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/81" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nested-irq-arbiter-v1", "taskId": "nested-irq-arbiter", "version": "1", "name": "nested-irq-arbiter", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/73" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "one-wire-rx-v1", "taskId": "one-wire-rx", "version": "1", "name": "one-wire-rx", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/70" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "open-drain-command-engine-v1", "taskId": "open-drain-command-engine", "version": "1", "name": "open-drain-command-engine", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/62" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "outstanding-req-bridge-v1", "taskId": "outstanding-req-bridge", "version": "1", "name": "outstanding-req-bridge", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/80" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "page-program-suspend-v1", "taskId": "page-program-suspend", "version": "1", "name": "page-program-suspend", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/page-program-suspend" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "quadrature-decoder-v1", "taskId": "quadrature-decoder", "version": "1", "name": "quadrature-decoder", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/72" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "register-file-v1", "taskId": "register-file", "version": "1", "name": "register-file", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/16" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "reload-tick-counter-v1", "taskId": "reload-tick-counter", "version": "1", "name": "reload-tick-counter", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/79" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "retire-trap-handshake-v1", "taskId": "retire-trap-handshake", "version": "1", "name": "retire-trap-handshake", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/66" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "round-robin-arbiter-grant-v1", "taskId": "round-robin-arbiter-grant", "version": "1", "name": "round-robin-arbiter-grant", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/round-robin-arbiter-grant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rr-arbiter-v1", "taskId": "rr-arbiter", "version": "1", "name": "rr-arbiter", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/17" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sat-mac-stream-v1", "taskId": "sat-mac-stream", "version": "1", "name": "sat-mac-stream", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/69" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "seq-signed-divider-v1", "taskId": "seq-signed-divider", "version": "1", "name": "seq-signed-divider", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/64" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "serial-break-resync-v1", "taskId": "serial-break-resync", "version": "1", "name": "serial-break-resync", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/serial-break-resync" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "serial-receiver-framed-v1", "taskId": "serial-receiver-framed", "version": "1", "name": "serial-receiver-framed", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/serial-receiver-framed" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "serial-rx-resync-v1", "taskId": "serial-rx-resync", "version": "1", "name": "serial-rx-resync", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/68" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "skid-buffer-v1", "taskId": "skid-buffer", "version": "1", "name": "skid-buffer", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/77" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spi-master-v1", "taskId": "spi-master", "version": "1", "name": "spi-master", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/18" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "stream-frame-hold-v1", "taskId": "stream-frame-hold", "version": "1", "name": "stream-frame-hold", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/electrical/stream-frame-hold" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sync-fifo-v1", "taskId": "sync-fifo", "version": "1", "name": "sync-fifo", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/19" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uart-tx-v1", "taskId": "uart-tx", "version": "1", "name": "uart-tx", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/20" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "window-watchdog-v1", "taskId": "window-watchdog", "version": "1", "name": "window-watchdog", "domain": "Electrical engineering (RTL)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/71" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "camera-shake-rig-v1", "taskId": "camera-shake-rig", "version": "1", "name": "camera-shake-rig", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/22" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "car-scene-assembly-v1", "taskId": "car-scene-assembly", "version": "1", "name": "car-scene-assembly", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/25" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "checkpoint-system-v1", "taskId": "checkpoint-system", "version": "1", "name": "checkpoint-system", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/106" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chess960-fischer-castling-perft-v1", "taskId": "chess960-fischer-castling-perft", "version": "1", "name": "chess960-fischer-castling-perft", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/systems-spec/chess960-fischer-castling-perft" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "combo-score-system-v1", "taskId": "combo-score-system", "version": "1", "name": "combo-score-system", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/107" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "crusader-sprite-assembly-v1", "taskId": "crusader-sprite-assembly", "version": "1", "name": "crusader-sprite-assembly", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/26" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "day-night-cycle-controller-v1", "taskId": "day-night-cycle-controller", "version": "1", "name": "day-night-cycle-controller", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/23" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "enemy-chase-navigation-v1", "taskId": "enemy-chase-navigation", "version": "1", "name": "enemy-chase-navigation", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/27" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lines-of-action-move-legality-v1", "taskId": "lines-of-action-move-legality", "version": "1", "name": "lines-of-action-move-legality", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/systems-spec/lines-of-action-move-legality" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "minimap-marker-logic-v1", "taskId": "minimap-marker-logic", "version": "1", "name": "minimap-marker-logic", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/30" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "minimap-ui-v1", "taskId": "minimap-ui", "version": "1", "name": "minimap-ui", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/29" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "renju-recursive-forbidden-point-v1", "taskId": "renju-recursive-forbidden-point", "version": "1", "name": "renju-recursive-forbidden-point", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/systems-spec/renju-recursive-forbidden-point" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-00a25436e5019dd4-v1", "taskId": "review-00a25436e5019dd4", "version": "1", "name": "Snorkel-reviewed task review-00a25436e5019dd4", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/406" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0588ba287be732ec-v1", "taskId": "review-0588ba287be732ec", "version": "1", "name": "lattice-arena-skirmish", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-05ce1ea333e37543-v1", "taskId": "review-05ce1ea333e37543", "version": "1", "name": "Snorkel-reviewed task review-05ce1ea333e37543", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-30T152620Z__272cf9f0__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0d03a77f6269e1e6-v1", "taskId": "review-0d03a77f6269e1e6", "version": "1", "name": "sysex-rhythm-replay-certificate", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-186c78b61bef7d57-v1", "taskId": "review-186c78b61bef7d57", "version": "1", "name": "racket-prismfall-coordinator", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2116b096d6c849e3-v1", "taskId": "review-2116b096d6c849e3", "version": "1", "name": "cablerun", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 4, "p": 0, "lo": 0, "hi": 0.4898908364545973, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.4898908364545973, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-33dc292effece0ee-v1", "taskId": "review-33dc292effece0ee", "version": "1", "name": "Snorkel-reviewed task review-33dc292effece0ee", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/228" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3595a0e87487e648-v1", "taskId": "review-3595a0e87487e648", "version": "1", "name": "Snorkel-reviewed task review-3595a0e87487e648", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/440" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-386bac885f7491f9-v1", "taskId": "review-386bac885f7491f9", "version": "1", "name": "Snorkel-reviewed task review-386bac885f7491f9", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/289" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-3b24306d3d51afe2-v1", "taskId": "review-3b24306d3d51afe2", "version": "1", "name": "tbrain-slantway-move-legality", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4117ff845d0f6436-v1", "taskId": "review-4117ff845d0f6436", "version": "1", "name": "racket-prismfall-coordinator", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4553b41e-v1", "taskId": "review-4553b41e", "version": "1", "name": "Snorkel-reviewed task review-4553b41e", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-22T170053Z__4553b41e__accept" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-460929c386107279-v1", "taskId": "review-460929c386107279", "version": "1", "name": "deterministic-match-recorder", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-59e1625dbd9c6ebe-v1", "taskId": "review-59e1625dbd9c6ebe", "version": "1", "name": "moidore", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5f5184751cda3c44-v1", "taskId": "review-5f5184751cda3c44", "version": "1", "name": "wave-feasibility-cert", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6b94f1c3c9373e1a-v1", "taskId": "review-6b94f1c3c9373e1a", "version": "1", "name": "Snorkel-reviewed task review-6b94f1c3c9373e1a", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/20260702T232023Z__d6638f6b__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6c7a149682ddda41-v1", "taskId": "review-6c7a149682ddda41", "version": "1", "name": "karst-pressure-balance", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6ef59691610b1103-v1", "taskId": "review-6ef59691610b1103", "version": "1", "name": "Snorkel-reviewed task review-6ef59691610b1103", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-72df8d6f-v1", "taskId": "review-72df8d6f", "version": "1", "name": "Snorkel-reviewed task review-72df8d6f", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7418b726e28cf10b-v1", "taskId": "review-7418b726e28cf10b", "version": "1", "name": "glass-harbor-time-loop", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-76bb9bbea4b3baa4-v1", "taskId": "review-76bb9bbea4b3baa4", "version": "1", "name": "go-sounding", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-78e56763c9b23d60-v1", "taskId": "review-78e56763c9b23d60", "version": "1", "name": "Snorkel-reviewed task review-78e56763c9b23d60", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/156" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-797f7315ec7f1376-v1", "taskId": "review-797f7315ec7f1376", "version": "1", "name": "racket-prismfall-coordinator", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7f4b695e342cf02e-v1", "taskId": "review-7f4b695e342cf02e", "version": "1", "name": "coldstore-pallet-clear", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-82d3a5b6a47f39f9-v1", "taskId": "review-82d3a5b6a47f39f9", "version": "1", "name": "rhythm-tactics-tournament-replay-engine", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-831984ea50964864-v1", "taskId": "review-831984ea50964864", "version": "1", "name": "quoridor-fence-path-adjudication-seal", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-84e6489f-v1", "taskId": "review-84e6489f", "version": "1", "name": "Snorkel-reviewed task review-84e6489f", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-869b13cd00c5aa50-v1", "taskId": "review-869b13cd00c5aa50", "version": "1", "name": "Snorkel-reviewed task review-869b13cd00c5aa50", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/196" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8b963fc0b5ef2d2e-v1", "taskId": "review-8b963fc0b5ef2d2e", "version": "1", "name": "Snorkel-reviewed task review-8b963fc0b5ef2d2e", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8fe4a8cb-v1", "taskId": "review-8fe4a8cb", "version": "1", "name": "Snorkel-reviewed task review-8fe4a8cb", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9102053c503d260b-v1", "taskId": "review-9102053c503d260b", "version": "1", "name": "registry-expedition", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-92aaeaf79aff8acb-v1", "taskId": "review-92aaeaf79aff8acb", "version": "1", "name": "rhythm-tactics-tournament-replay-engine", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 8, "n": 10, "p": 0.8, "lo": 0.4901624715366418, "hi": 0.9433178485456248, "tasks": 1, "macro": 0.8, "macroLo": 0.4901624715366418, "macroHi": 0.9433178485456248, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-94950cd7673b2085-v1", "taskId": "review-94950cd7673b2085", "version": "1", "name": "go-stone-arena", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-951cc4e805caad96-v1", "taskId": "review-951cc4e805caad96", "version": "1", "name": "pathfinding-memory-optimize-grid", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9d426aeef6cfcb29-v1", "taskId": "review-9d426aeef6cfcb29", "version": "1", "name": "Snorkel-reviewed task review-9d426aeef6cfcb29", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/386" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-af06f2beb23e58c8-v1", "taskId": "review-af06f2beb23e58c8", "version": "1", "name": "Snorkel-reviewed task review-af06f2beb23e58c8", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/141" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b4a08c2032adca40-v1", "taskId": "review-b4a08c2032adca40", "version": "1", "name": "Snorkel-reviewed task review-b4a08c2032adca40", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/235" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b634f4d2d12f541b-v1", "taskId": "review-b634f4d2d12f541b", "version": "1", "name": "Snorkel-reviewed task review-b634f4d2d12f541b", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/449" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b81f3182545d7f8d-v1", "taskId": "review-b81f3182545d7f8d", "version": "1", "name": "Snorkel-reviewed task review-b81f3182545d7f8d", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c07aced72086b242-v1", "taskId": "review-c07aced72086b242", "version": "1", "name": "Snorkel-reviewed task review-c07aced72086b242", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c79614abaf886585-v1", "taskId": "review-c79614abaf886585", "version": "1", "name": "Snorkel-reviewed task review-c79614abaf886585", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/444" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cc84a16e-v1", "taskId": "review-cc84a16e", "version": "1", "name": "Snorkel-reviewed task review-cc84a16e", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cd40c218884c6af4-v1", "taskId": "review-cd40c218884c6af4", "version": "1", "name": "Snorkel-reviewed task review-cd40c218884c6af4", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/334" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d09f8704-v1", "taskId": "review-d09f8704", "version": "1", "name": "Snorkel-reviewed task review-d09f8704", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d4878d8f40f68160-v1", "taskId": "review-d4878d8f40f68160", "version": "1", "name": "kelp-ring-auction", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d59d21187f586597-v1", "taskId": "review-d59d21187f586597", "version": "1", "name": "go-sounding", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d6107dbc892b275e-v1", "taskId": "review-d6107dbc892b275e", "version": "1", "name": "racket-prismfall-coordinator", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e1178bfc3fb7303f-v1", "taskId": "review-e1178bfc3fb7303f", "version": "1", "name": "modded-campaign-save-rebase", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e509c8a6-v1", "taskId": "review-e509c8a6", "version": "1", "name": "Snorkel-reviewed task review-e509c8a6", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e7c1e8d6b12e8bfe-v1", "taskId": "review-e7c1e8d6b12e8bfe", "version": "1", "name": "glass-harbor-time-loop", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ec253d16b45771f7-v1", "taskId": "review-ec253d16b45771f7", "version": "1", "name": "slagcrown-worker-placement-sim", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f0b0f7602ec3b29e-v1", "taskId": "review-f0b0f7602ec3b29e", "version": "1", "name": "lattice-arena-skirmish", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f4c7f05235f40e77-v1", "taskId": "review-f4c7f05235f40e77", "version": "1", "name": "Snorkel-reviewed task review-f4c7f05235f40e77", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 3, "p": 0.3333333333333333, "lo": 0.06149194472039615, "hi": 0.7923403991979523, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.06149194472039615, "macroHi": 0.7923403991979523, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 4, "p": 0, "lo": 0, "hi": 0.4898908364545973, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.4898908364545973, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/443" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f54412a3494babf1-v1", "taskId": "review-f54412a3494babf1", "version": "1", "name": "resonance-grid", "domain": "Games", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 3, "p": 0.6666666666666666, "lo": 0.20765960080204765, "hi": 0.9385080552796037, "tasks": 1, "macro": 0.6666666666666666, "macroLo": 0.20765960080204765, "macroHi": 0.9385080552796037, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-pathfinding-memory-optimize-grid-v1", "taskId": "review-slug-pathfinding-memory-optimize-grid", "version": "1", "name": "pathfinding-memory-optimize-grid", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0040_day_night_cycle_controller-v1", "taskId": "task_0040_day_night_cycle_controller", "version": "1", "name": "task_0040_day_night_cycle_controller", "domain": "Games", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-sonnet-4-6@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0042_pong_camera_spring_shake-v1", "taskId": "task_0042_pong_camera_spring_shake", "version": "1", "name": "task_0042_pong_camera_spring_shake", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/24" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0053_car_scene_assembly-v1", "taskId": "task_0053_car_scene_assembly", "version": "1", "name": "task_0053_car_scene_assembly", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0103_navigationagent2d_enemy_chase-v1", "taskId": "task_0103_navigationagent2d_enemy_chase", "version": "1", "name": "task_0103_navigationagent2d_enemy_chase", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0131_minimap_ui_complex-v1", "taskId": "task_0131_minimap_ui_complex", "version": "1", "name": "task_0131_minimap_ui_complex", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0132_minimap_marker_logic_complex-v1", "taskId": "task_0132_minimap_marker_logic_complex", "version": "1", "name": "task_0132_minimap_marker_logic_complex", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tile-gradient-particles-v1", "taskId": "tile-gradient-particles", "version": "1", "name": "tile-gradient-particles", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/28" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "weapon-effects-controller-v1", "taskId": "weapon-effects-controller", "version": "1", "name": "weapon-effects-controller", "domain": "Games", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/21" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "abalone-count-poisson-v1", "taskId": "abalone-count-poisson", "version": "1", "name": "abalone-count-poisson", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/abalone-count-poisson" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "acscoverage-cost-imbalance-v1", "taskId": "acscoverage-cost-imbalance", "version": "1", "name": "acscoverage-cost-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/acscoverage-cost-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "acsincome-shift-calibration-v1", "taskId": "acsincome-shift-calibration", "version": "1", "name": "acsincome-shift-calibration", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/acsincome-shift-calibration" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "adaptive-logged-propensity-reconstruction-v1", "taskId": "adaptive-logged-propensity-reconstruction", "version": "1", "name": "adaptive-logged-propensity-reconstruction", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/policyml/adaptive-logged-propensity-reconstruction" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "adult-calib-imbalance-v1", "taskId": "adult-calib-imbalance", "version": "1", "name": "adult-calib-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/adult-calib-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "adv-robust-pgd-v1", "taskId": "adv-robust-pgd", "version": "1", "name": "adv-robust-pgd", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/sf-data/tasks/research-engineering/adv-robust-pgd" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "adwin-nlms-streamflow-reset-v1", "taskId": "adwin-nlms-streamflow-reset", "version": "1", "name": "adwin-nlms-streamflow-reset", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/adwin-nlms-streamflow-reset" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "affairs-tobit-censored-regression-v1", "taskId": "affairs-tobit-censored-regression", "version": "1", "name": "affairs-tobit-censored-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/affairs-tobit-censored-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "aggregate-reward-sharp-bounds-policy-value-v1", "taskId": "aggregate-reward-sharp-bounds-policy-value", "version": "1", "name": "aggregate-reward-sharp-bounds-policy-value", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/policyml/aggregate-reward-sharp-bounds-policy-value" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ai4i-maintenance-leakage-imbalance-cost-v1", "taskId": "ai4i-maintenance-leakage-imbalance-cost", "version": "1", "name": "ai4i-maintenance-leakage-imbalance-cost", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/ai4i-maintenance-leakage-imbalance-cost" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "airfoil-self-noise-v1", "taskId": "airfoil-self-noise", "version": "1", "name": "airfoil-self-noise", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/95" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "airfrans-high-reynolds-drag-extrapolation-v1", "taskId": "airfrans-high-reynolds-drag-extrapolation", "version": "1", "name": "airfrans-high-reynolds-drag-extrapolation", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/96" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "airquality-benzene-sensor-leakage-v1", "taskId": "airquality-benzene-sensor-leakage", "version": "1", "name": "airquality-benzene-sensor-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/airquality-benzene-sensor-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "allbeauty-helpfulvotes-heteroscedastic-monotone-outlier-v1", "taskId": "allbeauty-helpfulvotes-heteroscedastic-monotone-outlier", "version": "1", "name": "allbeauty-helpfulvotes-heteroscedastic-monotone-outlier", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/allbeauty-helpfulvotes-heteroscedastic-monotone-outlier" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "allbeauty-skincare-cf-coldstartitem-complexsearch-metafusion-v1", "taskId": "allbeauty-skincare-cf-coldstartitem-complexsearch-metafusion", "version": "1", "name": "allbeauty-skincare-cf-coldstartitem-complexsearch-metafusion", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/allbeauty-skincare-cf-coldstartitem-complexsearch-metafusion" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "allstate-cat-encoding-regression-v1", "taskId": "allstate-cat-encoding-regression", "version": "1", "name": "allstate-cat-encoding-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/allstate-cat-encoding-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "appliances-helpfulvotes-heteroscedastic-interactions-selbias-v1", "taskId": "appliances-helpfulvotes-heteroscedastic-interactions-selbias", "version": "1", "name": "appliances-helpfulvotes-heteroscedastic-interactions-selbias", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/appliances-helpfulvotes-heteroscedastic-interactions-selbias" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "appliances-helpfulvotes-heteroscedastic-multiout-v1", "taskId": "appliances-helpfulvotes-heteroscedastic-multiout", "version": "1", "name": "appliances-helpfulvotes-heteroscedastic-multiout", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/appliances-helpfulvotes-heteroscedastic-multiout" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "aps-failure-cost-imbalance-v1", "taskId": "aps-failure-cost-imbalance", "version": "1", "name": "aps-failure-cost-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/aps-failure-cost-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "automotive-quality-reviewlen-text-v1", "taskId": "automotive-quality-reviewlen-text", "version": "1", "name": "automotive-quality-reviewlen-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/automotive-quality-reviewlen-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "automotive-reviewlen-text-vocabdrift-v1", "taskId": "automotive-reviewlen-text-vocabdrift", "version": "1", "name": "automotive-reviewlen-text-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/automotive-reviewlen-text-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "baby_products-infant-catencode-cost-metafusion-temporal-v1", "taskId": "baby_products-infant-catencode-cost-metafusion-temporal", "version": "1", "name": "baby_products-infant-catencode-cost-metafusion-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/baby_products-infant-catencode-cost-metafusion-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "babyproducts-baseline-calib-metmismatch-stratifiedeval-v1", "taskId": "babyproducts-baseline-calib-metmismatch-stratifiedeval", "version": "1", "name": "babyproducts-baseline-calib-metmismatch-stratifiedeval", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/babyproducts-baseline-calib-metmismatch-stratifiedeval" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "babyproducts-cf-coldstartitem-coldstartuser-groupcv-v1", "taskId": "babyproducts-cf-coldstartitem-coldstartuser-groupcv", "version": "1", "name": "babyproducts-cf-coldstartitem-coldstartuser-groupcv", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/babyproducts-cf-coldstartitem-coldstartuser-groupcv" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bank-additional-macro-duration-leakage-v1", "taskId": "bank-additional-macro-duration-leakage", "version": "1", "name": "bank-additional-macro-duration-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/bank-additional-macro-duration-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bank-marketing-calibration-imbalance-v1", "taskId": "bank-marketing-calibration-imbalance", "version": "1", "name": "bank-marketing-calibration-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/bank-marketing-calibration-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "beautypersonalcare-conceptdrift-imbalance-sentmismatch-xcat-v1", "taskId": "beautypersonalcare-conceptdrift-imbalance-sentmismatch-xcat", "version": "1", "name": "beautypersonalcare-conceptdrift-imbalance-sentmismatch-xcat", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/beautypersonalcare-conceptdrift-imbalance-sentmismatch-xcat" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "beijing-no2-copollutant-leakage-v1", "taskId": "beijing-no2-copollutant-leakage", "version": "1", "name": "beijing-no2-copollutant-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/beijing-no2-copollutant-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "beijing-pm25-quantile-coverage-v1", "taskId": "beijing-pm25-quantile-coverage", "version": "1", "name": "beijing-pm25-quantile-coverage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/beijing-pm25-quantile-coverage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bike-sharing-regression-timeaware-v1", "taskId": "bike-sharing-regression-timeaware", "version": "1", "name": "bike-sharing-regression-timeaware", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/bike-sharing-regression-timeaware" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bikeshare-daily-leakage-regression-v1", "taskId": "bikeshare-daily-leakage-regression", "version": "1", "name": "bikeshare-daily-leakage-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/bikeshare-daily-leakage-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bikeshare-demand-leakage-regression-v1", "taskId": "bikeshare-demand-leakage-regression", "version": "1", "name": "bikeshare-demand-leakage-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/bikeshare-demand-leakage-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "biochemists-zero-inflated-count-models-v1", "taskId": "biochemists-zero-inflated-count-models", "version": "1", "name": "biochemists-zero-inflated-count-models", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/biochemists-zero-inflated-count-models" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "birthwt-quantile-regression-v1", "taskId": "birthwt-quantile-regression", "version": "1", "name": "birthwt-quantile-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/birthwt-quantile-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "board-games-modeling-v1", "taskId": "board-games-modeling", "version": "1", "name": "board-games-modeling", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/board-games-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "books-conceptdrift-imbalance-sentmismatch-xcat-v1", "taskId": "books-conceptdrift-imbalance-sentmismatch-xcat", "version": "1", "name": "books-conceptdrift-imbalance-sentmismatch-xcat", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/books-conceptdrift-imbalance-sentmismatch-xcat" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "books-fiction-calib-imbalance-temporal-v1", "taskId": "books-fiction-calib-imbalance-temporal", "version": "1", "name": "books-fiction-calib-imbalance-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/books-fiction-calib-imbalance-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "books-nonfiction-sentmismatch-text-vocabdrift-v1", "taskId": "books-nonfiction-sentmismatch-text-vocabdrift", "version": "1", "name": "books-nonfiction-sentmismatch-text-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/books-nonfiction-sentmismatch-text-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-adult-calib-groupcv-svywt-xcat-v1", "taskId": "brfss-adult-calib-groupcv-svywt-xcat", "version": "1", "name": "brfss-adult-calib-groupcv-svywt-xcat", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/brfss-adult-calib-groupcv-svywt-xcat" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-adult-cost-groupcv-imbalance-svywt-v1", "taskId": "brfss-adult-cost-groupcv-imbalance-svywt", "version": "1", "name": "brfss-adult-cost-groupcv-imbalance-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/brfss-adult-cost-groupcv-imbalance-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-adult-imbalance-interactions-svywt-temporal-v1", "taskId": "brfss-adult-imbalance-interactions-svywt-temporal", "version": "1", "name": "brfss-adult-imbalance-interactions-svywt-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/brfss-adult-imbalance-interactions-svywt-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-agestd-groupcv-stratifiedeval-svywt-v1", "taskId": "brfss-agestd-groupcv-stratifiedeval-svywt", "version": "1", "name": "brfss-agestd-groupcv-stratifiedeval-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/brfss-agestd-groupcv-stratifiedeval-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-baseline-groupcv-stratifiedeval-svywt-v1", "taskId": "brfss-baseline-groupcv-stratifiedeval-svywt", "version": "1", "name": "brfss-baseline-groupcv-stratifiedeval-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/brfss-baseline-groupcv-stratifiedeval-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-calib-imbalance-cost-svywt-v1", "taskId": "brfss-calib-imbalance-cost-svywt", "version": "1", "name": "brfss-calib-imbalance-cost-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/brfss-calib-imbalance-cost-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-calib-imbalance-groupcv-svywt-v1", "taskId": "brfss-calib-imbalance-groupcv-svywt", "version": "1", "name": "brfss-calib-imbalance-groupcv-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/brfss-calib-imbalance-groupcv-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "brfss-nvss-baseline-xcat-groupcv-heteroscedastic-v1", "taskId": "brfss-nvss-baseline-xcat-groupcv-heteroscedastic", "version": "1", "name": "brfss-nvss-baseline-xcat-groupcv-heteroscedastic", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/brfss-nvss-baseline-xcat-groupcv-heteroscedastic" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cake-split-plot-mixed-model-v1", "taskId": "cake-split-plot-mixed-model", "version": "1", "name": "cake-split-plot-mixed-model", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/cake-split-plot-mixed-model" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "campaign-lift-winners-curse-v1", "taskId": "campaign-lift-winners-curse", "version": "1", "name": "campaign-lift-winners-curse", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/campaign-lift-winners-curse" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cdsandvinyl-conceptdrift-selbias-temporal-xcat-v1", "taskId": "cdsandvinyl-conceptdrift-selbias-temporal-xcat", "version": "1", "name": "cdsandvinyl-conceptdrift-selbias-temporal-xcat", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/cdsandvinyl-conceptdrift-selbias-temporal-xcat" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cdsandvinyl-imbalance-sentmismatch-text-vocabdrift-v1", "taskId": "cdsandvinyl-imbalance-sentmismatch-text-vocabdrift", "version": "1", "name": "cdsandvinyl-imbalance-sentmismatch-text-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/cdsandvinyl-imbalance-sentmismatch-text-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cdsvinyl-conceptdrift-xcat-selbias-vocabdrift-v1", "taskId": "cdsvinyl-conceptdrift-xcat-selbias-vocabdrift", "version": "1", "name": "cdsvinyl-conceptdrift-xcat-selbias-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/cdsvinyl-conceptdrift-xcat-selbias-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cellphonesandaccessories-imbalance-quality-text-vocabdrift-v1", "taskId": "cellphonesandaccessories-imbalance-quality-text-vocabdrift", "version": "1", "name": "cellphonesandaccessories-imbalance-quality-text-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/cellphonesandaccessories-imbalance-quality-text-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "clinical-trial-survival-censoring-v1", "taskId": "clinical-trial-survival-censoring", "version": "1", "name": "clinical-trial-survival-censoring", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/clinical-trial-survival-censoring" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "clothing-imbalance-leakage-metafusion-text-v1", "taskId": "clothing-imbalance-leakage-metafusion-text", "version": "1", "name": "clothing-imbalance-leakage-metafusion-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/clothing-imbalance-leakage-metafusion-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "coarsened-action-assignment-bounds-policy-value-v1", "taskId": "coarsened-action-assignment-bounds-policy-value", "version": "1", "name": "coarsened-action-assignment-bounds-policy-value", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/policyml/coarsened-action-assignment-bounds-policy-value" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "coil2000-imbalance-metric-v1", "taskId": "coil2000-imbalance-metric", "version": "1", "name": "coil2000-imbalance-metric", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/coil2000-imbalance-metric" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "colleges-imputation-regression-v1", "taskId": "colleges-imputation-regression", "version": "1", "name": "colleges-imputation-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/colleges-imputation-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "concrete-strength-isotonic-pava-blocks-v1", "taskId": "concrete-strength-isotonic-pava-blocks", "version": "1", "name": "concrete-strength-isotonic-pava-blocks", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/concrete-strength-isotonic-pava-blocks" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "countmin-dyadic-range-sketch-v1", "taskId": "countmin-dyadic-range-sketch", "version": "1", "name": "countmin-dyadic-range-sketch", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/countmin-dyadic-range-sketch" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cps1988-quantile-wage-v1", "taskId": "cps1988-quantile-wage", "version": "1", "name": "cps1988-quantile-wage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/cps1988-quantile-wage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "credit-default-cost-imbalance-v1", "taskId": "credit-default-cost-imbalance", "version": "1", "name": "credit-default-cost-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/credit-default-cost-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "credit-risk-china-imbalance-missing-v1", "taskId": "credit-risk-china-imbalance-missing", "version": "1", "name": "credit-risk-china-imbalance-missing", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/credit-risk-china-imbalance-missing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "creditcard-calibration-imbalance-v1", "taskId": "creditcard-calibration-imbalance", "version": "1", "name": "creditcard-calibration-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/creditcard-calibration-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "critical-care-survival-censoring-calibration-v1", "taskId": "critical-care-survival-censoring-calibration", "version": "1", "name": "critical-care-survival-censoring-calibration", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/critical-care-survival-censoring-calibration" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ctgov-conceptdrift-multilabel-text-vocabdrift-v1", "taskId": "ctgov-conceptdrift-multilabel-text-vocabdrift", "version": "1", "name": "ctgov-conceptdrift-multilabel-text-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/ctgov-conceptdrift-multilabel-text-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ctgov-rare-disease-enrollment-text-temporal-v1", "taskId": "ctgov-rare-disease-enrollment-text-temporal", "version": "1", "name": "ctgov-rare-disease-enrollment-text-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/ctgov-rare-disease-enrollment-text-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ctgov-results-reporting-calib-imbalance-text-v1", "taskId": "ctgov-results-reporting-calib-imbalance-text", "version": "1", "name": "ctgov-results-reporting-calib-imbalance-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/ctgov-results-reporting-calib-imbalance-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ctgov-trial-duration-missing-outlier-temporal-v1", "taskId": "ctgov-trial-duration-missing-outlier-temporal", "version": "1", "name": "ctgov-trial-duration-missing-outlier-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/ctgov-trial-duration-missing-outlier-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ctgov-trial-termination-calib-imbalance-text-v1", "taskId": "ctgov-trial-termination-calib-imbalance-text", "version": "1", "name": "ctgov-trial-termination-calib-imbalance-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/ctgov-trial-termination-calib-imbalance-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "delayed-censored-reward-policy-value-v1", "taskId": "delayed-censored-reward-policy-value", "version": "1", "name": "delayed-censored-reward-policy-value", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/policyml/delayed-censored-reward-policy-value" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "design-based-finite-population-policy-value-v1", "taskId": "design-based-finite-population-policy-value", "version": "1", "name": "design-based-finite-population-policy-value", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/policyml/design-based-finite-population-policy-value" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "diabetes-130-cv-strategy-class-imbalance-v1", "taskId": "diabetes-130-cv-strategy-class-imbalance", "version": "1", "name": "diabetes-130-cv-strategy-class-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/diabetes-130-cv-strategy-class-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dictionary-match-predictor-v1", "taskId": "dictionary-match-predictor", "version": "1", "name": "dictionary-match-predictor", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/dictionary-match-predictor" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "digitalmusic-conceptdrift-imbalance-selbias-xcat-v1", "taskId": "digitalmusic-conceptdrift-imbalance-selbias-xcat", "version": "1", "name": "digitalmusic-conceptdrift-imbalance-selbias-xcat", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/digitalmusic-conceptdrift-imbalance-selbias-xcat" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "digitalmusic-jazz-conceptdrift-helpfulvotes-monotone-quality-xcat-v1", "taskId": "digitalmusic-jazz-conceptdrift-helpfulvotes-monotone-quality-xcat", "version": "1", "name": "digitalmusic-jazz-conceptdrift-helpfulvotes-monotone-quality-xcat", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/digitalmusic-jazz-conceptdrift-helpfulvotes-monotone-quality-xcat" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "drug-directory-imputation-multiclass-v1", "taskId": "drug-directory-imputation-multiclass", "version": "1", "name": "drug-directory-imputation-multiclass", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/drug-directory-imputation-multiclass" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "electronics-headphones-calib-leakage-metafusion-v1", "taskId": "electronics-headphones-calib-leakage-metafusion", "version": "1", "name": "electronics-headphones-calib-leakage-metafusion", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/electronics-headphones-calib-leakage-metafusion" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "electronics-helpfulvotes-heteroscedastic-monotone-outlier-v1", "taskId": "electronics-helpfulvotes-heteroscedastic-monotone-outlier", "version": "1", "name": "electronics-helpfulvotes-heteroscedastic-monotone-outlier", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/electronics-helpfulvotes-heteroscedastic-monotone-outlier" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "electronics-imbalance-metafusion-quality-v1", "taskId": "electronics-imbalance-metafusion-quality", "version": "1", "name": "electronics-imbalance-metafusion-quality", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/electronics-imbalance-metafusion-quality" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "electronics-smarthome-calib-imbalance-quality-temporal-v1", "taskId": "electronics-smarthome-calib-imbalance-quality-temporal", "version": "1", "name": "electronics-smarthome-calib-imbalance-quality-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/electronics-smarthome-calib-imbalance-quality-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "empluk-arellano-bond-gmm-v1", "taskId": "empluk-arellano-bond-gmm", "version": "1", "name": "empluk-arellano-bond-gmm", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/empluk-arellano-bond-gmm" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "engel-food-quantile-regression-v1", "taskId": "engel-food-quantile-regression", "version": "1", "name": "engel-food-quantile-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/rdatasets/engel-food-quantile-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fars-leakage-multiclass-v1", "taskId": "fars-leakage-multiclass", "version": "1", "name": "fars-leakage-multiclass", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/fars-leakage-multiclass" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "financial-risk-missing-multiclass-v1", "taskId": "financial-risk-missing-multiclass", "version": "1", "name": "financial-risk-missing-multiclass", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/financial-risk-missing-multiclass" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "foodexpenditure-beta-dispersion-v1", "taskId": "foodexpenditure-beta-dispersion", "version": "1", "name": "foodexpenditure-beta-dispersion", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/foodexpenditure-beta-dispersion" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "forest-fires-skewed-regression-v1", "taskId": "forest-fires-skewed-regression", "version": "1", "name": "forest-fires-skewed-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/forest-fires-skewed-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ftrl-proximal-logistic-exceedance-v1", "taskId": "ftrl-proximal-logistic-exceedance", "version": "1", "name": "ftrl-proximal-logistic-exceedance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/ftrl-proximal-logistic-exceedance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gas-drift-multiclass-hyperparam-v1", "taskId": "gas-drift-multiclass-hyperparam", "version": "1", "name": "gas-drift-multiclass-hyperparam", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/gas-drift-multiclass-hyperparam" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "giftcards-calib-cost-imbalance-monotone-temporal-leakage-v1", "taskId": "giftcards-calib-cost-imbalance-monotone-temporal-leakage", "version": "1", "name": "giftcards-calib-cost-imbalance-monotone-temporal-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/giftcards-calib-cost-imbalance-monotone-temporal-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gk-quantile-sketch-streamflow-v1", "taskId": "gk-quantile-sketch-streamflow", "version": "1", "name": "gk-quantile-sketch-streamflow", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/gk-quantile-sketch-streamflow" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "global-crop-yields-missing-data-v1", "taskId": "global-crop-yields-missing-data", "version": "1", "name": "global-crop-yields-missing-data", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/global-crop-yields-missing-data" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "handmade-coldstartitem-complexsearch-embedretrieve-groupcv-v1", "taskId": "handmade-coldstartitem-complexsearch-embedretrieve-groupcv", "version": "1", "name": "handmade-coldstartitem-complexsearch-embedretrieve-groupcv", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/handmade-coldstartitem-complexsearch-embedretrieve-groupcv" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hausman-taylor-time-invariant-endogenous-effect-v1", "taskId": "hausman-taylor-time-invariant-endogenous-effect", "version": "1", "name": "hausman-taylor-time-invariant-endogenous-effect", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/causal/hausman-taylor-time-invariant-endogenous-effect" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "health-baseline-helpfulvotes-monotone-v1", "taskId": "health-baseline-helpfulvotes-monotone", "version": "1", "name": "health-baseline-helpfulvotes-monotone", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/health-baseline-helpfulvotes-monotone" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "healthandhousehold-helpfulvotes-heteroscedastic-missing-monotone-v1", "taskId": "healthandhousehold-helpfulvotes-heteroscedastic-missing-monotone", "version": "1", "name": "healthandhousehold-helpfulvotes-heteroscedastic-missing-monotone", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/healthandhousehold-helpfulvotes-heteroscedastic-missing-monotone" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "healthandhousehold-helpfulvotes-interactions-monotone-selbias-v1", "taskId": "healthandhousehold-helpfulvotes-interactions-monotone-selbias", "version": "1", "name": "healthandhousehold-helpfulvotes-interactions-monotone-selbias", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/healthandhousehold-helpfulvotes-interactions-monotone-selbias" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "himalayan-expeditions-modeling-v1", "taskId": "himalayan-expeditions-modeling", "version": "1", "name": "himalayan-expeditions-modeling", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/himalayan-expeditions-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hitters-penalized-regression-v1", "taskId": "hitters-penalized-regression", "version": "1", "name": "hitters-penalized-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/hitters-penalized-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hmda-fairness-leakage-v1", "taskId": "hmda-fairness-leakage", "version": "1", "name": "hmda-fairness-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/hmda-fairness-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hoeffding-split-tree-tempexceed-v1", "taskId": "hoeffding-split-tree-tempexceed", "version": "1", "name": "hoeffding-split-tree-tempexceed", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/hoeffding-split-tree-tempexceed" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hotel-bookings-modeling-v1", "taskId": "hotel-bookings-modeling", "version": "1", "name": "hotel-bookings-modeling", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/hotel-bookings-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "housing-ordinal-proportional-odds-v1", "taskId": "housing-ordinal-proportional-odds", "version": "1", "name": "housing-ordinal-proportional-odds", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/housing-ordinal-proportional-odds" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hysteresis-latch-anomaly-v1", "taskId": "hysteresis-latch-anomaly", "version": "1", "name": "hysteresis-latch-anomaly", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/hysteresis-latch-anomaly" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "immortal-time-bias-tvc-survival-hr-v1", "taskId": "immortal-time-bias-tvc-survival-hr", "version": "1", "name": "immortal-time-bias-tvc-survival-hr", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/causal/immortal-time-bias-tvc-survival-hr" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "incident-mgmt-leakage-timeaware-v1", "taskId": "incident-mgmt-leakage-timeaware", "version": "1", "name": "incident-mgmt-leakage-timeaware", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/incident-mgmt-leakage-timeaware" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "industrial-imbalance-metafusion-reviewlen-text-v1", "taskId": "industrial-imbalance-metafusion-reviewlen-text", "version": "1", "name": "industrial-imbalance-metafusion-reviewlen-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/industrial-imbalance-metafusion-reviewlen-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "industrialscientific-labequipment-catencode-imbalance-metmismatch-quality-smallsample-v1", "taskId": "industrialscientific-labequipment-catencode-imbalance-metmismatch-quality-smallsample", "version": "1", "name": "industrialscientific-labequipment-catencode-imbalance-metmismatch-quality-smallsample", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/industrialscientific-labequipment-catencode-imbalance-metmismatch-quality-smallsample" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "internet-firewall-catencode-metric-v1", "taskId": "internet-firewall-catencode-metric", "version": "1", "name": "internet-firewall-catencode-metric", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/internet-firewall-catencode-metric" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ipums-la-fairness-leakage-v1", "taskId": "ipums-la-fairness-leakage", "version": "1", "name": "ipums-la-fairness-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/ipums-la-fairness-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jobchange-encoding-imbalance-metric-v1", "taskId": "jobchange-encoding-imbalance-metric", "version": "1", "name": "jobchange-encoding-imbalance-metric", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/jobchange-encoding-imbalance-metric" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kddcup09-appetency-calibration-imbalance-v1", "taskId": "kddcup09-appetency-calibration-imbalance", "version": "1", "name": "kddcup09-appetency-calibration-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/kddcup09-appetency-calibration-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kddcup09-cost-metric-v1", "taskId": "kddcup09-cost-metric", "version": "1", "name": "kddcup09-cost-metric", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/kddcup09-cost-metric" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kddcup99-imbalance-multiclass-v1", "taskId": "kddcup99-imbalance-multiclass", "version": "1", "name": "kddcup99-imbalance-multiclass", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/kddcup99-imbalance-multiclass" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kick-carvana-cost-sensitive-imbalance-v1", "taskId": "kick-carvana-cost-sensitive-imbalance", "version": "1", "name": "kick-carvana-cost-sensitive-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/kick-carvana-cost-sensitive-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kindle-imbalance-sentmismatch-text-vocabdrift-v1", "taskId": "kindle-imbalance-sentmismatch-text-vocabdrift", "version": "1", "name": "kindle-imbalance-sentmismatch-text-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/kindle-imbalance-sentmismatch-text-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kindle-mystery-cost-heteroscedastic-multiout-outlier-text-v1", "taskId": "kindle-mystery-cost-heteroscedastic-multiout-outlier-text", "version": "1", "name": "kindle-mystery-cost-heteroscedastic-multiout-outlier-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/kindle-mystery-cost-heteroscedastic-multiout-outlier-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kindle_store-conceptdrift-xcat-selbias-vocabdrift-v1", "taskId": "kindle_store-conceptdrift-xcat-selbias-vocabdrift", "version": "1", "name": "kindle_store-conceptdrift-xcat-selbias-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/kindle_store-conceptdrift-xcat-selbias-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kindlestore-imbalance-selbias-sentmismatch-xcat-v1", "taskId": "kindlestore-imbalance-selbias-sentmismatch-xcat", "version": "1", "name": "kindlestore-imbalance-selbias-sentmismatch-xcat", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/kindlestore-imbalance-selbias-sentmismatch-xcat" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "klein-macro-system-estimation-v1", "taskId": "klein-macro-system-estimation", "version": "1", "name": "klein-macro-system-estimation", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/klein-macro-system-estimation" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ks-equation-1d-forecast-v1", "taskId": "ks-equation-1d-forecast", "version": "1", "name": "ks-equation-1d-forecast", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/100" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "label-budget-active-learner-v1", "taskId": "label-budget-active-learner", "version": "1", "name": "label-budget-active-learner", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/label-budget-active-learner" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lagged-difference-exceedance-v1", "taskId": "lagged-difference-exceedance", "version": "1", "name": "lagged-difference-exceedance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/lagged-difference-exceedance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lalonde-doubly-robust-ate-v1", "taskId": "lalonde-doubly-robust-ate", "version": "1", "name": "lalonde-doubly-robust-ate", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/lalonde-doubly-robust-ate" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lemur-data-modeling-v1", "taskId": "lemur-data-modeling", "version": "1", "name": "lemur-data-modeling", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/lemur-data-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "link-prediction-structural-features-ranking-v1", "taskId": "link-prediction-structural-features-ranking", "version": "1", "name": "link-prediction-structural-features-ranking", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/graphml/link-prediction-structural-features-ranking" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "loan-default-cost-imbalance-v1", "taskId": "loan-default-cost-imbalance", "version": "1", "name": "loan-default-cost-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/loan-default-cost-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "logistic-pagehinkley-exceedance-v1", "taskId": "logistic-pagehinkley-exceedance", "version": "1", "name": "logistic-pagehinkley-exceedance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/logistic-pagehinkley-exceedance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lsat6-irt-latent-trait-v1", "taskId": "lsat6-irt-latent-trait", "version": "1", "name": "lsat6-irt-latent-trait", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/lsat6-irt-latent-trait" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lung-cancer-ph-diagnostics-v1", "taskId": "lung-cancer-ph-diagnostics", "version": "1", "name": "lung-cancer-ph-diagnostics", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/lung-cancer-ph-diagnostics" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "magazinesubscriptions-conceptdrift-leakage-multilabel-selbias-vocabdrift-v1", "taskId": "magazinesubscriptions-conceptdrift-leakage-multilabel-selbias-vocabdrift", "version": "1", "name": "magazinesubscriptions-conceptdrift-leakage-multilabel-selbias-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/magazinesubscriptions-conceptdrift-leakage-multilabel-selbias-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "maintenance-therapy-competing-risks-cif-v1", "taskId": "maintenance-therapy-competing-risks-cif", "version": "1", "name": "maintenance-therapy-competing-risks-cif", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/causal/maintenance-therapy-competing-risks-cif" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mbias-pretreatment-collider-ate-v1", "taskId": "mbias-pretreatment-collider-ate", "version": "1", "name": "mbias-pretreatment-collider-ate", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/causal/mbias-pretreatment-collider-ate" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "meta-analysis-random-effects-hksj-v1", "taskId": "meta-analysis-random-effects-hksj", "version": "1", "name": "meta-analysis-random-effects-hksj", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/meta-analysis-random-effects-hksj" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "modular-interval-position-v1", "taskId": "modular-interval-position", "version": "1", "name": "modular-interval-position", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/modular-interval-position" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "movies-drama-imbalance-sentmismatch-text-v1", "taskId": "movies-drama-imbalance-sentmismatch-text", "version": "1", "name": "movies-drama-imbalance-sentmismatch-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/movies-drama-imbalance-sentmismatch-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "movies_and_tv-drama-calib-catencode-quality-leakage-v1", "taskId": "movies_and_tv-drama-calib-catencode-quality-leakage", "version": "1", "name": "movies_and_tv-drama-calib-catencode-quality-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/movies_and_tv-drama-calib-catencode-quality-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "multi-horizon-trend-signal-portfolio-sort-v1", "taskId": "multi-horizon-trend-signal-portfolio-sort", "version": "1", "name": "multi-horizon-trend-signal-portfolio-sort", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/multi-horizon-trend-signal-portfolio-sort" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "multiplicative-bar-exceedance-v1", "taskId": "multiplicative-bar-exceedance", "version": "1", "name": "multiplicative-bar-exceedance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/multiplicative-bar-exceedance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "musicalinstruments-catencode-leakage-missing-smallsample-v1", "taskId": "musicalinstruments-catencode-leakage-missing-smallsample", "version": "1", "name": "musicalinstruments-catencode-leakage-missing-smallsample", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/musicalinstruments-catencode-leakage-missing-smallsample" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "myocardial-infarction-metric-multiclass-v1", "taskId": "myocardial-infarction-metric-multiclass", "version": "1", "name": "myocardial-infarction-metric-multiclass", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/myocardial-infarction-metric-multiclass" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhanes-adult-calib-conceptdrift-imbalance-svywt-v1", "taskId": "nhanes-adult-calib-conceptdrift-imbalance-svywt", "version": "1", "name": "nhanes-adult-calib-conceptdrift-imbalance-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhanes-adult-calib-conceptdrift-imbalance-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhanes-adult-calib-cost-icdleak-svywt-v1", "taskId": "nhanes-adult-calib-cost-icdleak-svywt", "version": "1", "name": "nhanes-adult-calib-cost-icdleak-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhanes-adult-calib-cost-icdleak-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhanes-adult-calib-imbalance-svywt-leakage-v1", "taskId": "nhanes-adult-calib-imbalance-svywt-leakage", "version": "1", "name": "nhanes-adult-calib-imbalance-svywt-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhanes-adult-calib-imbalance-svywt-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhanes-adult-conceptdrift-imbalance-pooling-svywt-v1", "taskId": "nhanes-adult-conceptdrift-imbalance-pooling-svywt", "version": "1", "name": "nhanes-adult-conceptdrift-imbalance-pooling-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/nhanes-adult-conceptdrift-imbalance-pooling-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhanes-brfss-adult-conceptdrift-xcat-selbias-svywt-v1", "taskId": "nhanes-brfss-adult-conceptdrift-xcat-selbias-svywt", "version": "1", "name": "nhanes-brfss-adult-conceptdrift-xcat-selbias-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhanes-brfss-adult-conceptdrift-xcat-selbias-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhanes-elderly-heteroscedastic-outlier-svywt-v1", "taskId": "nhanes-elderly-heteroscedastic-outlier-svywt", "version": "1", "name": "nhanes-elderly-heteroscedastic-outlier-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhanes-elderly-heteroscedastic-outlier-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhanes-nvss-agestd-heteroscedastic-pooling-svywt-v1", "taskId": "nhanes-nvss-agestd-heteroscedastic-pooling-svywt", "version": "1", "name": "nhanes-nvss-agestd-heteroscedastic-pooling-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhanes-nvss-agestd-heteroscedastic-pooling-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhis-adult-calib-catencode-imbalance-svywt-v1", "taskId": "nhis-adult-calib-catencode-imbalance-svywt", "version": "1", "name": "nhis-adult-calib-catencode-imbalance-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhis-adult-calib-catencode-imbalance-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhis-adult-calib-pooling-selbias-svywt-v1", "taskId": "nhis-adult-calib-pooling-selbias-svywt", "version": "1", "name": "nhis-adult-calib-pooling-selbias-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhis-adult-calib-pooling-selbias-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhis-adult-catencode-missing-svywt-temporal-v1", "taskId": "nhis-adult-catencode-missing-svywt-temporal", "version": "1", "name": "nhis-adult-catencode-missing-svywt-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhis-adult-catencode-missing-svywt-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhis-adult-conceptdrift-pooling-svywt-v1", "taskId": "nhis-adult-conceptdrift-pooling-svywt", "version": "1", "name": "nhis-adult-conceptdrift-pooling-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhis-adult-conceptdrift-pooling-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhis-adult-conceptdrift-selbias-stratifiedeval-svywt-v1", "taskId": "nhis-adult-conceptdrift-selbias-stratifiedeval-svywt", "version": "1", "name": "nhis-adult-conceptdrift-selbias-stratifiedeval-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/nhis-adult-conceptdrift-selbias-stratifiedeval-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhis-adult-heteroscedastic-multiout-svywt-temporal-v1", "taskId": "nhis-adult-heteroscedastic-multiout-svywt-temporal", "version": "1", "name": "nhis-adult-heteroscedastic-multiout-svywt-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhis-adult-heteroscedastic-multiout-svywt-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nhis-elderly-agestd-baseline-stratifiedeval-svywt-v1", "taskId": "nhis-elderly-agestd-baseline-stratifiedeval-svywt", "version": "1", "name": "nhis-elderly-agestd-baseline-stratifiedeval-svywt", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nhis-elderly-agestd-baseline-stratifiedeval-svywt" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nvss-calib-conceptdrift-cost-imbalance-leakage-v1", "taskId": "nvss-calib-conceptdrift-cost-imbalance-leakage", "version": "1", "name": "nvss-calib-conceptdrift-cost-imbalance-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nvss-calib-conceptdrift-cost-imbalance-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nvss-conceptdrift-heteroscedastic-monotone-multiout-v1", "taskId": "nvss-conceptdrift-heteroscedastic-monotone-multiout", "version": "1", "name": "nvss-conceptdrift-heteroscedastic-monotone-multiout", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nvss-conceptdrift-heteroscedastic-monotone-multiout" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nvss-conceptdrift-heteroscedastic-multiout-selbias-v1", "taskId": "nvss-conceptdrift-heteroscedastic-multiout-selbias", "version": "1", "name": "nvss-conceptdrift-heteroscedastic-multiout-selbias", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nvss-conceptdrift-heteroscedastic-multiout-selbias" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nvss-heteroscedastic-helpfulvotes-outlier-baseline-v1", "taskId": "nvss-heteroscedastic-helpfulvotes-outlier-baseline", "version": "1", "name": "nvss-heteroscedastic-helpfulvotes-outlier-baseline", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/nvss-heteroscedastic-helpfulvotes-outlier-baseline" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nvss-heteroscedastic-monotone-outlier-smallsample-v1", "taskId": "nvss-heteroscedastic-monotone-outlier-smallsample", "version": "1", "name": "nvss-heteroscedastic-monotone-outlier-smallsample", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/nvss-heteroscedastic-monotone-outlier-smallsample" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nwtco-case-cohort-survival-v1", "taskId": "nwtco-case-cohort-survival", "version": "1", "name": "nwtco-case-cohort-survival", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/nwtco-case-cohort-survival" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "office-calib-catencode-cost-leakage-v1", "taskId": "office-calib-catencode-cost-leakage", "version": "1", "name": "office-calib-catencode-cost-leakage", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/office-calib-catencode-cost-leakage" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "office_products-baseline-groupcv-metmismatch-stratifiedeval-repro-v1", "taskId": "office_products-baseline-groupcv-metmismatch-stratifiedeval-repro", "version": "1", "name": "office_products-baseline-groupcv-metmismatch-stratifiedeval-repro", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/office_products-baseline-groupcv-metmismatch-stratifiedeval-repro" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "offline-pessimistic-policy-optimization-v1", "taskId": "offline-pessimistic-policy-optimization", "version": "1", "name": "offline-pessimistic-policy-optimization", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/policyml/offline-pessimistic-policy-optimization" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ohio-wheezing-gee-v1", "taskId": "ohio-wheezing-gee", "version": "1", "name": "ohio-wheezing-gee", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/ohio-wheezing-gee" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "online-kmeans-splitmerge-v1", "taskId": "online-kmeans-splitmerge", "version": "1", "name": "online-kmeans-splitmerge", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/online-kmeans-splitmerge" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "openfda-conceptdrift-cost-quality-vocabdrift-v1", "taskId": "openfda-conceptdrift-cost-quality-vocabdrift", "version": "1", "name": "openfda-conceptdrift-cost-quality-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/openfda-conceptdrift-cost-quality-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "openfda-drug-route-calib-imbalance-temporal-v1", "taskId": "openfda-drug-route-calib-imbalance-temporal", "version": "1", "name": "openfda-drug-route-calib-imbalance-temporal", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/openfda-drug-route-calib-imbalance-temporal" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "openfda-imbalance-multilabel-text-v1", "taskId": "openfda-imbalance-multilabel-text", "version": "1", "name": "openfda-imbalance-multilabel-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/openfda-imbalance-multilabel-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "openfda-metafusion-quality-text-vocabdrift-v1", "taskId": "openfda-metafusion-quality-text-vocabdrift", "version": "1", "name": "openfda-metafusion-quality-text-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/openfda-metafusion-quality-text-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "patio-gardentools-catencode-interactions-leakage-missing-v1", "taskId": "patio-gardentools-catencode-interactions-leakage-missing", "version": "1", "name": "patio-gardentools-catencode-interactions-leakage-missing", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/patio-gardentools-catencode-interactions-leakage-missing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "patiolawnandgarden-catencode-imbalance-interactions-metafusion-v1", "taskId": "patiolawnandgarden-catencode-imbalance-interactions-metafusion", "version": "1", "name": "patiolawnandgarden-catencode-imbalance-interactions-metafusion", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/patiolawnandgarden-catencode-imbalance-interactions-metafusion" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pbcseq-timevarying-survival-v1", "taskId": "pbcseq-timevarying-survival", "version": "1", "name": "pbcseq-timevarying-survival", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/pbcseq-timevarying-survival" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pdb-occupancy-residue-flexibility-regression-v1", "taskId": "pdb-occupancy-residue-flexibility-regression", "version": "1", "name": "pdb-occupancy-residue-flexibility-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/graphml/pdb-occupancy-residue-flexibility-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "periodic-threshold-alternation-v1", "taskId": "periodic-threshold-alternation", "version": "1", "name": "periodic-threshold-alternation", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/periodic-threshold-alternation" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pet_supplies-dog-helpfulvotes-heteroscedastic-outlier-v1", "taskId": "pet_supplies-dog-helpfulvotes-heteroscedastic-outlier", "version": "1", "name": "pet_supplies-dog-helpfulvotes-heteroscedastic-outlier", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/pet_supplies-dog-helpfulvotes-heteroscedastic-outlier" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "petsupplies-cf-coldstartitem-coldstartuser-groupcv-v1", "taskId": "petsupplies-cf-coldstartitem-coldstartuser-groupcv", "version": "1", "name": "petsupplies-cf-coldstartitem-coldstartuser-groupcv", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/petsupplies-cf-coldstartitem-coldstartuser-groupcv" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "petsupplies-coldstartuser-groupcv-metafusion-seqrec-v1", "taskId": "petsupplies-coldstartuser-groupcv-metafusion-seqrec", "version": "1", "name": "petsupplies-coldstartuser-groupcv-metafusion-seqrec", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/petsupplies-coldstartuser-groupcv-metafusion-seqrec" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pretrain-60m-from-scratch-v1", "taskId": "pretrain-60m-from-scratch", "version": "1", "name": "pretrain-60m-from-scratch", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/570" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "price-delay-cross-section-portfolio-sort-v1", "taskId": "price-delay-cross-section-portfolio-sort", "version": "1", "name": "price-delay-cross-section-portfolio-sort", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/price-delay-cross-section-portfolio-sort" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "psid1976-heckman-selection-v1", "taskId": "psid1976-heckman-selection", "version": "1", "name": "psid1976-heckman-selection", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/psid1976-heckman-selection" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "random-feature-ridge-market-timing-v1", "taskId": "random-feature-ridge-market-timing", "version": "1", "name": "random-feature-ridge-market-timing", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/random-feature-ridge-market-timing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rd-honest-ci-bounded-curvature-v1", "taskId": "rd-honest-ci-bounded-curvature", "version": "1", "name": "rd-honest-ci-bounded-curvature", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/causal/rd-honest-ci-bounded-curvature" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "residual-momentum-ff3-decile-sort-v1", "taskId": "residual-momentum-ff3-decile-sort", "version": "1", "name": "residual-momentum-ff3-decile-sort", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/residual-momentum-ff3-decile-sort" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "retinopathy-clustered-frailty-survival-v1", "taskId": "retinopathy-clustered-frailty-survival", "version": "1", "name": "retinopathy-clustered-frailty-survival", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/retinopathy-clustered-frailty-survival" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0c5e8e5ec5ac3535-v1", "taskId": "review-0c5e8e5ec5ac3535", "version": "1", "name": "Snorkel-reviewed task review-0c5e8e5ec5ac3535", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/253" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0c7726a53cc27b17-v1", "taskId": "review-0c7726a53cc27b17", "version": "1", "name": "stadium-egress-trace-bundle-0230", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0e19a00d8cbf23d5-v1", "taskId": "review-0e19a00d8cbf23d5", "version": "1", "name": "Snorkel-reviewed task review-0e19a00d8cbf23d5", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1384f63a27faff91-v1", "taskId": "review-1384f63a27faff91", "version": "1", "name": "hpo-log-search-space-infer", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1, needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1da75e480c20ded0-v1", "taskId": "review-1da75e480c20ded0", "version": "1", "name": "qualify-kotlin-h2-conformal-release-submit-reviewerfix", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1f02901c7966b335-v1", "taskId": "review-1f02901c7966b335", "version": "1", "name": ".", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/254" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-28ac19f6413c6146-v1", "taskId": "review-28ac19f6413c6146", "version": "1", "name": "Snorkel-reviewed task review-28ac19f6413c6146", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/174" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2bea92e628bdc3fa-v1", "taskId": "review-2bea92e628bdc3fa", "version": "1", "name": "coral-ranking-feasibility-certificate", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2c44ece630a3107d-v1", "taskId": "review-2c44ece630a3107d", "version": "1", "name": "go-federated-secure-aggregation-repair", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2cde66b5007c9853-v1", "taskId": "review-2cde66b5007c9853", "version": "1", "name": "kortexdl-order-aware-transformer-ood", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2d1b55ffe31e8107-v1", "taskId": "review-2d1b55ffe31e8107", "version": "1", "name": "kmeanspp-seeding-draw-ledger", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-32a79723e96a1957-v1", "taskId": "review-32a79723e96a1957", "version": "1", "name": "train-serve-skew-feature-hash", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-336683b7d1fded67-v1", "taskId": "review-336683b7d1fded67", "version": "1", "name": "Snorkel-reviewed task review-336683b7d1fded67", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/288" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-36f0b268ac54f4f5-v1", "taskId": "review-36f0b268ac54f4f5", "version": "1", "name": "graphsage-neighbor-sampler", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3703682fe32fb6c9-v1", "taskId": "review-3703682fe32fb6c9", "version": "1", "name": "compressor-fleet-onset-classifier", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 7, "n": 10, "p": 0.7, "lo": 0.39677814746114537, "hi": 0.8922087325936989, "tasks": 1, "macro": 0.7, "macroLo": 0.39677814746114537, "macroHi": 0.8922087325936989, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3e534ca69a994d4d-v1", "taskId": "review-3e534ca69a994d4d", "version": "1", "name": "insect-wing-query-curator", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-3fdd908cc0b6db74-v1", "taskId": "review-3fdd908cc0b6db74", "version": "1", "name": "ocaml-hmm-checkpoint", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-41080ae7e090c2e0-v1", "taskId": "review-41080ae7e090c2e0", "version": "1", "name": "rlhf-pairwise-seed-build", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 7, "n": 15, "p": 0.4666666666666667, "lo": 0.24809535365737387, "hi": 0.698830199745016, "tasks": 1, "macro": 0.4666666666666667, "macroLo": 0.24809535365737387, "macroHi": 0.698830199745016, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-478e23e8fb354efa-v1", "taskId": "review-478e23e8fb354efa", "version": "1", "name": "marrowfen-foldfit", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-48dfe9fa0383466d-v1", "taskId": "review-48dfe9fa0383466d", "version": "1", "name": "online-perceptron-classifier-resubmit", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4b19088f6b926282-v1", "taskId": "review-4b19088f6b926282", "version": "1", "name": "sensor-placement-shift-rare-activity", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4e3ad65b5dc08010-v1", "taskId": "review-4e3ad65b5dc08010", "version": "1", "name": "featplan-imr", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-50844bdc0b32f0b9-v1", "taskId": "review-50844bdc0b32f0b9", "version": "1", "name": "equity-rare-strata-memo", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-51b8441e76d0691c-v1", "taskId": "review-51b8441e76d0691c", "version": "1", "name": "delayed-label-triage-pipeline", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 15, "n": 15, "p": 1, "lo": 0.7961166989641515, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.7961166989641515, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 7, "n": 15, "p": 0.4666666666666667, "lo": 0.24809535365737387, "hi": 0.698830199745016, "tasks": 1, "macro": 0.4666666666666667, "macroLo": 0.24809535365737387, "macroHi": 0.698830199745016, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-55a233f45e278dac-v1", "taskId": "review-55a233f45e278dac", "version": "1", "name": "coral-bleach-rank-feasibility", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-577a03ffbf2ddf5a-v1", "taskId": "review-577a03ffbf2ddf5a", "version": "1", "name": "graph-rule-learning-js", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-59f8af9563cbe5ed-v1", "taskId": "review-59f8af9563cbe5ed", "version": "1", "name": "airfoil-missing-thickness-marginalized-gp", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5d5675ab0bfd64f3-v1", "taskId": "review-5d5675ab0bfd64f3", "version": "1", "name": "tidal-equity-capability-assessment", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6ca971d1253cf34b-v1", "taskId": "review-6ca971d1253cf34b", "version": "1", "name": "Snorkel-reviewed task review-6ca971d1253cf34b", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/245" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6ece93d97eee5d96-v1", "taskId": "review-6ece93d97eee5d96", "version": "1", "name": "Snorkel-reviewed task review-6ece93d97eee5d96", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/275" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-75f58780438ea226-v1", "taskId": "review-75f58780438ea226", "version": "1", "name": "Snorkel-reviewed task review-75f58780438ea226", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/237" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7bc2acac40ed7f5e-v1", "taskId": "review-7bc2acac40ed7f5e", "version": "1", "name": "Snorkel-reviewed task review-7bc2acac40ed7f5e", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 8, "n": 10, "p": 0.8, "lo": 0.4901624715366418, "hi": 0.9433178485456248, "tasks": 1, "macro": 0.8, "macroLo": 0.4901624715366418, "macroHi": 0.9433178485456248, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/230" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7f6ced1303079ede-v1", "taskId": "review-7f6ced1303079ede", "version": "1", "name": "Snorkel-reviewed task review-7f6ced1303079ede", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/255" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8114696746ad4a99-v1", "taskId": "review-8114696746ad4a99", "version": "1", "name": "go-transformer-quantizer", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8b0df0105b32aa9e-v1", "taskId": "review-8b0df0105b32aa9e", "version": "1", "name": "Snorkel-reviewed task review-8b0df0105b32aa9e", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/383" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8c8d0d6f38620ec1-v1", "taskId": "review-8c8d0d6f38620ec1", "version": "1", "name": "Snorkel-reviewed task review-8c8d0d6f38620ec1", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 4, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/268" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-927a718f0b0dbdd6-v1", "taskId": "review-927a718f0b0dbdd6", "version": "1", "name": "Snorkel-reviewed task review-927a718f0b0dbdd6", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-936c197eaabe6148-v1", "taskId": "review-936c197eaabe6148", "version": "1", "name": "Snorkel-reviewed task review-936c197eaabe6148", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/270" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-943cb1a5777d050f-v1", "taskId": "review-943cb1a5777d050f", "version": "1", "name": "Snorkel-reviewed task review-943cb1a5777d050f", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/371" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9bcf6a7ac7f589ce-v1", "taskId": "review-9bcf6a7ac7f589ce", "version": "1", "name": "beijing-sleeping-experts-delayed-hedge", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9e0d533702efde46-v1", "taskId": "review-9e0d533702efde46", "version": "1", "name": "foundry-shift-calibration-release", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a87234c5f684cf5f-v1", "taskId": "review-a87234c5f684cf5f", "version": "1", "name": "Snorkel-reviewed task review-a87234c5f684cf5f", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/430" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-aab8fedc075eb4c3-v1", "taskId": "review-aab8fedc075eb4c3", "version": "1", "name": "Snorkel-reviewed task review-aab8fedc075eb4c3", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/462" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ab45799234d4905d-v1", "taskId": "review-ab45799234d4905d", "version": "1", "name": "Snorkel-reviewed task review-ab45799234d4905d", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/134" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-af174d05c47c42f7-v1", "taskId": "review-af174d05c47c42f7", "version": "1", "name": "Snorkel-reviewed task review-af174d05c47c42f7", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/389" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-af83460793a10cc0-v1", "taskId": "review-af83460793a10cc0", "version": "1", "name": "foundry-shift-calibration-release", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b40a75f5744fa3e4-v1", "taskId": "review-b40a75f5744fa3e4", "version": "1", "name": "Snorkel-reviewed task review-b40a75f5744fa3e4", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/168" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b4671b445e94263e-v1", "taskId": "review-b4671b445e94263e", "version": "1", "name": "Snorkel-reviewed task review-b4671b445e94263e", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/492" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b9d8a7bedf8bbd9f-v1", "taskId": "review-b9d8a7bedf8bbd9f", "version": "1", "name": "Snorkel-reviewed task review-b9d8a7bedf8bbd9f", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/421" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-bd836ce62bc30795-v1", "taskId": "review-bd836ce62bc30795", "version": "1", "name": "Snorkel-reviewed task review-bd836ce62bc30795", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/154" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-be41256a22975e12-v1", "taskId": "review-be41256a22975e12", "version": "1", "name": "coral-bleach-rank-feasibility", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-bf1613c50e1e8a93-v1", "taskId": "review-bf1613c50e1e8a93", "version": "1", "name": "Snorkel-reviewed task review-bf1613c50e1e8a93", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/287" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c1b5ef16800cfe1a-v1", "taskId": "review-c1b5ef16800cfe1a", "version": "1", "name": "overlap-profile-realizer", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c5526eb2fe38c178-v1", "taskId": "review-c5526eb2fe38c178", "version": "1", "name": "Snorkel-reviewed task review-c5526eb2fe38c178", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/368" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c69418079ff48fbf-v1", "taskId": "review-c69418079ff48fbf", "version": "1", "name": "rust-qx-desk-audit", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 8, "p": 0.5, "lo": 0.21521606221387757, "hi": 0.7847839377861224, "tasks": 1, "macro": 0.5, "macroLo": 0.21521606221387757, "macroHi": 0.7847839377861224, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cc60acd6390736a2-v1", "taskId": "review-cc60acd6390736a2", "version": "1", "name": "endpoint-order-ledger", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d257aebc7410222f-v1", "taskId": "review-d257aebc7410222f", "version": "1", "name": "online-perceptron-classifier-resubmit", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-da4a7759-v1", "taskId": "review-da4a7759", "version": "1", "name": "Snorkel-reviewed task review-da4a7759", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "review-df5afc4833baa4ec-v1", "taskId": "review-df5afc4833baa4ec", "version": "1", "name": "Snorkel-reviewed task review-df5afc4833baa4ec", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/205" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ea6fde5a6a26ce26-v1", "taskId": "review-ea6fde5a6a26ce26", "version": "1", "name": "rock-geochem-compositional-closure", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ecfbd47c93392f7e-v1", "taskId": "review-ecfbd47c93392f7e", "version": "1", "name": "incremental-feature-planner", "domain": "Machine learning", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f66ae60930da536d-v1", "taskId": "review-f66ae60930da536d", "version": "1", "name": "online-perceptron-classifier-resubmit", "domain": "Machine learning", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-bikeshare-daily-leakage-regression-v1", "taskId": "review-slug-bikeshare-daily-leakage-regression", "version": "1", "name": "bikeshare-daily-leakage-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "seismic-cost-sensitive-v1", "taskId": "seismic-cost-sensitive", "version": "1", "name": "seismic-cost-sensitive", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/seismic-cost-sensitive" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sepsis-missing-imbalance-v1", "taskId": "sepsis-missing-imbalance", "version": "1", "name": "sepsis-missing-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/sepsis-missing-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "simjeb-bracket-fea-mass-prediction-real-v1", "taskId": "simjeb-bracket-fea-mass-prediction-real", "version": "1", "name": "simjeb-bracket-fea-mass-prediction-real", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/104" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "software-calib-conceptdrift-selbias-stratifiedeval-v1", "taskId": "software-calib-conceptdrift-selbias-stratifiedeval", "version": "1", "name": "software-calib-conceptdrift-selbias-stratifiedeval", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/software-calib-conceptdrift-selbias-stratifiedeval" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "software-reviewlen-quality-vocabdrift-v1", "taskId": "software-reviewlen-quality-vocabdrift", "version": "1", "name": "software-reviewlen-quality-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/software-reviewlen-quality-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sportsandoutdoors-calib-imbalance-metafusion-multilabel-v1", "taskId": "sportsandoutdoors-calib-imbalance-metafusion-multilabel", "version": "1", "name": "sportsandoutdoors-calib-imbalance-metafusion-multilabel", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/sportsandoutdoors-calib-imbalance-metafusion-multilabel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sportsandoutdoors-fitness-quality-reviewlen-sentmismatch-vocabdrift-v1", "taskId": "sportsandoutdoors-fitness-quality-reviewlen-sentmismatch-vocabdrift", "version": "1", "name": "sportsandoutdoors-fitness-quality-reviewlen-sentmismatch-vocabdrift", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/sportsandoutdoors-fitness-quality-reviewlen-sentmismatch-vocabdrift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spotify-songs-modeling-v1", "taskId": "spotify-songs-modeling", "version": "1", "name": "spotify-songs-modeling", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/spotify-songs-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "squeeze-qwen3-5-0-8b-300mb-v1", "taskId": "squeeze-qwen3.5-0.8b-300mb", "version": "1", "name": "squeeze-qwen3.5-0.8b-300mb", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/570" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "stars-cyg-robust-regression-v1", "taskId": "stars-cyg-robust-regression", "version": "1", "name": "stars-cyg-robust-regression", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/stars-cyg-robust-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "state-hpi-modeling-v1", "taskId": "state-hpi-modeling", "version": "1", "name": "state-hpi-modeling", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/tidy-tuesday/state-hpi-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "stepwise-weighted-mdp-policy-value-v1", "taskId": "stepwise-weighted-mdp-policy-value", "version": "1", "name": "stepwise-weighted-mdp-policy-value", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/policyml/stepwise-weighted-mdp-policy-value" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "streamflow-sequential-alarm-labeler-v1", "taskId": "streamflow-sequential-alarm-labeler", "version": "1", "name": "streamflow-sequential-alarm-labeler", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/streamflow-sequential-alarm-labeler" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "taiwan-bankruptcy-calib-imbalance-v1", "taskId": "taiwan-bankruptcy-calib-imbalance", "version": "1", "name": "taiwan-bankruptcy-calib-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/taiwan-bankruptcy-calib-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tools-helpfulvotes-multiout-outlier-v1", "taskId": "tools-helpfulvotes-multiout-outlier", "version": "1", "name": "tools-helpfulvotes-multiout-outlier", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/tools-helpfulvotes-multiout-outlier" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tools-imbalance-metafusion-multilabel-reviewlen-v1", "taskId": "tools-imbalance-metafusion-multilabel-reviewlen", "version": "1", "name": "tools-imbalance-metafusion-multilabel-reviewlen", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/tools-imbalance-metafusion-multilabel-reviewlen" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "toolsandhomeimprovement-imbalance-reviewlen-sentmismatch-text-v1", "taskId": "toolsandhomeimprovement-imbalance-reviewlen-sentmismatch-text", "version": "1", "name": "toolsandhomeimprovement-imbalance-reviewlen-sentmismatch-text", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/toolsandhomeimprovement-imbalance-reviewlen-sentmismatch-text" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "toysandgames-imbalance-reviewlen-sentmismatch-v1", "taskId": "toysandgames-imbalance-reviewlen-sentmismatch", "version": "1", "name": "toysandgames-imbalance-reviewlen-sentmismatch", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/toysandgames-imbalance-reviewlen-sentmismatch" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "toysgames-boardgames-catencode-groupcv-interactions-smallsample-v1", "taskId": "toysgames-boardgames-catencode-groupcv-interactions-smallsample", "version": "1", "name": "toysgames-boardgames-catencode-groupcv-interactions-smallsample", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/toysgames-boardgames-catencode-groupcv-interactions-smallsample" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "toysgames-calib-groupcv-imbalance-metmismatch-v1", "taskId": "toysgames-calib-groupcv-imbalance-metmismatch", "version": "1", "name": "toysgames-calib-groupcv-imbalance-metmismatch", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/mle/toysgames-calib-groupcv-imbalance-metmismatch" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tvs-loan-cost-imbalance-v1", "taskId": "tvs-loan-cost-imbalance", "version": "1", "name": "tvs-loan-cost-imbalance", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/tvs-loan-cost-imbalance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "unemployment-parametric-aft-duration-v1", "taskId": "unemployment-parametric-aft-duration", "version": "1", "name": "unemployment-parametric-aft-duration", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/unemployment-parametric-aft-duration" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "videogames-catencode-imbalance-conceptdrift-selbias-v1", "taskId": "videogames-catencode-imbalance-conceptdrift-selbias", "version": "1", "name": "videogames-catencode-imbalance-conceptdrift-selbias", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/mle/videogames-catencode-imbalance-conceptdrift-selbias" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "watermark-window-aggregator-v1", "taskId": "watermark-window-aggregator", "version": "1", "name": "watermark-window-aggregator", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/watermark-window-aggregator" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "windowed-majority-vote-v1", "taskId": "windowed-majority-vote", "version": "1", "name": "windowed-majority-vote", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/windowed-majority-vote" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "windowed-min-floor-v1", "taskId": "windowed-min-floor", "version": "1", "name": "windowed-min-floor", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/streamml/windowed-min-floor" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "worst-group-spurious-dfr-v1", "taskId": "worst-group-spurious-dfr", "version": "1", "name": "worst-group-spurious-dfr", "domain": "Machine learning", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/sf-data/tasks/research-engineering/worst-group-spurious-dfr" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "beam-deflection-solver-v1", "taskId": "beam-deflection-solver", "version": "1", "name": "beam-deflection-solver", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/31" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "collision2d-impulse-solver-v1", "taskId": "collision2d-impulse-solver", "version": "1", "name": "collision2d-impulse-solver", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/32" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "deepcad-canonical-equivalence-v1", "taskId": "deepcad-canonical-equivalence", "version": "1", "name": "deepcad-canonical-equivalence", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/mechanical/deepcad-canonical-equivalence" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "deepcad-invalid-command-localization-v1", "taskId": "deepcad-invalid-command-localization", "version": "1", "name": "deepcad-invalid-command-localization", "domain": "Mechanical engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 6, "p": 0, "lo": 0, "hi": 0.39033428790216534, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.39033428790216534, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 6, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 6, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/mechanical/deepcad-invalid-command-localization" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "deepcad-sketch-argument-imputation-v1", "taskId": "deepcad-sketch-argument-imputation", "version": "1", "name": "deepcad-sketch-argument-imputation", "domain": "Mechanical engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 10, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/mechanical/deepcad-sketch-argument-imputation" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "driveline-torsional-detune-v1", "taskId": "driveline-torsional-detune", "version": "1", "name": "driveline-torsional-detune", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/133" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gearbox-preload-stackup-v1", "taskId": "gearbox-preload-stackup", "version": "1", "name": "gearbox-preload-stackup", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/133" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "heat1d-conduction-solver-v1", "taskId": "heat1d-conduction-solver", "version": "1", "name": "heat1d-conduction-solver", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/33" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "heatsink-fin-solver-debug-v1", "taskId": "heatsink-fin-solver-debug", "version": "1", "name": "heatsink-fin-solver-debug", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/133" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "isolation-platform-sysid-v1", "taskId": "isolation-platform-sysid", "version": "1", "name": "isolation-platform-sysid", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/133" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pid-control-simulator-v1", "taskId": "pid-control-simulator", "version": "1", "name": "pid-control-simulator", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/34" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pipe-rack-truss-sizing-v1", "taskId": "pipe-rack-truss-sizing", "version": "1", "name": "pipe-rack-truss-sizing", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/133" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pipeflow-colebrook-solver-v1", "taskId": "pipeflow-colebrook-solver", "version": "1", "name": "pipeflow-colebrook-solver", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/35" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pipeline-scour-two-thirds-law-v1", "taskId": "pipeline-scour-two-thirds-law", "version": "1", "name": "pipeline-scour-two-thirds-law", "domain": "Mechanical engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 20, "p": 0, "lo": 0, "hi": 0.16112515805281938, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.16112515805281938, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 20, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/mechanical/pipeline-scour-two-thirds-law" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "projectile-drag-integrator-v1", "taskId": "projectile-drag-integrator", "version": "1", "name": "projectile-drag-integrator", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/36" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "quaternion-rotation-integrator-v1", "taskId": "quaternion-rotation-integrator", "version": "1", "name": "quaternion-rotation-integrator", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/37" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rk4-orbit-integrator-v1", "taskId": "rk4-orbit-integrator", "version": "1", "name": "rk4-orbit-integrator", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/38" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "springmass-modal-solver-v1", "taskId": "springmass-modal-solver", "version": "1", "name": "springmass-modal-solver", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/39" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "truss2d-solver-v1", "taskId": "truss2d-solver", "version": "1", "name": "truss2d-solver", "domain": "Mechanical engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/40" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vegetated-alternate-bar-stability-v1", "taskId": "vegetated-alternate-bar-stability", "version": "1", "name": "vegetated-alternate-bar-stability", "domain": "Mechanical engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 20, "p": 0.1, "lo": 0.027866481213768224, "hi": 0.3010336452284873, "tasks": 1, "macro": 0.1, "macroLo": 0.027866481213768224, "macroHi": 0.3010336452284873, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 20, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/turing/mechanical/vegetated-alternate-bar-stability" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "adaptive-quadrature-v1", "taskId": "adaptive-quadrature", "version": "1", "name": "adaptive-quadrature", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/41" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "anova-stats-v1", "taskId": "anova-stats", "version": "1", "name": "anova-stats", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/42" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cg-solver-v1", "taskId": "cg-solver", "version": "1", "name": "cg-solver", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/43" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cholesky-solver-v1", "taskId": "cholesky-solver", "version": "1", "name": "cholesky-solver", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/44" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "coffee-ratings-outliers-v1", "taskId": "coffee-ratings-outliers", "version": "1", "name": "coffee-ratings-outliers", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/97" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cubic-spline-v1", "taskId": "cubic-spline", "version": "1", "name": "cubic-spline", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/45" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fedavg-federated-noniid-mnist-v1", "taskId": "fedavg-federated-noniid-mnist", "version": "1", "name": "fedavg-federated-noniid-mnist", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/98" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fft-radix2-v1", "taskId": "fft-radix2", "version": "1", "name": "fft-radix2", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/46" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lending-club-lgd-bias-correction-r-v1", "taskId": "lending-club-lgd-bias-correction-r", "version": "1", "name": "lending-club-lgd-bias-correction-r", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/101" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lu-decomposition-v1", "taskId": "lu-decomposition", "version": "1", "name": "lu-decomposition", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/47" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "neonatal-drug-exposure-nlme-v1", "taskId": "neonatal-drug-exposure-nlme", "version": "1", "name": "neonatal-drug-exposure-nlme", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/102" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ols-regression-v1", "taskId": "ols-regression", "version": "1", "name": "ols-regression", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/48" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "product-recall-stock-price-event-v1", "taskId": "product-recall-stock-price-event", "version": "1", "name": "product-recall-stock-price-event", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/103" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "qr-decomposition-v1", "taskId": "qr-decomposition", "version": "1", "name": "qr-decomposition", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/49" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1e0d7065-v1", "taskId": "review-1e0d7065", "version": "1", "name": "Snorkel-reviewed task review-1e0d7065", "domain": "Science / STEM", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-5b65500f-v1", "taskId": "review-5b65500f", "version": "1", "name": "Snorkel-reviewed task review-5b65500f", "domain": "Science / STEM", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-5d65bdca-v1", "taskId": "review-5d65bdca", "version": "1", "name": "Snorkel-reviewed task review-5d65bdca", "domain": "Science / STEM", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8d6a0ae6-v1", "taskId": "review-8d6a0ae6", "version": "1", "name": "Snorkel-reviewed task review-8d6a0ae6", "domain": "Science / STEM", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cad7d2e8-v1", "taskId": "review-cad7d2e8", "version": "1", "name": "Snorkel-reviewed task review-cad7d2e8", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cd933c9b-v1", "taskId": "review-cd933c9b", "version": "1", "name": "Snorkel-reviewed task review-cd933c9b", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e874f7d6-v1", "taskId": "review-e874f7d6", "version": "1", "name": "Snorkel-reviewed task review-e874f7d6", "domain": "Science / STEM", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-acleval-v1", "taskId": "review-slug-acleval", "version": "1", "name": "acleval", "domain": "Science / STEM", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-netns-veth-egress-isolation-v1", "taskId": "review-slug-netns-veth-egress-isolation", "version": "1", "name": "netns-veth-egress-isolation", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-partial-deploy-rollback-orchestrator-v1", "taskId": "review-slug-partial-deploy-rollback-orchestrator", "version": "1", "name": "partial-deploy-rollback-orchestrator", "domain": "Science / STEM", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-runit-finish-epoch-pid-stale-v1", "taskId": "review-slug-runit-finish-epoch-pid-stale", "version": "1", "name": "runit-finish-epoch-pid-stale", "domain": "Science / STEM", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "root-finder-v1", "taskId": "root-finder", "version": "1", "name": "root-finder", "domain": "Science / STEM", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/50" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "accruals-earnings-persistence-v1", "taskId": "accruals-earnings-persistence", "version": "1", "name": "accruals-earnings-persistence", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/accruals-earnings-persistence" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "affairs-tobit-mm-bootstrap-v1", "taskId": "affairs-tobit-mm-bootstrap", "version": "1", "name": "affairs-tobit-mm-bootstrap", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/affairs-tobit-mm-bootstrap" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "amihud-goyenko-2013-mutual-fund-rsq-v1", "taskId": "amihud-goyenko-2013-mutual-fund-rsq", "version": "1", "name": "amihud-goyenko-2013-mutual-fund-rsq", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/amihud-goyenko-2013-mutual-fund-rsq" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "amortization-v1", "taskId": "amortization", "version": "1", "name": "amortization", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/amortization" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "annual-earnings-abnormal-performance-index-v1", "taskId": "annual-earnings-abnormal-performance-index", "version": "1", "name": "annual-earnings-abnormal-performance-index", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/annual-earnings-abnormal-performance-index" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "antitakeover-law-investment-staggered-did-v1", "taskId": "antitakeover-law-investment-staggered-did", "version": "1", "name": "antitakeover-law-investment-staggered-did", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/antitakeover-law-investment-staggered-did" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "assignment-conflict-pairs-v1", "taskId": "assignment-conflict-pairs", "version": "1", "name": "assignment-conflict-pairs", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/assignment-conflict-pairs" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "balance-sheet-accrual-persistence-v1", "taskId": "balance-sheet-accrual-persistence", "version": "1", "name": "balance-sheet-accrual-persistence", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/balance-sheet-accrual-persistence" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "barycentric-chebyshev-lobatto-v1", "taskId": "barycentric-chebyshev-lobatto", "version": "1", "name": "barycentric-chebyshev-lobatto", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/ms-numerical/barycentric-chebyshev-lobatto" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "berger-klapper-turk-ariss-2009-bank-competition-stability-v1", "taskId": "berger-klapper-turk-ariss-2009-bank-competition-stability", "version": "1", "name": "berger-klapper-turk-ariss-2009-bank-competition-stability", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/berger-klapper-turk-ariss-2009-bank-competition-stability" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "binpack-incompatible-pairs-v1", "taskId": "binpack-incompatible-pairs", "version": "1", "name": "binpack-incompatible-pairs", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/binpack-incompatible-pairs" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "blood-bank-perishable-crossmatch-recourse-v1", "taskId": "blood-bank-perishable-crossmatch-recourse", "version": "1", "name": "blood-bank-perishable-crossmatch-recourse", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/blood-bank-perishable-crossmatch-recourse" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "boudoukh-michaely-2007-payout-yield-return-predictability-v1", "taskId": "boudoukh-michaely-2007-payout-yield-return-predictability", "version": "1", "name": "boudoukh-michaely-2007-payout-yield-return-predictability", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/boudoukh-michaely-2007-payout-yield-return-predictability" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bundle-award-lane-conflicts-v1", "taskId": "bundle-award-lane-conflicts", "version": "1", "name": "bundle-award-lane-conflicts", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/bundle-award-lane-conflicts" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "calendar-setup-deteriorating-maxlag-scheduling-v1", "taskId": "calendar-setup-deteriorating-maxlag-scheduling", "version": "1", "name": "calendar-setup-deteriorating-maxlag-scheduling", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/calendar-setup-deteriorating-maxlag-scheduling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "capacitated-facility-single-sourcing-v1", "taskId": "capacitated-facility-single-sourcing", "version": "1", "name": "capacitated-facility-single-sourcing", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/capacitated-facility-single-sourcing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "capacitated-vehicle-routing-load-cost-v1", "taskId": "capacitated-vehicle-routing-load-cost", "version": "1", "name": "capacitated-vehicle-routing-load-cost", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/capacitated-vehicle-routing-load-cost" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "car-sequencing-v1", "taskId": "car-sequencing", "version": "1", "name": "car-sequencing", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/car-sequencing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chiah-gharghori-zhong-2023-ivol-trend-v1", "taskId": "chiah-gharghori-zhong-2023-ivol-trend", "version": "1", "name": "chiah-gharghori-zhong-2023-ivol-trend", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/chiah-gharghori-zhong-2023-ivol-trend" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cloud-tenant-placement-failover-minimax-v1", "taskId": "cloud-tenant-placement-failover-minimax", "version": "1", "name": "cloud-tenant-placement-failover-minimax", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/cloud-tenant-placement-failover-minimax" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "clsp-setup-time-capacity-v1", "taskId": "clsp-setup-time-capacity", "version": "1", "name": "clsp-setup-time-capacity", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/clsp-setup-time-capacity" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "collateral-channel-real-estate-investment-v1", "taskId": "collateral-channel-real-estate-investment", "version": "1", "name": "collateral-channel-real-estate-investment", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/collateral-channel-real-estate-investment" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "college-distance-iv-v1", "taskId": "college-distance-iv", "version": "1", "name": "college-distance-iv", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/college-distance-iv" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "common-due-date-scheduling-v1", "taskId": "common-due-date-scheduling", "version": "1", "name": "common-due-date-scheduling", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/common-due-date-scheduling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "compensated-determinant-kernel-v1", "taskId": "compensated-determinant-kernel", "version": "1", "name": "compensated-determinant-kernel", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/systems-spec/compensated-determinant-kernel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "convex-layers-onion-peeling-exact-v1", "taskId": "convex-layers-onion-peeling-exact", "version": "1", "name": "convex-layers-onion-peeling-exact", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/origami/convex-layers-onion-peeling-exact" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cooper-gulen-schill-2008-asset-growth-decile-spread-anomaly-v1", "taskId": "cooper-gulen-schill-2008-asset-growth-decile-spread-anomaly", "version": "1", "name": "cooper-gulen-schill-2008-asset-growth-decile-spread-anomaly", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/cooper-gulen-schill-2008-asset-growth-decile-spread-anomaly" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "correa-goldberg-rice-2015-international-banking-us-v1", "taskId": "correa-goldberg-rice-2015-international-banking-us", "version": "1", "name": "correa-goldberg-rice-2015-international-banking-us", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/correa-goldberg-rice-2015-international-banking-us" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "crime-nc-county-panel-2sls-v1", "taskId": "crime-nc-county-panel-2sls", "version": "1", "name": "crime-nc-county-panel-2sls", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/crime-nc-county-panel-2sls" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cutting-stock-trim-limit-v1", "taskId": "cutting-stock-trim-limit", "version": "1", "name": "cutting-stock-trim-limit", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/cutting-stock-trim-limit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cvrp-capacity-v1", "taskId": "cvrp-capacity", "version": "1", "name": "cvrp-capacity", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/cvrp-capacity" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-boyer-moore-majority-invariant-v1", "taskId": "dafny-boyer-moore-majority-invariant", "version": "1", "name": "dafny-boyer-moore-majority-invariant", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/dafny-boyer-moore-majority-invariant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-leftist-heap-merge-rank-invariant-v1", "taskId": "dafny-leftist-heap-merge-rank-invariant", "version": "1", "name": "dafny-leftist-heap-merge-rank-invariant", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/dafny-leftist-heap-merge-rank-invariant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-min-stack-valid-invariant-v1", "taskId": "dafny-min-stack-valid-invariant", "version": "1", "name": "dafny-min-stack-valid-invariant", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/dafny-min-stack-valid-invariant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-persistent-segment-tree-lazy-propagation-invariant-v1", "taskId": "dafny-persistent-segment-tree-lazy-propagation-invariant", "version": "1", "name": "dafny-persistent-segment-tree-lazy-propagation-invariant", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/dafny-persistent-segment-tree-lazy-propagation-invariant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-red-black-tree-insert-balance-invariant-v1", "taskId": "dafny-red-black-tree-insert-balance-invariant", "version": "1", "name": "dafny-red-black-tree-insert-balance-invariant", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/dafny-red-black-tree-insert-balance-invariant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-ring-buffer-queue-invariant-v1", "taskId": "dafny-ring-buffer-queue-invariant", "version": "1", "name": "dafny-ring-buffer-queue-invariant", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/dafny-ring-buffer-queue-invariant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-two-pointer-pair-sum-completeness-v1", "taskId": "dafny-two-pointer-pair-sum-completeness", "version": "1", "name": "dafny-two-pointer-pair-sum-completeness", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/proofgen/dafny-two-pointer-pair-sum-completeness" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dafny-union-find-rank-invariant-v1", "taskId": "dafny-union-find-rank-invariant", "version": "1", "name": "dafny-union-find-rank-invariant", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/dafny-union-find-rank-invariant" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "degradation-recurrent-terminal-mcf-v1", "taskId": "degradation-recurrent-terminal-mcf", "version": "1", "name": "degradation-recurrent-terminal-mcf", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/survival/degradation-recurrent-terminal-mcf" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "deteriorating-setup-lag-tardiness-v1", "taskId": "deteriorating-setup-lag-tardiness", "version": "1", "name": "deteriorating-setup-lag-tardiness", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/deteriorating-setup-lag-tardiness" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "detzel-duarte-2023-volatility-replication-v1", "taskId": "detzel-duarte-2023-volatility-replication", "version": "1", "name": "detzel-duarte-2023-volatility-replication", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/detzel-duarte-2023-volatility-replication" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "empluk-dynamic-labor-demand-v1", "taskId": "empluk-dynamic-labor-demand", "version": "1", "name": "empluk-dynamic-labor-demand", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/rdatasets/empluk-dynamic-labor-demand" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "equity-factor-spanning-grs-test-v1", "taskId": "equity-factor-spanning-grs-test", "version": "1", "name": "equity-factor-spanning-grs-test", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/equity-factor-spanning-grs-test" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "exam-invigilator-time-conflict-v1", "taskId": "exam-invigilator-time-conflict", "version": "1", "name": "exam-invigilator-time-conflict", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/exam-invigilator-time-conflict" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "facility-siting-conflict-capacity-v1", "taskId": "facility-siting-conflict-capacity", "version": "1", "name": "facility-siting-conflict-capacity", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/facility-siting-conflict-capacity" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fama-bliss-1987-forward-spread-bond-risk-premium-v1", "taskId": "fama-bliss-1987-forward-spread-bond-risk-premium", "version": "1", "name": "fama-bliss-1987-forward-spread-bond-risk-premium", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/fama-bliss-1987-forward-spread-bond-risk-premium" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "family-overhead-bin-packing-cost-v1", "taskId": "family-overhead-bin-packing-cost", "version": "1", "name": "family-overhead-bin-packing-cost", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/family-overhead-bin-packing-cost" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fatalities-beer-tax-panel-v1", "taskId": "fatalities-beer-tax-panel", "version": "1", "name": "fatalities-beer-tax-panel", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/fatalities-beer-tax-panel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "filip-qr-least-squares-v1", "taskId": "filip-qr-least-squares", "version": "1", "name": "filip-qr-least-squares", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/ms-numerical/filip-qr-least-squares" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fixed-charge-transportation-v1", "taskId": "fixed-charge-transportation", "version": "1", "name": "fixed-charge-transportation", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/fixed-charge-transportation" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "four-factor-momentum-attribution-v1", "taskId": "four-factor-momentum-attribution", "version": "1", "name": "four-factor-momentum-attribution", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/four-factor-momentum-attribution" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "four-leg-chair-v1", "taskId": "four-leg-chair", "version": "1", "name": "four-leg-chair", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 10, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 10, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/119" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "frazzini-pedersen-2014-betting-against-beta-bab-factor-v1", "taskId": "frazzini-pedersen-2014-betting-against-beta-bab-factor", "version": "1", "name": "frazzini-pedersen-2014-betting-against-beta-bab-factor", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/frazzini-pedersen-2014-betting-against-beta-bab-factor" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "freq-assign-l21-span-v1", "taskId": "freq-assign-l21-span", "version": "1", "name": "freq-assign-l21-span", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/freq-assign-l21-span" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "givens-qr-factorisation-v1", "taskId": "givens-qr-factorisation", "version": "1", "name": "givens-qr-factorisation", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/ms-numerical/givens-qr-factorisation" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "han-2022-bab-components-v1", "taskId": "han-2022-bab-components", "version": "1", "name": "han-2022-bab-components", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/han-2022-bab-components" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "holden-nam-2019-illiquidity-replication-v1", "taskId": "holden-nam-2019-illiquidity-replication", "version": "1", "name": "holden-nam-2019-illiquidity-replication", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/holden-nam-2019-illiquidity-replication" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hollow-bowl-v1", "taskId": "hollow-bowl", "version": "1", "name": "hollow-bowl", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 8, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 8, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/121" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "home-health-routing-skills-breaks-v1", "taskId": "home-health-routing-skills-breaks", "version": "1", "name": "home-health-routing-skills-breaks", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/home-health-routing-skills-breaks" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "indometh-pk-biexponential-nlme-v1", "taskId": "indometh-pk-biexponential-nlme", "version": "1", "name": "indometh-pk-biexponential-nlme", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/indometh-pk-biexponential-nlme" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "investment-spending-persistence-dynamic-panel-v1", "taskId": "investment-spending-persistence-dynamic-panel", "version": "1", "name": "investment-spending-persistence-dynamic-panel", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/investment-spending-persistence-dynamic-panel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jensen-1968-performance-mutual-funds-1945-1964-v1", "taskId": "jensen-1968-performance-mutual-funds-1945-1964", "version": "1", "name": "jensen-1968-performance-mutual-funds-1945-1964", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/jensen-1968-performance-mutual-funds-1945-1964" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kahan-compensated-sum-v1", "taskId": "kahan-compensated-sum", "version": "1", "name": "kahan-compensated-sum", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/ms-numerical/kahan-compensated-sum" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kg-thermoelastic-porous-cylinder-v1", "taskId": "kg-thermoelastic-porous-cylinder", "version": "1", "name": "kg-thermoelastic-porous-cylinder", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/126" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "knapsack-prerequisites-v1", "taskId": "knapsack-prerequisites", "version": "1", "name": "knapsack-prerequisites", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/knapsack-prerequisites" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lag-1-autocorrelation-kernel-v1", "taskId": "lag-1-autocorrelation-kernel", "version": "1", "name": "lag-1-autocorrelation-kernel", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/systems-spec/lag-1-autocorrelation-kernel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lean-list-reverse-involution-v1", "taskId": "lean-list-reverse-involution", "version": "1", "name": "lean-list-reverse-involution", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/proofgen/lean-list-reverse-involution" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lean4-bezout-gcd-linear-combination-v1", "taskId": "lean4-bezout-gcd-linear-combination", "version": "1", "name": "lean4-bezout-gcd-linear-combination", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/proofgen/lean4-bezout-gcd-linear-combination" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "leverage-speed-of-adjustment-gmm-v1", "taskId": "leverage-speed-of-adjustment-gmm", "version": "1", "name": "leverage-speed-of-adjustment-gmm", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/leverage-speed-of-adjustment-gmm" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "long-run-bhar-test-specification-v1", "taskId": "long-run-bhar-test-specification", "version": "1", "name": "long-run-bhar-test-specification", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/long-run-bhar-test-specification" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "matallin-saez-2023-fund-r2-active-management-v1", "taskId": "matallin-saez-2023-fund-r2-active-management", "version": "1", "name": "matallin-saez-2023-fund-r2-active-management", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/matallin-saez-2023-fund-r2-active-management" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "maximal-covering-location-v1", "taskId": "maximal-covering-location", "version": "1", "name": "maximal-covering-location", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/maximal-covering-location" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "maximal-covering-p-facility-v1", "taskId": "maximal-covering-p-facility", "version": "1", "name": "maximal-covering-p-facility", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/maximal-covering-p-facility" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "microgrid-unit-commitment-contingencies-v1", "taskId": "microgrid-unit-commitment-contingencies", "version": "1", "name": "microgrid-unit-commitment-contingencies", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/microgrid-unit-commitment-contingencies" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "min-area-enclosing-rectangle-v1", "taskId": "min-area-enclosing-rectangle", "version": "1", "name": "min-area-enclosing-rectangle", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/origami/min-area-enclosing-rectangle" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "minimum-dominating-set-v1", "taskId": "minimum-dominating-set", "version": "1", "name": "minimum-dominating-set", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/minimum-dominating-set" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mobile-clinic-reassignment-surcharge-v1", "taskId": "mobile-clinic-reassignment-surcharge", "version": "1", "name": "mobile-clinic-reassignment-surcharge", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/mobile-clinic-reassignment-surcharge" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "momentum-credit-rating-double-sort-v1", "taskId": "momentum-credit-rating-double-sort", "version": "1", "name": "momentum-credit-rating-double-sort", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/momentum-credit-rating-double-sort" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "momentum-premium-alpha-v1", "taskId": "momentum-premium-alpha", "version": "1", "name": "momentum-premium-alpha", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/momentum-premium-alpha" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "moskowitz-ooi-pedersen-2012-time-series-momentum-v1", "taskId": "moskowitz-ooi-pedersen-2012-time-series-momentum", "version": "1", "name": "moskowitz-ooi-pedersen-2012-time-series-momentum", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/moskowitz-ooi-pedersen-2012-time-series-momentum" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ms-ecdf-ks-statistic-v1", "taskId": "ms-ecdf-ks-statistic", "version": "1", "name": "ms-ecdf-ks-statistic", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/ms-numerical/ms-ecdf-ks-statistic" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ms-lanczos-lgamma-v1", "taskId": "ms-lanczos-lgamma", "version": "1", "name": "ms-lanczos-lgamma", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/ms-numerical/ms-lanczos-lgamma" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ms-logsumexp-stable-v1", "taskId": "ms-logsumexp-stable", "version": "1", "name": "ms-logsumexp-stable", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/ms-numerical/ms-logsumexp-stable" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ms-quantile-type-convention-v1", "taskId": "ms-quantile-type-convention", "version": "1", "name": "ms-quantile-type-convention", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/ms-numerical/ms-quantile-type-convention" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "multi-depot-reload-routing-tenant-overhead-v1", "taskId": "multi-depot-reload-routing-tenant-overhead", "version": "1", "name": "multi-depot-reload-routing-tenant-overhead", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/multi-depot-reload-routing-tenant-overhead" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "multi-mode-rcpsp-calendar-maxlags-v1", "taskId": "multi-mode-rcpsp-calendar-maxlags", "version": "1", "name": "multi-mode-rcpsp-calendar-maxlags", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/multi-mode-rcpsp-calendar-maxlags" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "multicommodity-shared-arc-cap-v1", "taskId": "multicommodity-shared-arc-cap", "version": "1", "name": "multicommodity-shared-arc-cap", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/multicommodity-shared-arc-cap" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "multiple-knapsack-balance-v1", "taskId": "multiple-knapsack-balance", "version": "1", "name": "multiple-knapsack-balance", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/multiple-knapsack-balance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "no-wait-flowshop-v1", "taskId": "no-wait-flowshop", "version": "1", "name": "no-wait-flowshop", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/no-wait-flowshop" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "olympics-modeling-v1", "taskId": "olympics-modeling", "version": "1", "name": "olympics-modeling", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/tidy-tuesday/olympics-modeling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "open-shop-makespan-v1", "taskId": "open-shop-makespan", "version": "1", "name": "open-shop-makespan", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/open-shop-makespan" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "operating-room-cleanup-transitions-v1", "taskId": "operating-room-cleanup-transitions", "version": "1", "name": "operating-room-cleanup-transitions", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/operating-room-cleanup-transitions" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "order-acceptance-scheduling-v1", "taskId": "order-acceptance-scheduling", "version": "1", "name": "order-acceptance-scheduling", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/order-acceptance-scheduling" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "orienteering-budget-v1", "taskId": "orienteering-budget", "version": "1", "name": "orienteering-budget", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/orienteering-budget" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "orthodont-growth-curve-random-slopes-v1", "taskId": "orthodont-growth-curve-random-slopes", "version": "1", "name": "orthodont-growth-curve-random-slopes", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/orthodont-growth-curve-random-slopes" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "p-median-equity-v1", "taskId": "p-median-equity", "version": "1", "name": "p-median-equity", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/p-median-equity" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "parallel-machine-eligibility-v1", "taskId": "parallel-machine-eligibility", "version": "1", "name": "parallel-machine-eligibility", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/parallel-machine-eligibility" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pastor-stambaugh-taylor-zhu-2022-diseconomies-scale-active-management-v1", "taskId": "pastor-stambaugh-taylor-zhu-2022-diseconomies-scale-active-management", "version": "1", "name": "pastor-stambaugh-taylor-zhu-2022-diseconomies-scale-active-management", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/pastor-stambaugh-taylor-zhu-2022-diseconomies-scale-active-management" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pcenter-minimax-v1", "taskId": "pcenter-minimax", "version": "1", "name": "pcenter-minimax", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/pcenter-minimax" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pead-sue-decile-drift-v1", "taskId": "pead-sue-decile-drift", "version": "1", "name": "pead-sue-decile-drift", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/pead-sue-decile-drift" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "perishable-capacitated-lot-sizing-v1", "taskId": "perishable-capacitated-lot-sizing", "version": "1", "name": "perishable-capacitated-lot-sizing", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/perishable-capacitated-lot-sizing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "point-set-halving-lines-exact-v1", "taskId": "point-set-halving-lines-exact", "version": "1", "name": "point-set-halving-lines-exact", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/origami/point-set-halving-lines-exact" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "produc-state-productivity-panel-v1", "taskId": "produc-state-productivity-panel", "version": "1", "name": "produc-state-productivity-panel", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/produc-state-productivity-panel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "protein-rotamer-multistate-disulfide-design-v1", "taskId": "protein-rotamer-multistate-disulfide-design", "version": "1", "name": "protein-rotamer-multistate-disulfide-design", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/protein-rotamer-multistate-disulfide-design" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "qap-flow-distance-v1", "taskId": "qap-flow-distance", "version": "1", "name": "qap-flow-distance", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/qap-flow-distance" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "quine-nb-dispersion-heterogeneity-v1", "taskId": "quine-nb-dispersion-heterogeneity", "version": "1", "name": "quine-nb-dispersion-heterogeneity", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/quine-nb-dispersion-heterogeneity" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "radiotherapy-fraction-outage-recourse-v1", "taskId": "radiotherapy-fraction-outage-recourse", "version": "1", "name": "radiotherapy-fraction-outage-recourse", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/radiotherapy-fraction-outage-recourse" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "resource-constrained-shortest-path-v1", "taskId": "resource-constrained-shortest-path", "version": "1", "name": "resource-constrained-shortest-path", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/resource-constrained-shortest-path" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-026dfb9d2b48dec9-v1", "taskId": "review-026dfb9d2b48dec9", "version": "1", "name": "Snorkel-reviewed task review-026dfb9d2b48dec9", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/435" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-04d3d031ce54bfc2-v1", "taskId": "review-04d3d031ce54bfc2", "version": "1", "name": "Snorkel-reviewed task review-04d3d031ce54bfc2", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/404" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-06d1f0750367d8fd-v1", "taskId": "review-06d1f0750367d8fd", "version": "1", "name": "Snorkel-reviewed task review-06d1f0750367d8fd", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/376" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0d48168cf3c80656-v1", "taskId": "review-0d48168cf3c80656", "version": "1", "name": "rust-popgen-fst-scan", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 8, "n": 9, "p": 0.8888888888888888, "lo": 0.565000294423344, "hi": 0.980109112361455, "tasks": 1, "macro": 0.8888888888888888, "macroLo": 0.565000294423344, "macroHi": 0.980109112361455, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0e92905c4482e13a-v1", "taskId": "review-0e92905c4482e13a", "version": "1", "name": "synthesize-css-encoder-from-parity-checks", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-10dcfa97346ee443-v1", "taskId": "review-10dcfa97346ee443", "version": "1", "name": "Snorkel-reviewed task review-10dcfa97346ee443", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/189" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-10e3faa4cbb66d60-v1", "taskId": "review-10e3faa4cbb66d60", "version": "1", "name": "Snorkel-reviewed task review-10e3faa4cbb66d60", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/388" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1108e91a04568719-v1", "taskId": "review-1108e91a04568719", "version": "1", "name": "Snorkel-reviewed task review-1108e91a04568719", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/310" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-16018f2ccaa8e791-v1", "taskId": "review-16018f2ccaa8e791", "version": "1", "name": "506-lindblad-optimal-control", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-161974e3367cb34d-v1", "taskId": "review-161974e3367cb34d", "version": "1", "name": "aqs-delayed-rejection-reverse-alpha-tape", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1958d0e6-v1", "taskId": "review-1958d0e6", "version": "1", "name": "Snorkel-reviewed task review-1958d0e6", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1e7cd5f2fcce2c96-v1", "taskId": "review-1e7cd5f2fcce2c96", "version": "1", "name": "Snorkel-reviewed task review-1e7cd5f2fcce2c96", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/415" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-207a20238cebd242-v1", "taskId": "review-207a20238cebd242", "version": "1", "name": "Snorkel-reviewed task review-207a20238cebd242", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/186" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-227a749bd5e933a8-v1", "taskId": "review-227a749bd5e933a8", "version": "1", "name": "cluster-forge-nbody-integrator", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-23ae94fd5fbd496f-v1", "taskId": "review-23ae94fd5fbd496f", "version": "1", "name": "Snorkel-reviewed task review-23ae94fd5fbd496f", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/155" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2400093057d93f12-v1", "taskId": "review-2400093057d93f12", "version": "1", "name": "Snorkel-reviewed task review-2400093057d93f12", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/428" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-266f0a625eeee32b-v1", "taskId": "review-266f0a625eeee32b", "version": "1", "name": "python-hydrology-rating-board-reconcile", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-27553927a87a2b2b-v1", "taskId": "review-27553927a87a2b2b", "version": "1", "name": "Snorkel-reviewed task review-27553927a87a2b2b", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/243" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-28f467262fcb63a9-v1", "taskId": "review-28f467262fcb63a9", "version": "1", "name": "Snorkel-reviewed task review-28f467262fcb63a9", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-29ce6920-v1", "taskId": "review-29ce6920", "version": "1", "name": "Snorkel-reviewed task review-29ce6920", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2c63fe9e6bbccddc-v1", "taskId": "review-2c63fe9e6bbccddc", "version": "1", "name": "go-arrhenius-batch-invert", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-2ed49d470728a22a-v1", "taskId": "review-2ed49d470728a22a", "version": "1", "name": "Snorkel-reviewed task review-2ed49d470728a22a", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/414" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-33162d3c9e2c56c2-v1", "taskId": "review-33162d3c9e2c56c2", "version": "1", "name": "Snorkel-reviewed task review-33162d3c9e2c56c2", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/329" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3429234ae120a53e-v1", "taskId": "review-3429234ae120a53e", "version": "1", "name": "Snorkel-reviewed task review-3429234ae120a53e", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/157" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-371f0cf9ffde5c79-v1", "taskId": "review-371f0cf9ffde5c79", "version": "1", "name": "python-hydrology-rating-board-reconcile", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-38b4f33efbafa79f-v1", "taskId": "review-38b4f33efbafa79f", "version": "1", "name": "composite-laminate-response-go", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3acab3984f345e41-v1", "taskId": "review-3acab3984f345e41", "version": "1", "name": "karst-pressure-balance", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3b4106f6-v1", "taskId": "review-3b4106f6", "version": "1", "name": "Snorkel-reviewed task review-3b4106f6", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3bdd2a68295b770e-v1", "taskId": "review-3bdd2a68295b770e", "version": "1", "name": "Snorkel-reviewed task review-3bdd2a68295b770e", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/403" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3c6b4144aa1bb171-v1", "taskId": "review-3c6b4144aa1bb171", "version": "1", "name": "thermal-vacuum-test-article-heat-flux-reconstructor", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3dce264b3190e928-v1", "taskId": "review-3dce264b3190e928", "version": "1", "name": "Snorkel-reviewed task review-3dce264b3190e928", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 4, "p": 0, "lo": 0, "hi": 0.4898908364545973, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.4898908364545973, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/164" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-41e27bff6fae9901-v1", "taskId": "review-41e27bff6fae9901", "version": "1", "name": "neb-hybrid", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-46777564f4cb9119-v1", "taskId": "review-46777564f4cb9119", "version": "1", "name": "mycelial-front-atlas", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-467c7428d8e11fbc-v1", "taskId": "review-467c7428d8e11fbc", "version": "1", "name": "Snorkel-reviewed task review-467c7428d8e11fbc", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-47492406-v1", "taskId": "review-47492406", "version": "1", "name": "Snorkel-reviewed task review-47492406", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/216" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-477a2258cb280593-v1", "taskId": "review-477a2258cb280593", "version": "1", "name": "tolerance-zone-form-evaluator", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-496d16c3b6ce2377-v1", "taskId": "review-496d16c3b6ce2377", "version": "1", "name": "cathode-stripping-v4", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4a0a179714d79252-v1", "taskId": "review-4a0a179714d79252", "version": "1", "name": "Snorkel-reviewed task review-4a0a179714d79252", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/221" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4aff9b764db687e9-v1", "taskId": "review-4aff9b764db687e9", "version": "1", "name": "thermoacoustic-eigenmode-solver-cpp", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4c7bf988dde81875-v1", "taskId": "review-4c7bf988dde81875", "version": "1", "name": "Snorkel-reviewed task review-4c7bf988dde81875", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/279" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4cd8ddb545042876-v1", "taskId": "review-4cd8ddb545042876", "version": "1", "name": "rust-popgen-fst-scan", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5338a1c9241a9064-v1", "taskId": "review-5338a1c9241a9064", "version": "1", "name": "Snorkel-reviewed task review-5338a1c9241a9064", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T033956Z__45800f16__accept" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-5500027766d80265-v1", "taskId": "review-5500027766d80265", "version": "1", "name": "Snorkel-reviewed task review-5500027766d80265", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 15, "p": 0.13333333333333333, "lo": 0.03736125910306537, "hi": 0.37881982832322364, "tasks": 1, "macro": 0.13333333333333333, "macroLo": 0.03736125910306537, "macroHi": 0.37881982832322364, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 15, "p": 0, "lo": 0, "hi": 0.20388330103584862, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.20388330103584862, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/266" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-550d6a532a01b966-v1", "taskId": "review-550d6a532a01b966", "version": "1", "name": "Snorkel-reviewed task review-550d6a532a01b966", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 7, "n": 10, "p": 0.7, "lo": 0.39677814746114537, "hi": 0.8922087325936989, "tasks": 1, "macro": 0.7, "macroLo": 0.39677814746114537, "macroHi": 0.8922087325936989, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/356" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5b3a4033ef333e51-v1", "taskId": "review-5b3a4033ef333e51", "version": "1", "name": "Snorkel-reviewed task review-5b3a4033ef333e51", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-22T155653Z__70509683__accept" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5d68c8fad9dc6835-v1", "taskId": "review-5d68c8fad9dc6835", "version": "1", "name": "Snorkel-reviewed task review-5d68c8fad9dc6835", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/20260702T224038Z__1868c114__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-5dee009434fc219b-v1", "taskId": "review-5dee009434fc219b", "version": "1", "name": "neutral-delay-logistic-convergence", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 7, "n": 10, "p": 0.7, "lo": 0.39677814746114537, "hi": 0.8922087325936989, "tasks": 1, "macro": 0.7, "macroLo": 0.39677814746114537, "macroHi": 0.8922087325936989, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 8, "n": 10, "p": 0.8, "lo": 0.4901624715366418, "hi": 0.9433178485456248, "tasks": 1, "macro": 0.8, "macroLo": 0.4901624715366418, "macroHi": 0.9433178485456248, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-61202127a84e7871-v1", "taskId": "review-61202127a84e7871", "version": "1", "name": "gin-cpp-tide-residual-spectra-task", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6725958240f8f986-v1", "taskId": "review-6725958240f8f986", "version": "1", "name": "vaccine-cold-chain-outage-recourse", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-690137fd9ef8cd31-v1", "taskId": "review-690137fd9ef8cd31", "version": "1", "name": "r-sturm-root-counter", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-697a15ef0032ab79-v1", "taskId": "review-697a15ef0032ab79", "version": "1", "name": "Snorkel-reviewed task review-697a15ef0032ab79", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/417" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6a2f3d13-v1", "taskId": "review-6a2f3d13", "version": "1", "name": "Snorkel-reviewed task review-6a2f3d13", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6e68a5da65b3fa4b-v1", "taskId": "review-6e68a5da65b3fa4b", "version": "1", "name": "Snorkel-reviewed task review-6e68a5da65b3fa4b", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6e75d22d0a2aebc7-v1", "taskId": "review-6e75d22d0a2aebc7", "version": "1", "name": "Snorkel-reviewed task review-6e75d22d0a2aebc7", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/438" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6e830bd5eead1e91-v1", "taskId": "review-6e830bd5eead1e91", "version": "1", "name": "lease-quorum-replicated-log-replay", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6ed24c71f1be22cb-v1", "taskId": "review-6ed24c71f1be22cb", "version": "1", "name": "Snorkel-reviewed task review-6ed24c71f1be22cb", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-15T112656Z__343318d6__accept" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6f47f180a8fd4298-v1", "taskId": "review-6f47f180a8fd4298", "version": "1", "name": "linen-roa-certifier", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6f79bb218c1e97a6-v1", "taskId": "review-6f79bb218c1e97a6", "version": "1", "name": "Snorkel-reviewed task review-6f79bb218c1e97a6", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 4, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T054541Z__9be9543c__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-70cbd867fd524c19-v1", "taskId": "review-70cbd867fd524c19", "version": "1", "name": "slope-yield-v2", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-729db547f14398f8-v1", "taskId": "review-729db547f14398f8", "version": "1", "name": "mycelial-front-atlas", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-79ddfafabc458dcf-v1", "taskId": "review-79ddfafabc458dcf", "version": "1", "name": "shellweave-radial-remap-go-v2-adaptive-moment-path", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7a1fee3b2f235c4b-v1", "taskId": "review-7a1fee3b2f235c4b", "version": "1", "name": "Snorkel-reviewed task review-7a1fee3b2f235c4b", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/277" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7b3dbe69-v1", "taskId": "review-7b3dbe69", "version": "1", "name": "Snorkel-reviewed task review-7b3dbe69", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-22T164731Z__7b3dbe69__accept" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7e462e5b536d2c9f-v1", "taskId": "review-7e462e5b536d2c9f", "version": "1", "name": "Snorkel-reviewed task review-7e462e5b536d2c9f", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/384" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-81c121087133efe2-v1", "taskId": "review-81c121087133efe2", "version": "1", "name": "Snorkel-reviewed task review-81c121087133efe2", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T034848Z__8b8f2c50__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-82cbb1ef69bf87a9-v1", "taskId": "review-82cbb1ef69bf87a9", "version": "1", "name": "Snorkel-reviewed task review-82cbb1ef69bf87a9", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/294" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-82d8246f6227712a-v1", "taskId": "review-82d8246f6227712a", "version": "1", "name": "replacement-maxbias-breakdown-certificate", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 3, "p": 0.6666666666666666, "lo": 0.20765960080204765, "hi": 0.9385080552796037, "tasks": 1, "macro": 0.6666666666666666, "macroLo": 0.20765960080204765, "macroHi": 0.9385080552796037, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-87c406270428c341-v1", "taskId": "review-87c406270428c341", "version": "1", "name": "Snorkel-reviewed task review-87c406270428c341", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T020722Z__6a9d7a55__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8898ad9cafba8772-v1", "taskId": "review-8898ad9cafba8772", "version": "1", "name": "Snorkel-reviewed task review-8898ad9cafba8772", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/153" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8b19c2c2-v1", "taskId": "review-8b19c2c2", "version": "1", "name": "Snorkel-reviewed task review-8b19c2c2", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8c9ee754759701bd-v1", "taskId": "review-8c9ee754759701bd", "version": "1", "name": "case698-dose-ledger-repair-cpp", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8d6135d99c7e3f94-v1", "taskId": "review-8d6135d99c7e3f94", "version": "1", "name": "Snorkel-reviewed task review-8d6135d99c7e3f94", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/233" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-95f00f0f5839ad85-v1", "taskId": "review-95f00f0f5839ad85", "version": "1", "name": "Snorkel-reviewed task review-95f00f0f5839ad85", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/359" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-99883b9d39b9d601-v1", "taskId": "review-99883b9d39b9d601", "version": "1", "name": "cluster-forge-nbody-integrator", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9b2ca93d48c0658f-v1", "taskId": "review-9b2ca93d48c0658f", "version": "1", "name": "Snorkel-reviewed task review-9b2ca93d48c0658f", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/162" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a1ec4dce1e698bca-v1", "taskId": "review-a1ec4dce1e698bca", "version": "1", "name": "Snorkel-reviewed task review-a1ec4dce1e698bca", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/401" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a30f28f10efbf454-v1", "taskId": "review-a30f28f10efbf454", "version": "1", "name": "Snorkel-reviewed task review-a30f28f10efbf454", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/293" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a619d54b24f65a96-v1", "taskId": "review-a619d54b24f65a96", "version": "1", "name": "Snorkel-reviewed task review-a619d54b24f65a96", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 4, "p": 0.25, "lo": 0.04558726080970055, "hi": 0.6993581574175981, "tasks": 1, "macro": 0.25, "macroLo": 0.04558726080970055, "macroHi": 0.6993581574175981, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a9598b0e04a695b6-v1", "taskId": "review-a9598b0e04a695b6", "version": "1", "name": "Snorkel-reviewed task review-a9598b0e04a695b6", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/362" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ab572ea1f9672c0d-v1", "taskId": "review-ab572ea1f9672c0d", "version": "1", "name": "Snorkel-reviewed task review-ab572ea1f9672c0d", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/136" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-abdddf065881e360-v1", "taskId": "review-abdddf065881e360", "version": "1", "name": "replacement-maxbias-breakdown-certificate", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-abfb20ca08362c79-v1", "taskId": "review-abfb20ca08362c79", "version": "1", "name": "enzyme-kinetics-recovery", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-adb2155ee40048b8-v1", "taskId": "review-adb2155ee40048b8", "version": "1", "name": "seismic-hypocenter-location", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-af9804f996324a19-v1", "taskId": "review-af9804f996324a19", "version": "1", "name": "Snorkel-reviewed task review-af9804f996324a19", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/184" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b22c2eb54617ecfd-v1", "taskId": "review-b22c2eb54617ecfd", "version": "1", "name": "greenwave", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b424a9776bf8b8ce-v1", "taskId": "review-b424a9776bf8b8ce", "version": "1", "name": "Snorkel-reviewed task review-b424a9776bf8b8ce", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/212" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b6c5a3df723f3f36-v1", "taskId": "review-b6c5a3df723f3f36", "version": "1", "name": "palladium-kernel-synthesis-drift", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-bcc1eb64ec7e5238-v1", "taskId": "review-bcc1eb64ec7e5238", "version": "1", "name": "Snorkel-reviewed task review-bcc1eb64ec7e5238", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/460" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-bcd814b12a39fee4-v1", "taskId": "review-bcd814b12a39fee4", "version": "1", "name": "Snorkel-reviewed task review-bcd814b12a39fee4", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/333" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-bd5655e671f3e54d-v1", "taskId": "review-bd5655e671f3e54d", "version": "1", "name": "vellumark-michaelis-menten-atlas", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x2, needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c1803ab760f77a73-v1", "taskId": "review-c1803ab760f77a73", "version": "1", "name": "Snorkel-reviewed task review-c1803ab760f77a73", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/375" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c24b7bdb484f393c-v1", "taskId": "review-c24b7bdb484f393c", "version": "1", "name": "Snorkel-reviewed task review-c24b7bdb484f393c", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/313" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c2d11d69a920097c-v1", "taskId": "review-c2d11d69a920097c", "version": "1", "name": "decimal128-quantize-invalid-preferred-exponent", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c32b16db4d2f64d0-v1", "taskId": "review-c32b16db4d2f64d0", "version": "1", "name": "Snorkel-reviewed task review-c32b16db4d2f64d0", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/408" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c4e36e1a39328b90-v1", "taskId": "review-c4e36e1a39328b90", "version": "1", "name": "Snorkel-reviewed task review-c4e36e1a39328b90", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/432" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c875f9607c39717f-v1", "taskId": "review-c875f9607c39717f", "version": "1", "name": "Snorkel-reviewed task review-c875f9607c39717f", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/276" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c8e8c202-v1", "taskId": "review-c8e8c202", "version": "1", "name": "Snorkel-reviewed task review-c8e8c202", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cbc316323a1068d5-v1", "taskId": "review-cbc316323a1068d5", "version": "1", "name": "Snorkel-reviewed task review-cbc316323a1068d5", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/357" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cdc3cb2c682a86a5-v1", "taskId": "review-cdc3cb2c682a86a5", "version": "1", "name": "degenerate-mode-spectrum", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d0881ed0c975cb52-v1", "taskId": "review-d0881ed0c975cb52", "version": "1", "name": "Snorkel-reviewed task review-d0881ed0c975cb52", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 3, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 4, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/369" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d222c47197a601e7-v1", "taskId": "review-d222c47197a601e7", "version": "1", "name": "fv-amr-reflux-sync-residual", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d35c83843f557c76-v1", "taskId": "review-d35c83843f557c76", "version": "1", "name": "Snorkel-reviewed task review-d35c83843f557c76", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/238" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d69dcc09aa568ee1-v1", "taskId": "review-d69dcc09aa568ee1", "version": "1", "name": "Snorkel-reviewed task review-d69dcc09aa568ee1", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d70682073ae41807-v1", "taskId": "review-d70682073ae41807", "version": "1", "name": "tbrain-emd-sift-decompose-cpp", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d740bc515a9b527e-v1", "taskId": "review-d740bc515a9b527e", "version": "1", "name": "slope-yield-v2", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d8437d87ff1eb546-v1", "taskId": "review-d8437d87ff1eb546", "version": "1", "name": "Snorkel-reviewed task review-d8437d87ff1eb546", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d8473951f87183f2-v1", "taskId": "review-d8473951f87183f2", "version": "1", "name": "sturm-sequence-real-root-count", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dad9ce3164ab6aaf-v1", "taskId": "review-dad9ce3164ab6aaf", "version": "1", "name": "Snorkel-reviewed task review-dad9ce3164ab6aaf", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-10T200041Z__4c7c372e__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-dfb877f9bd267d59-v1", "taskId": "review-dfb877f9bd267d59", "version": "1", "name": "cpp-geometric-measurement-cli", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dfbbf51107cbf803-v1", "taskId": "review-dfbbf51107cbf803", "version": "1", "name": "Snorkel-reviewed task review-dfbbf51107cbf803", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 4, "p": 0.25, "lo": 0.04558726080970055, "hi": 0.6993581574175981, "tasks": 1, "macro": 0.25, "macroLo": 0.04558726080970055, "macroHi": 0.6993581574175981, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-29T134333Z__7ad6ea86__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-dfc1160e8ab6f23b-v1", "taskId": "review-dfc1160e8ab6f23b", "version": "1", "name": "lease-quorum-log-recovery", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x4", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 15, "p": 0, "lo": 0, "hi": 0.20388330103584862, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.20388330103584862, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 15, "p": 0.2, "lo": 0.07047549346981563, "hi": 0.45185448715169363, "tasks": 1, "macro": 0.2, "macroLo": 0.07047549346981563, "macroHi": 0.45185448715169363, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e03218c9-v1", "taskId": "review-e03218c9", "version": "1", "name": "Snorkel-reviewed task review-e03218c9", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-e1a883e3a902805a-v1", "taskId": "review-e1a883e3a902805a", "version": "1", "name": "Snorkel-reviewed task review-e1a883e3a902805a", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/399" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e2bd53e777a49cdc-v1", "taskId": "review-e2bd53e777a49cdc", "version": "1", "name": "synthesize-css-encoder-from-parity-checks", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-e90bbced5e3e5edb-v1", "taskId": "review-e90bbced5e3e5edb", "version": "1", "name": "pair-heralding-rust-sizer", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f3dc20fb6f7df522-v1", "taskId": "review-f3dc20fb6f7df522", "version": "1", "name": "vaccine-cold-chain-stochastic-plan", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fcbc9a030e857334-v1", "taskId": "review-fcbc9a030e857334", "version": "1", "name": "permix-effective-medium-solver", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fccb0079ec7f2459-v1", "taskId": "review-fccb0079ec7f2459", "version": "1", "name": "Snorkel-reviewed task review-fccb0079ec7f2459", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/436" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fed730d1ae9dc266-v1", "taskId": "review-fed730d1ae9dc266", "version": "1", "name": "cluster-forge-nbody-integrator", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ffaf9ecf817d034e-v1", "taskId": "review-ffaf9ecf817d034e", "version": "1", "name": "vaccine-cold-chain-outage-recourse", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-c-sparsereport-milestones-v1", "taskId": "review-slug-c-sparsereport-milestones", "version": "1", "name": "c-sparsereport-milestones", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x4", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-complex-product-log-modulus-v1", "taskId": "review-slug-complex-product-log-modulus", "version": "1", "name": "complex-product-log-modulus", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-icao-isa-atmosphere-atlas-v1", "taskId": "review-slug-icao-isa-atmosphere-atlas", "version": "1", "name": "icao-isa-atmosphere-atlas", "domain": "Scientific computing", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-ring-damping-v1", "taskId": "review-slug-ring-damping", "version": "1", "name": "ring-damping", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "salbp-line-balancing-v1", "taskId": "salbp-line-balancing", "version": "1", "name": "salbp-line-balancing", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/salbp-line-balancing" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "santa-maria-bridge-scour-v1", "taskId": "santa-maria-bridge-scour", "version": "1", "name": "santa-maria-bridge-scour", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/126" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sequence-setup-weighted-tardiness-v1", "taskId": "sequence-setup-weighted-tardiness", "version": "1", "name": "sequence-setup-weighted-tardiness", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/sequence-setup-weighted-tardiness" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "set-cover-mutual-exclusion-v1", "taskId": "set-cover-mutual-exclusion", "version": "1", "name": "set-cover-mutual-exclusion", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/set-cover-mutual-exclusion" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "set-cover-redundant-double-cover-v1", "taskId": "set-cover-redundant-double-cover", "version": "1", "name": "set-cover-redundant-double-cover", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/optmodel/set-cover-redundant-double-cover" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sga-revenue-asymmetric-elasticity-panel-v1", "taskId": "sga-revenue-asymmetric-elasticity-panel", "version": "1", "name": "sga-revenue-asymmetric-elasticity-panel", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/sga-revenue-asymmetric-elasticity-panel" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "shrinking-cross-section-sdf-v1", "taskId": "shrinking-cross-section-sdf", "version": "1", "name": "shrinking-cross-section-sdf", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/shrinking-cross-section-sdf" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "single-machine-weighted-completion-time-v1", "taskId": "single-machine-weighted-completion-time", "version": "1", "name": "single-machine-weighted-completion-time", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/single-machine-weighted-completion-time" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sleepstudy-serial-correlation-ar1-v1", "taskId": "sleepstudy-serial-correlation-ar1", "version": "1", "name": "sleepstudy-serial-correlation-ar1", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/sleepstudy-serial-correlation-ar1" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spline-shaft-three-section-v1", "taskId": "spline-shaft-three-section", "version": "1", "name": "spline-shaft-three-section", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 9, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 9, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/122" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spur-gear-hub-bore-v1", "taskId": "spur-gear-hub-bore", "version": "1", "name": "spur-gear-hub-bore", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 10, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 10, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/123" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spur-gear-stepped-shaft-v1", "taskId": "spur-gear-stepped-shaft", "version": "1", "name": "spur-gear-stepped-shaft", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/120" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "star-class-size-rct-v1", "taskId": "star-class-size-rct", "version": "1", "name": "star-class-size-rct", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/star-class-size-rct" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sumhes-penn-world-growth-regression-v1", "taskId": "sumhes-penn-world-growth-regression", "version": "1", "name": "sumhes-penn-world-growth-regression", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/sumhes-penn-world-growth-regression" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "teoh-welch-wong-1998-ipo-earnings-management-returns-v1", "taskId": "teoh-welch-wong-1998-ipo-earnings-management-returns", "version": "1", "name": "teoh-welch-wong-1998-ipo-earnings-management-returns", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/teoh-welch-wong-1998-ipo-earnings-management-returns" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "travelmode-multinomial-choice-v1", "taskId": "travelmode-multinomial-choice", "version": "1", "name": "travelmode-multinomial-choice", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/travelmode-multinomial-choice" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "two-machine-flowshop-v1", "taskId": "two-machine-flowshop", "version": "1", "name": "two-machine-flowshop", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/two-machine-flowshop" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "value-momentum-everywhere-v1", "taskId": "value-momentum-everywhere", "version": "1", "name": "value-momentum-everywhere", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/finpaper/value-momentum-everywhere" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vehicle-route-reload-overhead-risk-v1", "taskId": "vehicle-route-reload-overhead-risk", "version": "1", "name": "vehicle-route-reload-overhead-risk", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/optmodel/vehicle-route-reload-overhead-risk" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "visible-lattice-points-in-rational-region-v1", "taskId": "visible-lattice-points-in-rational-region", "version": "1", "name": "visible-lattice-points-in-rational-region", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/origami/visible-lattice-points-in-rational-region" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wagepan-system-gmm-v1", "taskId": "wagepan-system-gmm", "version": "1", "name": "wagepan-system-gmm", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/wagepan-system-gmm" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wages-static-panel-hausman-v1", "taskId": "wages-static-panel-hausman", "version": "1", "name": "wages-static-panel-hausman", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/rdatasets/wages-static-panel-hausman" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wall-jet-scour-scaling-v1", "taskId": "wall-jet-scour-scaling", "version": "1", "name": "wall-jet-scour-scaling", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/126" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "woven-system-reduced-order-model-v1", "taskId": "woven-system-reduced-order-model", "version": "1", "name": "woven-system-reduced-order-model", "domain": "Scientific computing", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/126" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "xiangqi-perft-flying-general-cannon-screen-v1", "taskId": "xiangqi-perft-flying-general-cannon-screen", "version": "1", "name": "xiangqi-perft-flying-general-cannon-screen", "domain": "Scientific computing", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/gambit/xiangqi-perft-flying-general-cannon-screen" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "accesscontrol-nested-group-permission-depth-v1", "taskId": "accesscontrol-nested-group-permission-depth", "version": "1", "name": "accesscontrol-nested-group-permission-depth", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lattice/accesscontrol-nested-group-permission-depth" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "arcslip-go-archive-extractor-v1", "taskId": "arcslip-go-archive-extractor", "version": "1", "name": "arcslip-go-archive-extractor", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/arcslip-go-archive-extractor" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "c-uaf-idcval-v1", "taskId": "c-uaf-idcval", "version": "1", "name": "c-uaf-idcval", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/c-uaf-idcval" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chunkward-rust-http-chunked-decoder-v1", "taskId": "chunkward-rust-http-chunked-decoder", "version": "1", "name": "chunkward-rust-http-chunked-decoder", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/chunkward-rust-http-chunked-decoder" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cmd-injection-fix-v1", "taskId": "cmd-injection-fix", "version": "1", "name": "cmd-injection-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/1" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "compensated-fp-reduction-x86-v1", "taskId": "compensated-fp-reduction-x86", "version": "1", "name": "compensated-fp-reduction-x86", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/compensated-fp-reduction-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "conn-pool-uaf-fix-v1", "taskId": "conn-pool-uaf-fix", "version": "1", "name": "conn-pool-uaf-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/2" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cosecbor-c-canonical-encoder-v1", "taskId": "cosecbor-c-canonical-encoder", "version": "1", "name": "cosecbor-c-canonical-encoder", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/cosecbor-c-canonical-encoder" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "crlward-go-crl-revocation-audit-v1", "taskId": "crlward-go-crl-revocation-audit", "version": "1", "name": "crlward-go-crl-revocation-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/crlward-go-crl-revocation-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "csv-field-overflow-fix-v1", "taskId": "csv-field-overflow-fix", "version": "1", "name": "csv-field-overflow-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/3" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dkimcanon-c-message-canonicalization-v1", "taskId": "dkimcanon-c-message-canonicalization", "version": "1", "name": "dkimcanon-c-message-canonicalization", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/dkimcanon-c-message-canonicalization" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dmarcgate-go-alignment-policy-audit-v1", "taskId": "dmarcgate-go-alignment-policy-audit", "version": "1", "name": "dmarcgate-go-alignment-policy-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/dmarcgate-go-alignment-policy-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "entitlement-service-v1", "taskId": "entitlement-service", "version": "1", "name": "entitlement-service", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/entitlement-service" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "format-string-fix-v1", "taskId": "format-string-fix", "version": "1", "name": "format-string-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/4" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gitwire-go-transport-argv-audit-v1", "taskId": "gitwire-go-transport-argv-audit", "version": "1", "name": "gitwire-go-transport-argv-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/gitwire-go-transport-argv-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gltf-asset-validator-v1", "taskId": "gltf-asset-validator", "version": "1", "name": "gltf-asset-validator", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/gltf-asset-validator" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-archive-extract-traversal-debug-v1", "taskId": "go-archive-extract-traversal-debug", "version": "1", "name": "go-archive-extract-traversal-debug", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/go-archive-extract-traversal-debug" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-authz-bypass-gateway-v1", "taskId": "go-authz-bypass-gateway", "version": "1", "name": "go-authz-bypass-gateway", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/go-authz-bypass-gateway" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-bech32m-segwit-address-audit-v1", "taskId": "go-bech32m-segwit-address-audit", "version": "1", "name": "go-bech32m-segwit-address-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/go-bech32m-segwit-address-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-cbor-canonical-encode-audit-v1", "taskId": "go-cbor-canonical-encode-audit", "version": "1", "name": "go-cbor-canonical-encode-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/go-cbor-canonical-encode-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-gguf-parser-audit-v1", "taskId": "go-gguf-parser-audit", "version": "1", "name": "go-gguf-parser-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/go-gguf-parser-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-http1-smuggling-audit-v1", "taskId": "go-http1-smuggling-audit", "version": "1", "name": "go-http1-smuggling-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/go-http1-smuggling-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-jwt-resource-exhaustion-audit-v1", "taskId": "go-jwt-resource-exhaustion-audit", "version": "1", "name": "go-jwt-resource-exhaustion-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/go-jwt-resource-exhaustion-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-multidnsd-resource-audit-v1", "taskId": "go-multidnsd-resource-audit", "version": "1", "name": "go-multidnsd-resource-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/go-multidnsd-resource-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-pem-quadratic-decode-debug-v1", "taskId": "go-pem-quadratic-decode-debug", "version": "1", "name": "go-pem-quadratic-decode-debug", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/go-pem-quadratic-decode-debug" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-rfc6979-deterministic-ecdsa-v1", "taskId": "go-rfc6979-deterministic-ecdsa", "version": "1", "name": "go-rfc6979-deterministic-ecdsa", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/go-rfc6979-deterministic-ecdsa" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-sigv4-canonical-audit-v1", "taskId": "go-sigv4-canonical-audit", "version": "1", "name": "go-sigv4-canonical-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/go-sigv4-canonical-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-x509-name-constraint-audit-v1", "taskId": "go-x509-name-constraint-audit", "version": "1", "name": "go-x509-name-constraint-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/go-x509-name-constraint-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-zipslip-archive-extractor-v1", "taskId": "go-zipslip-archive-extractor", "version": "1", "name": "go-zipslip-archive-extractor", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/go-zipslip-archive-extractor" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "js-media-param-injection-audit-v1", "taskId": "js-media-param-injection-audit", "version": "1", "name": "js-media-param-injection-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/js-media-param-injection-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "js-rfc5322-address-parser-v1", "taskId": "js-rfc5322-address-parser", "version": "1", "name": "js-rfc5322-address-parser", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/js-rfc5322-address-parser" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kvstore-mvcc-v1", "taskId": "kvstore-mvcc", "version": "1", "name": "kvstore-mvcc", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/kvstore-mvcc" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "length-truncation-fix-v1", "taskId": "length-truncation-fix", "version": "1", "name": "length-truncation-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/5" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lfsr-selfsync-scrambler-x86-v1", "taskId": "lfsr-selfsync-scrambler-x86", "version": "1", "name": "lfsr-selfsync-scrambler-x86", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/lfsr-selfsync-scrambler-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lockwarden-v1", "taskId": "lockwarden", "version": "1", "name": "lockwarden", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/lockwarden" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mailflow-session-relay-v1", "taskId": "mailflow-session-relay", "version": "1", "name": "mailflow-session-relay", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mwgate-ts-trusted-header-audit-v1", "taskId": "mwgate-ts-trusted-header-audit", "version": "1", "name": "mwgate-ts-trusted-header-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/mwgate-ts-trusted-header-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pak-extractor-v1", "taskId": "pak-extractor", "version": "1", "name": "pak-extractor", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/pak-extractor" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "papershare-v1", "taskId": "papershare", "version": "1", "name": "papershare", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/papershare" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "parqcheck-java-avro-deser-audit-v1", "taskId": "parqcheck-java-avro-deser-audit", "version": "1", "name": "parqcheck-java-avro-deser-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/parqcheck-java-avro-deser-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "php-phpspreadsheet-ssrf-audit-v1", "taskId": "php-phpspreadsheet-ssrf-audit", "version": "1", "name": "php-phpspreadsheet-ssrf-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/php-phpspreadsheet-ssrf-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pluglock-java-plugin-unzip-audit-v1", "taskId": "pluglock-java-plugin-unzip-audit", "version": "1", "name": "pluglock-java-plugin-unzip-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/pluglock-java-plugin-unzip-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "postwick-v1", "taskId": "postwick", "version": "1", "name": "postwick", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/postwick" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "protocol-parser-fix-v1", "taskId": "protocol-parser-fix", "version": "1", "name": "protocol-parser-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/6" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "provenance-crossrelation-agent-attribution-v1", "taskId": "provenance-crossrelation-agent-attribution", "version": "1", "name": "provenance-crossrelation-agent-attribution", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lattice/provenance-crossrelation-agent-attribution" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "quota-ledger-service-v1", "taskId": "quota-ledger-service", "version": "1", "name": "quota-ledger-service", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/quota-ledger-service" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "recursion-depth-limit-fix-v1", "taskId": "recursion-depth-limit-fix", "version": "1", "name": "recursion-depth-limit-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/7" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0035c2516745aa41-v1", "taskId": "review-0035c2516745aa41", "version": "1", "name": "Snorkel-reviewed task review-0035c2516745aa41", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/208" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0186fa7cb35661fb-v1", "taskId": "review-0186fa7cb35661fb", "version": "1", "name": "Snorkel-reviewed task review-0186fa7cb35661fb", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/250" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-02c74cb81f861416-v1", "taskId": "review-02c74cb81f861416", "version": "1", "name": "hmac-or-gate-sealed-archive-decrypt-custody-audit", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-030d08a6fd455197-v1", "taskId": "review-030d08a6fd455197", "version": "1", "name": "classward-rust-stackmap-admission-witness", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-037f69b8a5560514-v1", "taskId": "review-037f69b8a5560514", "version": "1", "name": "Snorkel-reviewed task review-037f69b8a5560514", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/282" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-08177af29a8a7af1-v1", "taskId": "review-08177af29a8a7af1", "version": "1", "name": "Snorkel-reviewed task review-08177af29a8a7af1", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/309" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0f77bf242f2ac69e-v1", "taskId": "review-0f77bf242f2ac69e", "version": "1", "name": "Snorkel-reviewed task review-0f77bf242f2ac69e", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/441" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-126d3a9890db4fa9-v1", "taskId": "review-126d3a9890db4fa9", "version": "1", "name": "Snorkel-reviewed task review-126d3a9890db4fa9", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/391" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-149be51e67ce5ecb-v1", "taskId": "review-149be51e67ce5ecb", "version": "1", "name": "Snorkel-reviewed task review-149be51e67ce5ecb", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/422" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-17c97007a7eed342-v1", "taskId": "review-17c97007a7eed342", "version": "1", "name": "Snorkel-reviewed task review-17c97007a7eed342", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/267" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-18050a0f9d2f3a5a-v1", "taskId": "review-18050a0f9d2f3a5a", "version": "1", "name": "was-scan-policy-cookie-drift", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-18abb64a7dc0675c-v1", "taskId": "review-18abb64a7dc0675c", "version": "1", "name": "Snorkel-reviewed task review-18abb64a7dc0675c", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1c9a6f6524f14a9d-v1", "taskId": "review-1c9a6f6524f14a9d", "version": "1", "name": "Snorkel-reviewed task review-1c9a6f6524f14a9d", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/139" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1d58c0cd74185094-v1", "taskId": "review-1d58c0cd74185094", "version": "1", "name": "Snorkel-reviewed task review-1d58c0cd74185094", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/299" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1f9310d3fd232145-v1", "taskId": "review-1f9310d3fd232145", "version": "1", "name": "northwind-files-incident-report", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-205bffc808526e0d-v1", "taskId": "review-205bffc808526e0d", "version": "1", "name": "Snorkel-reviewed task review-205bffc808526e0d", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/420" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-233e075e083e0789-v1", "taskId": "review-233e075e083e0789", "version": "1", "name": "Snorkel-reviewed task review-233e075e083e0789", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/142" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-24f19b8f8d19a6c9-v1", "taskId": "review-24f19b8f8d19a6c9", "version": "1", "name": "Snorkel-reviewed task review-24f19b8f8d19a6c9", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/361" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-27aa266b74303b06-v1", "taskId": "review-27aa266b74303b06", "version": "1", "name": "Snorkel-reviewed task review-27aa266b74303b06", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/204" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-284b084afb6c5611-v1", "taskId": "review-284b084afb6c5611", "version": "1", "name": "cragford-loadout-freeze-seal-go", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2854e1df2e64a088-v1", "taskId": "review-2854e1df2e64a088", "version": "1", "name": "Snorkel-reviewed task review-2854e1df2e64a088", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/147" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-291baee5cfc30feb-v1", "taskId": "review-291baee5cfc30feb", "version": "1", "name": "Snorkel-reviewed task review-291baee5cfc30feb", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/332" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2a358c2768c80fb2-v1", "taskId": "review-2a358c2768c80fb2", "version": "1", "name": "Snorkel-reviewed task review-2a358c2768c80fb2", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/410" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-2b21ec902d5da785-v1", "taskId": "review-2b21ec902d5da785", "version": "1", "name": "Snorkel-reviewed task review-2b21ec902d5da785", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/281" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2c66d77d2df99605-v1", "taskId": "review-2c66d77d2df99605", "version": "1", "name": "hidden-group-order-scalar-recovery-rust", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2d28e154db7fd568-v1", "taskId": "review-2d28e154db7fd568", "version": "1", "name": "Snorkel-reviewed task review-2d28e154db7fd568", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/340" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2efbb812acea5d01-v1", "taskId": "review-2efbb812acea5d01", "version": "1", "name": "Snorkel-reviewed task review-2efbb812acea5d01", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/285" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-2f7b68a7fd0911fe-v1", "taskId": "review-2f7b68a7fd0911fe", "version": "1", "name": "Snorkel-reviewed task review-2f7b68a7fd0911fe", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/295" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-31b6ed4c-v1", "taskId": "review-31b6ed4c", "version": "1", "name": "Snorkel-reviewed task review-31b6ed4c", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3642e903d34bbcd4-v1", "taskId": "review-3642e903d34bbcd4", "version": "1", "name": "Snorkel-reviewed task review-3642e903d34bbcd4", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 3, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/167" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-36c36db89046cb22-v1", "taskId": "review-36c36db89046cb22", "version": "1", "name": "firmware-release-security-adjudication", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-378b77d3a879a3a1-v1", "taskId": "review-378b77d3a879a3a1", "version": "1", "name": "Snorkel-reviewed task review-378b77d3a879a3a1", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T033448Z__5813c95d__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-38602a4d87dd01ee-v1", "taskId": "review-38602a4d87dd01ee", "version": "1", "name": "Snorkel-reviewed task review-38602a4d87dd01ee", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/328" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3c4025ecea9fef68-v1", "taskId": "review-3c4025ecea9fef68", "version": "1", "name": "Snorkel-reviewed task review-3c4025ecea9fef68", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/187" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3d2db394644e9fdf-v1", "taskId": "review-3d2db394644e9fdf", "version": "1", "name": "Snorkel-reviewed task review-3d2db394644e9fdf", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/385" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-401b796f445022e0-v1", "taskId": "review-401b796f445022e0", "version": "1", "name": "jwks-kid-algorithm-confusion", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-40eef911548f2a0d-v1", "taskId": "review-40eef911548f2a0d", "version": "1", "name": "Snorkel-reviewed task review-40eef911548f2a0d", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-30T003215Z__22d670a4__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-41eed6fb2a9ccef0-v1", "taskId": "review-41eed6fb2a9ccef0", "version": "1", "name": "recover-lua-aes-gcm-escrow", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 8, "n": 10, "p": 0.8, "lo": 0.4901624715366418, "hi": 0.9433178485456248, "tasks": 1, "macro": 0.8, "macroLo": 0.4901624715366418, "macroHi": 0.9433178485456248, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-44111f4923a86e69-v1", "taskId": "review-44111f4923a86e69", "version": "1", "name": "Snorkel-reviewed task review-44111f4923a86e69", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/210" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-44e7db992f16c6df-v1", "taskId": "review-44e7db992f16c6df", "version": "1", "name": "Snorkel-reviewed task review-44e7db992f16c6df", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/395" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-481a52d8a80061ea-v1", "taskId": "review-481a52d8a80061ea", "version": "1", "name": "Snorkel-reviewed task review-481a52d8a80061ea", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4b023432-v1", "taskId": "review-4b023432", "version": "1", "name": "Snorkel-reviewed task review-4b023432", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4b69307aa78150fa-v1", "taskId": "review-4b69307aa78150fa", "version": "1", "name": "Snorkel-reviewed task review-4b69307aa78150fa", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/269" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4c4689055c972086-v1", "taskId": "review-4c4689055c972086", "version": "1", "name": "Snorkel-reviewed task review-4c4689055c972086", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/236" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4cdcef10c9344e3f-v1", "taskId": "review-4cdcef10c9344e3f", "version": "1", "name": "Snorkel-reviewed task review-4cdcef10c9344e3f", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/448" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4ec6b9d96161a30c-v1", "taskId": "review-4ec6b9d96161a30c", "version": "1", "name": "automaton-inverse", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 7, "p": 0.42857142857142855, "lo": 0.15821985525146975, "hi": 0.7495416354723428, "tasks": 1, "macro": 0.42857142857142855, "macroLo": 0.15821985525146975, "macroHi": 0.7495416354723428, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-51661814318e4c9e-v1", "taskId": "review-51661814318e4c9e", "version": "1", "name": "Snorkel-reviewed task review-51661814318e4c9e", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/169" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5391cd51fc1d4030-v1", "taskId": "review-5391cd51fc1d4030", "version": "1", "name": "chainward-x509-validation", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 15, "p": 0.6666666666666666, "lo": 0.41713547738519774, "hi": 0.8482367556028526, "tasks": 1, "macro": 0.6666666666666666, "macroLo": 0.41713547738519774, "macroHi": 0.8482367556028526, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5483f2641d761199-v1", "taskId": "review-5483f2641d761199", "version": "1", "name": "session-ward-desk", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-54b8f54f767e89bb-v1", "taskId": "review-54b8f54f767e89bb", "version": "1", "name": "Snorkel-reviewed task review-54b8f54f767e89bb", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/229" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-57881d7d5dae2c1b-v1", "taskId": "review-57881d7d5dae2c1b", "version": "1", "name": "tbench-task", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-58059ffc52d5e055-v1", "taskId": "review-58059ffc52d5e055", "version": "1", "name": "sshd-match-block-effective-policy-ledger", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-58133a92c9deeb38-v1", "taskId": "review-58133a92c9deeb38", "version": "1", "name": "nocturne-ed25519-replay-registry", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1, needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5ade0897-v1", "taskId": "review-5ade0897", "version": "1", "name": "Snorkel-reviewed task review-5ade0897", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5b392f7d9604d37c-v1", "taskId": "review-5b392f7d9604d37c", "version": "1", "name": "Snorkel-reviewed task review-5b392f7d9604d37c", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 3, "p": 0.3333333333333333, "lo": 0.06149194472039615, "hi": 0.7923403991979523, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.06149194472039615, "macroHi": 0.7923403991979523, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/177" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5d386b60ce4f5683-v1", "taskId": "review-5d386b60ce4f5683", "version": "1", "name": "fleet-svid-federation-policy", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-5e0c1c343e13cea9-v1", "taskId": "review-5e0c1c343e13cea9", "version": "1", "name": "classward-rust-stackmap-admission-witness", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-5f684c5d8b8bfdc6-v1", "taskId": "review-5f684c5d8b8bfdc6", "version": "1", "name": "Snorkel-reviewed task review-5f684c5d8b8bfdc6", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6059fa630bd694e8-v1", "taskId": "review-6059fa630bd694e8", "version": "1", "name": "Snorkel-reviewed task review-6059fa630bd694e8", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 10, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 10, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 10, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 10, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-61a7da17711a8011-v1", "taskId": "review-61a7da17711a8011", "version": "1", "name": "Snorkel-reviewed task review-61a7da17711a8011", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/160" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-644053fc049e1c96-v1", "taskId": "review-644053fc049e1c96", "version": "1", "name": "Snorkel-reviewed task review-644053fc049e1c96", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-04T231423Z__590732be__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6506bb96440f414d-v1", "taskId": "review-6506bb96440f414d", "version": "1", "name": "dsa-nonce-reuse-keyrecovery", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-676705ed3a8fe260-v1", "taskId": "review-676705ed3a8fe260", "version": "1", "name": "Snorkel-reviewed task review-676705ed3a8fe260", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/172" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-676904c996e70641-v1", "taskId": "review-676904c996e70641", "version": "1", "name": "Snorkel-reviewed task review-676904c996e70641", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T013742Z__9ea84243__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-68c7e00aa070274f-v1", "taskId": "review-68c7e00aa070274f", "version": "1", "name": "Snorkel-reviewed task review-68c7e00aa070274f", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-30T165921Z__902b0338__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6cde584bf6f2d6f9-v1", "taskId": "review-6cde584bf6f2d6f9", "version": "1", "name": "Snorkel-reviewed task review-6cde584bf6f2d6f9", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/394" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6d5a521598c5b969-v1", "taskId": "review-6d5a521598c5b969", "version": "1", "name": "Snorkel-reviewed task review-6d5a521598c5b969", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/416" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6e6ebde46db706df-v1", "taskId": "review-6e6ebde46db706df", "version": "1", "name": "keyring-policy-auditor-medium-restored", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-709c83cb10d627bf-v1", "taskId": "review-709c83cb10d627bf", "version": "1", "name": "bell-vault-custody-gate", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-75627ad59aca1d13-v1", "taskId": "review-75627ad59aca1d13", "version": "1", "name": "provenance-worker", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-75f3c4831e4a2dd6-v1", "taskId": "review-75f3c4831e4a2dd6", "version": "1", "name": "depversion-threat-enforcer-cpp", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-793073c90bf60a15-v1", "taskId": "review-793073c90bf60a15", "version": "1", "name": "Snorkel-reviewed task review-793073c90bf60a15", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/453" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7a16ffd6c1baf084-v1", "taskId": "review-7a16ffd6c1baf084", "version": "1", "name": "reverse-vm-keygen", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7cf6ad942044739a-v1", "taskId": "review-7cf6ad942044739a", "version": "1", "name": "signed-plugin-trust-rebind", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7d76c6df84d78165-v1", "taskId": "review-7d76c6df84d78165", "version": "1", "name": "dsu-union-find-connectivity-coupled-digest-x86_v4", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7fb70d9a303c19b3-v1", "taskId": "review-7fb70d9a303c19b3", "version": "1", "name": "ixl-north-privilege-bypass", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7fd5e2fbb1ce7897-v1", "taskId": "review-7fd5e2fbb1ce7897", "version": "1", "name": "Snorkel-reviewed task review-7fd5e2fbb1ce7897", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/215" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-81b445c5060dd5ee-v1", "taskId": "review-81b445c5060dd5ee", "version": "1", "name": "Snorkel-reviewed task review-81b445c5060dd5ee", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/431" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-81bf888c2712fd13-v1", "taskId": "review-81bf888c2712fd13", "version": "1", "name": "java-ffmpeg-signing-gateway-h2", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-82caf9657274b2b5-v1", "taskId": "review-82caf9657274b2b5", "version": "1", "name": "Snorkel-reviewed task review-82caf9657274b2b5", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/365" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8888f36b17016eca-v1", "taskId": "review-8888f36b17016eca", "version": "1", "name": "Snorkel-reviewed task review-8888f36b17016eca", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T010813Z__541a9ad9__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8ec6a0622cea136d-v1", "taskId": "review-8ec6a0622cea136d", "version": "1", "name": "Snorkel-reviewed task review-8ec6a0622cea136d", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/183" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8f2542e3d74b77bc-v1", "taskId": "review-8f2542e3d74b77bc", "version": "1", "name": "Snorkel-reviewed task review-8f2542e3d74b77bc", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T015412Z__0f6c83ec__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8f5eba26-v1", "taskId": "review-8f5eba26", "version": "1", "name": "Snorkel-reviewed task review-8f5eba26", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8fac6e9e12e3f15a-v1", "taskId": "review-8fac6e9e12e3f15a", "version": "1", "name": "Snorkel-reviewed task review-8fac6e9e12e3f15a", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-90e933e0b6f09d7e-v1", "taskId": "review-90e933e0b6f09d7e", "version": "1", "name": "dsa-nonce-reuse-keyrecovery", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-90ff4be3adb37352-v1", "taskId": "review-90ff4be3adb37352", "version": "1", "name": "signed-plugin-trust-rebind", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-96c8f01b35118d5a-v1", "taskId": "review-96c8f01b35118d5a", "version": "1", "name": "Snorkel-reviewed task review-96c8f01b35118d5a", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/424" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-97b9cfdd7ce5bc62-v1", "taskId": "review-97b9cfdd7ce5bc62", "version": "1", "name": "Snorkel-reviewed task review-97b9cfdd7ce5bc62", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/374" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-993af7c619021f4c-v1", "taskId": "review-993af7c619021f4c", "version": "1", "name": "Snorkel-reviewed task review-993af7c619021f4c", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/146" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-99a7b4d2238c5c9c-v1", "taskId": "review-99a7b4d2238c5c9c", "version": "1", "name": "Snorkel-reviewed task review-99a7b4d2238c5c9c", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 7, "n": 10, "p": 0.7, "lo": 0.39677814746114537, "hi": 0.8922087325936989, "tasks": 1, "macro": 0.7, "macroLo": 0.39677814746114537, "macroHi": 0.8922087325936989, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/298" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9d26520e572be460-v1", "taskId": "review-9d26520e572be460", "version": "1", "name": "Snorkel-reviewed task review-9d26520e572be460", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/225" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a06271859ba7cab0-v1", "taskId": "review-a06271859ba7cab0", "version": "1", "name": "Snorkel-reviewed task review-a06271859ba7cab0", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 9, "p": 0.2222222222222222, "lo": 0.06322510711784673, "hi": 0.5474110308930111, "tasks": 1, "macro": 0.2222222222222222, "macroLo": 0.06322510711784673, "macroHi": 0.5474110308930111, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/364" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a51176485533a8ef-v1", "taskId": "review-a51176485533a8ef", "version": "1", "name": "Snorkel-reviewed task review-a51176485533a8ef", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a7841473-v1", "taskId": "review-a7841473", "version": "1", "name": "Snorkel-reviewed task review-a7841473", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/419" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-aab47bdfffc29d7d-v1", "taskId": "review-aab47bdfffc29d7d", "version": "1", "name": "Snorkel-reviewed task review-aab47bdfffc29d7d", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 3, "p": 0.6666666666666666, "lo": 0.20765960080204765, "hi": 0.9385080552796037, "tasks": 1, "macro": 0.6666666666666666, "macroLo": 0.20765960080204765, "macroHi": 0.9385080552796037, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/381" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ae1476cb2639e72d-v1", "taskId": "review-ae1476cb2639e72d", "version": "1", "name": "rsa-known-low-bits-d-recovery", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-aee13f3cb897e640-v1", "taskId": "review-aee13f3cb897e640", "version": "1", "name": "Snorkel-reviewed task review-aee13f3cb897e640", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/456" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b3cec9992e9002ee-v1", "taskId": "review-b3cec9992e9002ee", "version": "1", "name": "exploit-priority-enforcer-cpp", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b4098694cf0bfbca-v1", "taskId": "review-b4098694cf0bfbca", "version": "1", "name": "Snorkel-reviewed task review-b4098694cf0bfbca", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b43f6254f82ac360-v1", "taskId": "review-b43f6254f82ac360", "version": "1", "name": "Snorkel-reviewed task review-b43f6254f82ac360", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-22T161109Z__fe558461__accept" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b4d868f9b77ea88c-v1", "taskId": "review-b4d868f9b77ea88c", "version": "1", "name": "Snorkel-reviewed task review-b4d868f9b77ea88c", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/206" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b5b89f73e450c580-v1", "taskId": "review-b5b89f73e450c580", "version": "1", "name": "jwks-kid-algorithm-confusion", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b5e00c7ca2cff589-v1", "taskId": "review-b5e00c7ca2cff589", "version": "1", "name": "336d1a7c-live", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b79fc96337aa8db1-v1", "taskId": "review-b79fc96337aa8db1", "version": "1", "name": "Snorkel-reviewed task review-b79fc96337aa8db1", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/373" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b8c10becc74d8e73-v1", "taskId": "review-b8c10becc74d8e73", "version": "1", "name": "Snorkel-reviewed task review-b8c10becc74d8e73", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-bc7c72cb35f10666-v1", "taskId": "review-bc7c72cb35f10666", "version": "1", "name": "Snorkel-reviewed task review-bc7c72cb35f10666", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-bee260b7da87db5f-v1", "taskId": "review-bee260b7da87db5f", "version": "1", "name": "tbench-task", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c060e012bd7e56d0-v1", "taskId": "review-c060e012bd7e56d0", "version": "1", "name": "Snorkel-reviewed task review-c060e012bd7e56d0", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c09ba36468df9fc6-v1", "taskId": "review-c09ba36468df9fc6", "version": "1", "name": "Snorkel-reviewed task review-c09ba36468df9fc6", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c1dedcb95d7fc54d-v1", "taskId": "review-c1dedcb95d7fc54d", "version": "1", "name": "Snorkel-reviewed task review-c1dedcb95d7fc54d", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/274" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c255716e0a2dffe0-v1", "taskId": "review-c255716e0a2dffe0", "version": "1", "name": "Snorkel-reviewed task review-c255716e0a2dffe0", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/409" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c4adec863d17bd83-v1", "taskId": "review-c4adec863d17bd83", "version": "1", "name": "quarantine-plan", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c502e96849222074-v1", "taskId": "review-c502e96849222074", "version": "1", "name": "reconcile-delegated-admin-authority-ledger", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c538cfde4db7e6a4-v1", "taskId": "review-c538cfde4db7e6a4", "version": "1", "name": "Snorkel-reviewed task review-c538cfde4db7e6a4", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/459" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c7329f55bf372cce-v1", "taskId": "review-c7329f55bf372cce", "version": "1", "name": "rust-envelope-admission-gate", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 4, "p": 0, "lo": 0, "hi": 0.4898908364545973, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.4898908364545973, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c8c4102c45a8d027-v1", "taskId": "review-c8c4102c45a8d027", "version": "1", "name": "Snorkel-reviewed task review-c8c4102c45a8d027", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/224" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cab0cfc98e25a800-v1", "taskId": "review-cab0cfc98e25a800", "version": "1", "name": "endpoint-kext-rollback-enforcement-drift", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cf636d6c0c0fed4f-v1", "taskId": "review-cf636d6c0c0fed4f", "version": "1", "name": "tbrain-cohort-key-admission-r", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d0592dae19f52448-v1", "taskId": "review-d0592dae19f52448", "version": "1", "name": "Snorkel-reviewed task review-d0592dae19f52448", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/330" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d404a4dab923f55e-v1", "taskId": "review-d404a4dab923f55e", "version": "1", "name": "Snorkel-reviewed task review-d404a4dab923f55e", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/193" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d8318208dea23ff4-v1", "taskId": "review-d8318208dea23ff4", "version": "1", "name": "Snorkel-reviewed task review-d8318208dea23ff4", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/345" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d9100094-v1", "taskId": "review-d9100094", "version": "1", "name": "Snorkel-reviewed task review-d9100094", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/418" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-db431123502e492e-v1", "taskId": "review-db431123502e492e", "version": "1", "name": "attested-exposure-gate", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dbae21997168368f-v1", "taskId": "review-dbae21997168368f", "version": "1", "name": "trust-chain-revoker", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dd175ae76bbe080f-v1", "taskId": "review-dd175ae76bbe080f", "version": "1", "name": "Snorkel-reviewed task review-dd175ae76bbe080f", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/179" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-debc3a9fb4c6e890-v1", "taskId": "review-debc3a9fb4c6e890", "version": "1", "name": "cragford-loadout-freeze-seal-go", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e27eb5788e04c983-v1", "taskId": "review-e27eb5788e04c983", "version": "1", "name": "revocation-cache-split-brain", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e4ba86843400b7d4-v1", "taskId": "review-e4ba86843400b7d4", "version": "1", "name": "Snorkel-reviewed task review-e4ba86843400b7d4", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/392" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e8166b6ee4357b77-v1", "taskId": "review-e8166b6ee4357b77", "version": "1", "name": "switchyard-key-release-profile", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e8bf2a5c599901e1-v1", "taskId": "review-e8bf2a5c599901e1", "version": "1", "name": "Snorkel-reviewed task review-e8bf2a5c599901e1", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/301" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e9463df47e85e255-v1", "taskId": "review-e9463df47e85e255", "version": "1", "name": "Snorkel-reviewed task review-e9463df47e85e255", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 1, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 1, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-29T145349Z__f8cb86ab__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-eba912c27975555c-v1", "taskId": "review-eba912c27975555c", "version": "1", "name": "endpoint-isolation-policy-validator", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ec8cc54462969f02-v1", "taskId": "review-ec8cc54462969f02", "version": "1", "name": "Snorkel-reviewed task review-ec8cc54462969f02", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/434" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ede96eb4aa415901-v1", "taskId": "review-ede96eb4aa415901", "version": "1", "name": "Snorkel-reviewed task review-ede96eb4aa415901", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/149" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ee697ac2e7289a0a-v1", "taskId": "review-ee697ac2e7289a0a", "version": "1", "name": "Snorkel-reviewed task review-ee697ac2e7289a0a", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/439" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-f4a54793-v1", "taskId": "review-f4a54793", "version": "1", "name": "Snorkel-reviewed task review-f4a54793", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f5218e5615888800-v1", "taskId": "review-f5218e5615888800", "version": "1", "name": "Snorkel-reviewed task review-f5218e5615888800", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/437" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f595bc80e313bd6c-v1", "taskId": "review-f595bc80e313bd6c", "version": "1", "name": "Snorkel-reviewed task review-f595bc80e313bd6c", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/382" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f92021447716ac6d-v1", "taskId": "review-f92021447716ac6d", "version": "1", "name": "Snorkel-reviewed task review-f92021447716ac6d", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/492" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fa77ccad77e83cd6-v1", "taskId": "review-fa77ccad77e83cd6", "version": "1", "name": "Snorkel-reviewed task review-fa77ccad77e83cd6", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/217" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-go-sigilmesh-transparency-adjudicator-v1", "taskId": "review-slug-go-sigilmesh-transparency-adjudicator", "version": "1", "name": "go-sigilmesh-transparency-adjudicator", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x2, needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-tessera-mint-v1-v1", "taskId": "review-slug-tessera-mint-v1", "version": "1", "name": "tessera-mint-v1", "domain": "Security / cyber", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-tokentrust-cpp-reconcile-task-v1", "taskId": "review-slug-tokentrust-cpp-reconcile-task", "version": "1", "name": "tokentrust-cpp-reconcile-task", "domain": "Security / cyber", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rust-interp-recursion-depth-audit-v1", "taskId": "rust-interp-recursion-depth-audit", "version": "1", "name": "rust-interp-recursion-depth-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/rust-interp-recursion-depth-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "seccompward-go-bpf-filter-reachability-v1", "taskId": "seccompward-go-bpf-filter-reachability", "version": "1", "name": "seccompward-go-bpf-filter-reachability", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/seccompward-go-bpf-filter-reachability" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sql-injection-fix-v1", "taskId": "sql-injection-fix", "version": "1", "name": "sql-injection-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/8" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "static-file-traversal-fix-v1", "taskId": "static-file-traversal-fix", "version": "1", "name": "static-file-traversal-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/9" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "switch-jumptable-dispatch-x86-v1", "taskId": "switch-jumptable-dispatch-x86", "version": "1", "name": "switch-jumptable-dispatch-x86", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/switch-jumptable-dispatch-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tailcall-retry-ledger-x86-v1", "taskId": "tailcall-retry-ledger-x86", "version": "1", "name": "tailcall-retry-ledger-x86", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/tailcall-retry-ledger-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tiled-column-maxdiff-remainder-x86-v1", "taskId": "tiled-column-maxdiff-remainder-x86", "version": "1", "name": "tiled-column-maxdiff-remainder-x86", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/tiled-column-maxdiff-remainder-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tufward-go-metadata-threshold-audit-v1", "taskId": "tufward-go-metadata-threshold-audit", "version": "1", "name": "tufward-go-metadata-threshold-audit", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/vulns/tufward-go-metadata-threshold-audit" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uts39-skeleton-restriction-level-v1", "taskId": "uts39-skeleton-restriction-level", "version": "1", "name": "uts39-skeleton-restriction-level", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/polyglot/uts39-skeleton-restriction-level" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "weak-token-csprng-fix-v1", "taskId": "weak-token-csprng-fix", "version": "1", "name": "weak-token-csprng-fix", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/10" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "workflow-rules-engine-v1", "taskId": "workflow-rules-engine", "version": "1", "name": "workflow-rules-engine", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/bigcode/workflow-rules-engine" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wraparound-hash-mixer-x86-v1", "taskId": "wraparound-hash-mixer-x86", "version": "1", "name": "wraparound-hash-mixer-x86", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/rosetta/wraparound-hash-mixer-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wrapping-sequence-counter-x86-v1", "taskId": "wrapping-sequence-counter-x86", "version": "1", "name": "wrapping-sequence-counter-x86", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/lazarus/wrapping-sequence-counter-x86" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "xmlmini-c-xml-dtd-parser-v1", "taskId": "xmlmini-c-xml-dtd-parser", "version": "1", "name": "xmlmini-c-xml-dtd-parser", "domain": "Security / cyber", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/vulns/xmlmini-c-xml-dtd-parser" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-021483b3-v1", "taskId": "review-021483b3", "version": "1", "name": "virality-prediction-engine", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0316959d-v1", "taskId": "review-0316959d", "version": "1", "name": "capgate-unify-activation", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-035ac75c-v1", "taskId": "review-035ac75c", "version": "1", "name": "Snorkel-reviewed task review-035ac75c", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Recorded concern about the instruction or evaluation contract." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/463" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-05698706-v1", "taskId": "review-05698706", "version": "1", "name": "Snorkel-reviewed task review-05698706", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/510" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-059d9214-v1", "taskId": "review-059d9214", "version": "1", "name": "Snorkel-reviewed task review-059d9214", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/490" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-05e05d94-v1", "taskId": "review-05e05d94", "version": "1", "name": "Snorkel-reviewed task review-05e05d94", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-06445c7b-v1", "taskId": "review-06445c7b", "version": "1", "name": "Snorkel-reviewed task review-06445c7b", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/412" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-06989859-v1", "taskId": "review-06989859", "version": "1", "name": "Snorkel-reviewed task review-06989859", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-09d3bbb5-v1", "taskId": "review-09d3bbb5", "version": "1", "name": "Snorkel-reviewed task review-09d3bbb5", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/366" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-09f54e96-v1", "taskId": "review-09f54e96", "version": "1", "name": "go-spack-version-ordering", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-09f77e64-v1", "taskId": "review-09f77e64", "version": "1", "name": "Snorkel-reviewed task review-09f77e64", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/472" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-09fe49a6-v1", "taskId": "review-09fe49a6", "version": "1", "name": "Snorkel-reviewed task review-09fe49a6", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about the instruction or evaluation contract." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/476" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0ae3def2-v1", "taskId": "review-0ae3def2", "version": "1", "name": "chronic-kidney-missing", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0b98dcb5-v1", "taskId": "review-0b98dcb5", "version": "1", "name": "Snorkel-reviewed task review-0b98dcb5", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/488" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0ba69fdf-v1", "taskId": "review-0ba69fdf", "version": "1", "name": "Snorkel-reviewed task review-0ba69fdf", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/312" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0c0bbc15-v1", "taskId": "review-0c0bbc15", "version": "1", "name": "coupled-task-single-machine-exact-gap", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0c74fb52-v1", "taskId": "review-0c74fb52", "version": "1", "name": "Snorkel-reviewed task review-0c74fb52", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0ffff240-v1", "taskId": "review-0ffff240", "version": "1", "name": "Snorkel-reviewed task review-0ffff240", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/521" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1255ba7d-v1", "taskId": "review-1255ba7d", "version": "1", "name": "Snorkel-reviewed task review-1255ba7d", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1257ccfe-v1", "taskId": "review-1257ccfe", "version": "1", "name": "echo-maze-go", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1400b74c-v1", "taskId": "review-1400b74c", "version": "1", "name": "Snorkel-reviewed task review-1400b74c", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-167b44d0-v1", "taskId": "review-167b44d0", "version": "1", "name": "Snorkel-reviewed task review-167b44d0", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/342" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1696b0f6-v1", "taskId": "review-1696b0f6", "version": "1", "name": "course-dropout-term", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-19a2c668-v1", "taskId": "review-19a2c668", "version": "1", "name": "Snorkel-reviewed task review-19a2c668", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/166" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1b61dbf9-v1", "taskId": "review-1b61dbf9", "version": "1", "name": "Snorkel-reviewed task review-1b61dbf9", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2004255c-v1", "taskId": "review-2004255c", "version": "1", "name": "Snorkel-reviewed task review-2004255c", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2188962f-v1", "taskId": "review-2188962f", "version": "1", "name": "Snorkel-reviewed task review-2188962f", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/489" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-218bd13a-v1", "taskId": "review-218bd13a", "version": "1", "name": "carrier-accessorial-ledger", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2282116e-v1", "taskId": "review-2282116e", "version": "1", "name": "c-distro-staging-gate", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-22cc58b8-v1", "taskId": "review-22cc58b8", "version": "1", "name": "image-release-evidence-docket-calibration-hardened-v16-policy", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2579b6ad-v1", "taskId": "review-2579b6ad", "version": "1", "name": "Snorkel-reviewed task review-2579b6ad", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/495" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-25ea53f8-v1", "taskId": "review-25ea53f8", "version": "1", "name": "Snorkel-reviewed task review-25ea53f8", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/315" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-26e77869-v1", "taskId": "review-26e77869", "version": "1", "name": "automate_rego_scoring5", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2751f896-v1", "taskId": "review-2751f896", "version": "1", "name": "Snorkel-reviewed task review-2751f896", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/527" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-290ca158-v1", "taskId": "review-290ca158", "version": "1", "name": "trial-conversion-triage", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2ae4d30a-v1", "taskId": "review-2ae4d30a", "version": "1", "name": "Snorkel-reviewed task review-2ae4d30a", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/523" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2b8c21c9-v1", "taskId": "review-2b8c21c9", "version": "1", "name": "Snorkel-reviewed task review-2b8c21c9", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2c79073f-v1", "taskId": "review-2c79073f", "version": "1", "name": "Snorkel-reviewed task review-2c79073f", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/473" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-2d7a0ad2-v1", "taskId": "review-2d7a0ad2", "version": "1", "name": "Snorkel-reviewed task review-2d7a0ad2", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/493" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2f031105-v1", "taskId": "review-2f031105", "version": "1", "name": "Snorkel-reviewed task review-2f031105", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-34152ec6-v1", "taskId": "review-34152ec6", "version": "1", "name": "Snorkel-reviewed task review-34152ec6", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/348" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-3bafbbc9-v1", "taskId": "review-3bafbbc9", "version": "1", "name": "Snorkel-reviewed task review-3bafbbc9", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/471" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-3caace97-v1", "taskId": "review-3caace97", "version": "1", "name": "Snorkel-reviewed task review-3caace97", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/352" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-3d3f6f38-v1", "taskId": "review-3d3f6f38", "version": "1", "name": "train-serve-skew-feature-hash", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3e569043-v1", "taskId": "review-3e569043", "version": "1", "name": "Snorkel-reviewed task review-3e569043", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/514" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3eb25ae1-v1", "taskId": "review-3eb25ae1", "version": "1", "name": "Snorkel-reviewed task review-3eb25ae1", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/507" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3eb94561-v1", "taskId": "review-3eb94561", "version": "1", "name": "Snorkel-reviewed task review-3eb94561", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/341" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3f2f8645-v1", "taskId": "review-3f2f8645", "version": "1", "name": "Snorkel-reviewed task review-3f2f8645", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/474" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-3fcbfd12-v1", "taskId": "review-3fcbfd12", "version": "1", "name": "Snorkel-reviewed task review-3fcbfd12", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4070eef7-v1", "taskId": "review-4070eef7", "version": "1", "name": "Snorkel-reviewed task review-4070eef7", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Recorded concern about the instruction or evaluation contract." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/470" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4358e917-v1", "taskId": "review-4358e917", "version": "1", "name": "Snorkel-reviewed task review-4358e917", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/525" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-44abc022-v1", "taskId": "review-44abc022", "version": "1", "name": "Snorkel-reviewed task review-44abc022", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/118" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-48537222-v1", "taskId": "review-48537222", "version": "1", "name": "go-minmax-othello", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-49da815c-v1", "taskId": "review-49da815c", "version": "1", "name": "cargo-no-std-feature-matrix-release-forge", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4a508f1e-v1", "taskId": "review-4a508f1e", "version": "1", "name": "regression-kink-design-benefit-schedule-marginal-effect", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4baa6f94-v1", "taskId": "review-4baa6f94", "version": "1", "name": "Snorkel-reviewed task review-4baa6f94", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 5, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4c85ed25-v1", "taskId": "review-4c85ed25", "version": "1", "name": "fraud-priorshift-calibration-r2", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4cf4c29b-v1", "taskId": "review-4cf4c29b", "version": "1", "name": "Snorkel-reviewed task review-4cf4c29b", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-4d659497-v1", "taskId": "review-4d659497", "version": "1", "name": "Snorkel-reviewed task review-4d659497", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/530" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-4f6152eb-v1", "taskId": "review-4f6152eb", "version": "1", "name": "Snorkel-reviewed task review-4f6152eb", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/339" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5006c113-v1", "taskId": "review-5006c113", "version": "1", "name": "c-pkgplan-transaction-planner", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-50c0ec32-v1", "taskId": "review-50c0ec32", "version": "1", "name": "go-minmax-othello", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5191d811-v1", "taskId": "review-5191d811", "version": "1", "name": "Snorkel-reviewed task review-5191d811", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/308" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5cade10d-v1", "taskId": "review-5cade10d", "version": "1", "name": "po-restore-satellite-pass-forecasts", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-608034e5-v1", "taskId": "review-608034e5", "version": "1", "name": "Snorkel-reviewed task review-608034e5", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/504" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-61c70002-v1", "taskId": "review-61c70002", "version": "1", "name": "Snorkel-reviewed task review-61c70002", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/511" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-62303775-v1", "taskId": "review-62303775", "version": "1", "name": "Snorkel-reviewed task review-62303775", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-62c71850-v1", "taskId": "review-62c71850", "version": "1", "name": "Snorkel-reviewed task review-62c71850", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-64f69010-v1", "taskId": "review-64f69010", "version": "1", "name": "coverage-map-wake-word", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-65a594de-v1", "taskId": "review-65a594de", "version": "1", "name": "provenance-worker", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-666189c5-v1", "taskId": "review-666189c5", "version": "1", "name": "catalog-prune-recovery-gap", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-67ee34a4-v1", "taskId": "review-67ee34a4", "version": "1", "name": "c-sudoku-dlx-solver", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-68b558eb-v1", "taskId": "review-68b558eb", "version": "1", "name": "Snorkel-reviewed task review-68b558eb", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/344" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6eff3038-v1", "taskId": "review-6eff3038", "version": "1", "name": "course-dropout-term", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6f0ed122-v1", "taskId": "review-6f0ed122", "version": "1", "name": "Snorkel-reviewed task review-6f0ed122", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/522" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-6ff13da3-v1", "taskId": "review-6ff13da3", "version": "1", "name": "lunar-lab-udev-coldplug-rename", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-726c61d7-v1", "taskId": "review-726c61d7", "version": "1", "name": "Snorkel-reviewed task review-726c61d7", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/363" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-73d4385f-v1", "taskId": "review-73d4385f", "version": "1", "name": "Snorkel-reviewed task review-73d4385f", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/325" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-73ec77ab-v1", "taskId": "review-73ec77ab", "version": "1", "name": "iceberg-snapshot-thinner", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7565f545-v1", "taskId": "review-7565f545", "version": "1", "name": "Snorkel-reviewed task review-7565f545", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/529" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-76857547-v1", "taskId": "review-76857547", "version": "1", "name": "Snorkel-reviewed task review-76857547", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/500" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-76eb64dc-v1", "taskId": "review-76eb64dc", "version": "1", "name": "Snorkel-reviewed task review-76eb64dc", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/524" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-77aa0de1-v1", "taskId": "review-77aa0de1", "version": "1", "name": "autit-typescript-setup-viloations-with", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-77d54253-v1", "taskId": "review-77d54253", "version": "1", "name": "nix-flake-follows-override-precedence", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7d73ae1d-v1", "taskId": "review-7d73ae1d", "version": "1", "name": "Snorkel-reviewed task review-7d73ae1d", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/501" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7daf208a-v1", "taskId": "review-7daf208a", "version": "1", "name": "kiln-shelf-registry-revised-v16", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7e1a9989-v1", "taskId": "review-7e1a9989", "version": "1", "name": "Snorkel-reviewed task review-7e1a9989", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/372" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-80152a37-v1", "taskId": "review-80152a37", "version": "1", "name": "Snorkel-reviewed task review-80152a37", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-83917f8a-v1", "taskId": "review-83917f8a", "version": "1", "name": "c-distro-staging-gate", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-84b93bfa-v1", "taskId": "review-84b93bfa", "version": "1", "name": "go-minmax-othello", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-870fe6ee-v1", "taskId": "review-870fe6ee", "version": "1", "name": "Snorkel-reviewed task review-870fe6ee", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/468" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8732a06c-v1", "taskId": "review-8732a06c", "version": "1", "name": "copperline-rails-invitational", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-883dbf43-v1", "taskId": "review-883dbf43", "version": "1", "name": "signed-plugin-trust-rebind", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-899aafc6-v1", "taskId": "review-899aafc6", "version": "1", "name": "Snorkel-reviewed task review-899aafc6", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/498" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-8b5ffd03-v1", "taskId": "review-8b5ffd03", "version": "1", "name": "Snorkel-reviewed task review-8b5ffd03", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/479" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-914a6778-v1", "taskId": "review-914a6778", "version": "1", "name": "Snorkel-reviewed task review-914a6778", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-91d44e53-v1", "taskId": "review-91d44e53", "version": "1", "name": "edge-session-router", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-93df1a8e-v1", "taskId": "review-93df1a8e", "version": "1", "name": "Snorkel-reviewed task review-93df1a8e", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/481" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-989d8dda-v1", "taskId": "review-989d8dda", "version": "1", "name": "c-sudoku-dlx-solver", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-98c42892-v1", "taskId": "review-98c42892", "version": "1", "name": "coupled-task-single-machine-exact-gap", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9a6c391c-v1", "taskId": "review-9a6c391c", "version": "1", "name": "change-ring-touch-composer", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9cd8ede8-v1", "taskId": "review-9cd8ede8", "version": "1", "name": "Snorkel-reviewed task review-9cd8ede8", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/496" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9ce69360-v1", "taskId": "review-9ce69360", "version": "1", "name": "Snorkel-reviewed task review-9ce69360", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/343" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-9d44798d-v1", "taskId": "review-9d44798d", "version": "1", "name": "Snorkel-reviewed task review-9d44798d", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/497" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9e81f0d9-v1", "taskId": "review-9e81f0d9", "version": "1", "name": "pam-faillock-reset-across-namespace", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a14d0f5e-v1", "taskId": "review-a14d0f5e", "version": "1", "name": "gas-sensor-t-optimal-model-discrimination", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a25affd5-v1", "taskId": "review-a25affd5", "version": "1", "name": "Snorkel-reviewed task review-a25affd5", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/411" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a4c8bb78-v1", "taskId": "review-a4c8bb78", "version": "1", "name": "Snorkel-reviewed task review-a4c8bb78", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/318" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a6427c77-v1", "taskId": "review-a6427c77", "version": "1", "name": "c-warranty-claim-etl-long-bulletins", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a66f6fa9-v1", "taskId": "review-a66f6fa9", "version": "1", "name": "mlflow-corpus-classifier-training-18", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a804f0a8-v1", "taskId": "review-a804f0a8", "version": "1", "name": "Snorkel-reviewed task review-a804f0a8", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/478" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a992e191-v1", "taskId": "review-a992e191", "version": "1", "name": "regression-kink-design-benefit-schedule-marginal-effect", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-aa02dc89-v1", "taskId": "review-aa02dc89", "version": "1", "name": "Snorkel-reviewed task review-aa02dc89", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/506" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ab72997f-v1", "taskId": "review-ab72997f", "version": "1", "name": "Snorkel-reviewed task review-ab72997f", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ac246b12-v1", "taskId": "review-ac246b12", "version": "1", "name": "Snorkel-reviewed task review-ac246b12", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/526" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ace7b73d-v1", "taskId": "review-ace7b73d", "version": "1", "name": "Snorkel-reviewed task review-ace7b73d", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ae55d570-v1", "taskId": "review-ae55d570", "version": "1", "name": "Snorkel-reviewed task review-ae55d570", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b02d3b90-v1", "taskId": "review-b02d3b90", "version": "1", "name": "Snorkel-reviewed task review-b02d3b90", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/350" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b0b72977-v1", "taskId": "review-b0b72977", "version": "1", "name": "Snorkel-reviewed task review-b0b72977", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/513" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b27600c1-v1", "taskId": "review-b27600c1", "version": "1", "name": "tokenizer-byte-fallback-ladder-revised", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b313179b-v1", "taskId": "review-b313179b", "version": "1", "name": "Snorkel-reviewed task review-b313179b", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/491" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b50d3320-v1", "taskId": "review-b50d3320", "version": "1", "name": "Snorkel-reviewed task review-b50d3320", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/477" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b688b813-v1", "taskId": "review-b688b813", "version": "1", "name": "Snorkel-reviewed task review-b688b813", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/484" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b690ac17-v1", "taskId": "review-b690ac17", "version": "1", "name": "gas-sensor-t-optimal-model-discrimination", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b83f17eb-v1", "taskId": "review-b83f17eb", "version": "1", "name": "Snorkel-reviewed task review-b83f17eb", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/518" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b9c4985b-v1", "taskId": "review-b9c4985b", "version": "1", "name": "Snorkel-reviewed task review-b9c4985b", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/327" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-be765d16-v1", "taskId": "review-be765d16", "version": "1", "name": "Snorkel-reviewed task review-be765d16", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/335" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-bec1c7e9-v1", "taskId": "review-bec1c7e9", "version": "1", "name": "Snorkel-reviewed task review-bec1c7e9", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/528" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-bf33b396-v1", "taskId": "review-bf33b396", "version": "1", "name": "autit-typescript-setup-viloations-with", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-bfcdbecb-v1", "taskId": "review-bfcdbecb", "version": "1", "name": "Snorkel-reviewed task review-bfcdbecb", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/509" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c1bf9542-v1", "taskId": "review-c1bf9542", "version": "1", "name": "Snorkel-reviewed task review-c1bf9542", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/520" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c213dff6-v1", "taskId": "review-c213dff6", "version": "1", "name": "Snorkel-reviewed task review-c213dff6", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/465" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c47a99c8-v1", "taskId": "review-c47a99c8", "version": "1", "name": "copperline-rails-invitational", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ca0e7752-v1", "taskId": "review-ca0e7752", "version": "1", "name": "Snorkel-reviewed task review-ca0e7752", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/469" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ccbd5d94-v1", "taskId": "review-ccbd5d94", "version": "1", "name": "Snorkel-reviewed task review-ccbd5d94", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/508" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d444a1f1-v1", "taskId": "review-d444a1f1", "version": "1", "name": "Snorkel-reviewed task review-d444a1f1", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/502" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d4eda92afd081963-v1", "taskId": "review-d4eda92afd081963", "version": "1", "name": "booking-slot-merge", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d95d209d-v1", "taskId": "review-d95d209d", "version": "1", "name": "Snorkel-reviewed task review-d95d209d", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/494" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d9c48a32-v1", "taskId": "review-d9c48a32", "version": "1", "name": "Snorkel-reviewed task review-d9c48a32", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/517" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d9edee65-v1", "taskId": "review-d9edee65", "version": "1", "name": "Snorkel-reviewed task review-d9edee65", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/487" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-daca54c7-v1", "taskId": "review-daca54c7", "version": "1", "name": "build-rust-clinical-lab-proficiency-accreditation-ledger7", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-db43a374-v1", "taskId": "review-db43a374", "version": "1", "name": "Snorkel-reviewed task review-db43a374", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dc3e868b-v1", "taskId": "review-dc3e868b", "version": "1", "name": "shardveil-contingency-go-v6-campaign-reset", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dca2afdb-v1", "taskId": "review-dca2afdb", "version": "1", "name": "Snorkel-reviewed task review-dca2afdb", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/515" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dd05eb72-v1", "taskId": "review-dd05eb72", "version": "1", "name": "Snorkel-reviewed task review-dd05eb72", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/482" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-deb658da-v1", "taskId": "review-deb658da", "version": "1", "name": "Snorkel-reviewed task review-deb658da", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/512" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-dfb57583-v1", "taskId": "review-dfb57583", "version": "1", "name": "Snorkel-reviewed task review-dfb57583", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/480" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-e04b3e85-v1", "taskId": "review-e04b3e85", "version": "1", "name": "Snorkel-reviewed task review-e04b3e85", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/466" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-e0a516c5-v1", "taskId": "review-e0a516c5", "version": "1", "name": "tbrain-setcookie-store", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e159dfbe-v1", "taskId": "review-e159dfbe", "version": "1", "name": "go-minmax-othello", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e1f2b23f-v1", "taskId": "review-e1f2b23f", "version": "1", "name": "Snorkel-reviewed task review-e1f2b23f", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/516" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e577ee3f-v1", "taskId": "review-e577ee3f", "version": "1", "name": "Snorkel-reviewed task review-e577ee3f", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/320" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e6aef72e-v1", "taskId": "review-e6aef72e", "version": "1", "name": "carrier-accessorial-ledger", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e73d0dd2-v1", "taskId": "review-e73d0dd2", "version": "1", "name": "Snorkel-reviewed task review-e73d0dd2", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/247" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ea158463-v1", "taskId": "review-ea158463", "version": "1", "name": "Snorkel-reviewed task review-ea158463", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/485" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ea297109-v1", "taskId": "review-ea297109", "version": "1", "name": "Snorkel-reviewed task review-ea297109", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/354" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ea976c0c-v1", "taskId": "review-ea976c0c", "version": "1", "name": "build-snmp-mib-oid-overlap-compiler-java-json", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-eb2079fe-v1", "taskId": "review-eb2079fe", "version": "1", "name": "Snorkel-reviewed task review-eb2079fe", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/503" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ebfbc7cb-v1", "taskId": "review-ebfbc7cb", "version": "1", "name": "shardveil-controller-go-v9-balanced-medium", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ec803d6c-v1", "taskId": "review-ec803d6c", "version": "1", "name": "gas-sensor-t-optimal-model-discrimination", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ed1f82e6-v1", "taskId": "review-ed1f82e6", "version": "1", "name": "Snorkel-reviewed task review-ed1f82e6", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/323" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ed5d7e87-v1", "taskId": "review-ed5d7e87", "version": "1", "name": "Snorkel-reviewed task review-ed5d7e87", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/467" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ee3cedae-v1", "taskId": "review-ee3cedae", "version": "1", "name": "po-restore-satellite-pass-forecasts", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f05b80a0-v1", "taskId": "review-f05b80a0", "version": "1", "name": "Snorkel-reviewed task review-f05b80a0", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/483" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f45e7468-v1", "taskId": "review-f45e7468", "version": "1", "name": "Snorkel-reviewed task review-f45e7468", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/499" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f4810aa9-v1", "taskId": "review-f4810aa9", "version": "1", "name": "Snorkel-reviewed task review-f4810aa9", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f4b67619-v1", "taskId": "review-f4b67619", "version": "1", "name": "Snorkel-reviewed task review-f4b67619", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/519" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f5cbf0d1-v1", "taskId": "review-f5cbf0d1", "version": "1", "name": "quantized-checkpoint-reconciler", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f60ef080-v1", "taskId": "review-f60ef080", "version": "1", "name": "Snorkel-reviewed task review-f60ef080", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/505" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f613b8b6-v1", "taskId": "review-f613b8b6", "version": "1", "name": "Snorkel-reviewed task review-f613b8b6", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/307" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f61675fd-v1", "taskId": "review-f61675fd", "version": "1", "name": "Snorkel-reviewed task review-f61675fd", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/486" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fbf96820-v1", "taskId": "review-fbf96820", "version": "1", "name": "Snorkel-reviewed task review-fbf96820", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/475" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-fc63051d-v1", "taskId": "review-fc63051d", "version": "1", "name": "Snorkel-reviewed task review-fc63051d", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/464" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-slug-chainward-x509-validation-v1", "taskId": "review-slug-chainward-x509-validation", "version": "1", "name": "chainward-x509-validation", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-firmware-release-security-adjudication-v1", "taskId": "review-slug-firmware-release-security-adjudication", "version": "1", "name": "firmware-release-security-adjudication", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-sanctuary-ledger-trust-certification-v1", "taskId": "review-slug-sanctuary-ledger-trust-certification", "version": "1", "name": "sanctuary-ledger-trust-certification", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-vaccine-cold-chain-stochastic-plan-v1", "taskId": "review-slug-vaccine-cold-chain-stochastic-plan", "version": "1", "name": "vaccine-cold-chain-stochastic-plan", "domain": "Snorkel-reviewed tasks (category unknown)", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "59-fix-broken-cognito-m2m-httpapi-jwt-scope-gated-v1", "taskId": "59-fix-broken-cognito-m2m-httpapi-jwt-scope-gated", "version": "1", "name": "59-fix-broken-cognito-m2m-httpapi-jwt-scope-gated", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/109" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "60-fix-broken-ecs-fargate-secrets-kms-exec-role-v1", "taskId": "60-fix-broken-ecs-fargate-secrets-kms-exec-role", "version": "1", "name": "60-fix-broken-ecs-fargate-secrets-kms-exec-role", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/110" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "_reference-v1", "taskId": "_reference", "version": "1", "name": "_reference", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ab-glyph-rasterizer-v1", "taskId": "ab-glyph-rasterizer", "version": "1", "name": "ab-glyph-rasterizer", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "access-parser-jet-conformance-v1", "taskId": "access-parser-jet-conformance", "version": "1", "name": "access-parser-jet-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/585" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "acorn-es5-interpreter-v1", "taskId": "acorn-es5-interpreter", "version": "1", "name": "acorn-es5-interpreter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "acorn-loose-repair-log-v1", "taskId": "acorn-loose-repair-log", "version": "1", "name": "acorn-loose-repair-log", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "allauth-webauthn-verify-v1", "taskId": "allauth-webauthn-verify", "version": "1", "name": "allauth-webauthn-verify", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ammonia-sanitizer-api-v1", "taskId": "ammonia-sanitizer-api", "version": "1", "name": "ammonia-sanitizer-api", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "androidbinary-arsc-conformance-v1", "taskId": "androidbinary-arsc-conformance", "version": "1", "name": "androidbinary-arsc-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/583" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "anytree-tree-edit-v1", "taskId": "anytree-tree-edit", "version": "1", "name": "anytree-tree-edit", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "apache-datafusion-21057-v1", "taskId": "apache-datafusion-21057", "version": "1", "name": "apache-datafusion-21057", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/swepr/apache-datafusion-21057" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "apigw-http-api-jwt-authorizer-lambda-integration-v1", "taskId": "apigw-http-api-jwt-authorizer-lambda-integration", "version": "1", "name": "apigw-http-api-jwt-authorizer-lambda-integration", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/111" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "apigw-sqs-fifo-direct-integration-v1", "taskId": "apigw-sqs-fifo-direct-integration", "version": "1", "name": "apigw-sqs-fifo-direct-integration", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "arrow-calendars-engine-v1", "taskId": "arrow-calendars-engine", "version": "1", "name": "arrow-calendars-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "asn1crypto-x509-pathval-v1", "taskId": "asn1crypto-x509-pathval", "version": "1", "name": "asn1crypto-x509-pathval", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "asteval-analysis-v1", "taskId": "asteval-analysis", "version": "1", "name": "asteval-analysis", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "astral-sh-ruff-25544-v1", "taskId": "astral-sh-ruff-25544", "version": "1", "name": "astral-sh-ruff-25544", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/swepr/astral-sh-ruff-25544" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "athena-workgroup-result-encryption-cmk-enforced-v1", "taskId": "athena-workgroup-result-encryption-cmk-enforced", "version": "1", "name": "athena-workgroup-result-encryption-cmk-enforced", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/112" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ava-property-engine-v1", "taskId": "ava-property-engine", "version": "1", "name": "ava-property-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "babel-messageformat2-v1", "taskId": "babel-messageformat2", "version": "1", "name": "babel-messageformat2", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "babylon-navmesh-engine-v1", "taskId": "babylon-navmesh-engine", "version": "1", "name": "babylon-navmesh-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bbolt-filter-engine-v1", "taskId": "bbolt-filter-engine", "version": "1", "name": "bbolt-filter-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "beep-flac-decoder-v1", "taskId": "beep-flac-decoder", "version": "1", "name": "beep-flac-decoder", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "beets-fingerprint-engine-v1", "taskId": "beets-fingerprint-engine", "version": "1", "name": "beets-fingerprint-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "benthos-windowing-engine-v1", "taskId": "benthos-windowing-engine", "version": "1", "name": "benthos-windowing-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bibtexparser-biblatex-model-v1", "taskId": "bibtexparser-biblatex-model", "version": "1", "name": "bibtexparser-biblatex-model", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bibtexparser-csl-engine-v1", "taskId": "bibtexparser-csl-engine", "version": "1", "name": "bibtexparser-csl-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/581" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "biopython-msa-progressive-v1", "taskId": "biopython-msa-progressive", "version": "1", "name": "biopython-msa-progressive", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "biopython-phylo-engine-v1", "taskId": "biopython-phylo-engine", "version": "1", "name": "biopython-phylo-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/579" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bitarray-coding-v1", "taskId": "bitarray-coding", "version": "1", "name": "bitarray-coding", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "boardgameio-search-engine-v1", "taskId": "boardgameio-search-engine", "version": "1", "name": "boardgameio-search-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "booleanpy-bdd-engine-v1", "taskId": "booleanpy-bdd-engine", "version": "1", "name": "booleanpy-bdd-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bpmnjs-execution-engine-v1", "taskId": "bpmnjs-execution-engine", "version": "1", "name": "bpmnjs-execution-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "browserslist-compat-engine-v1", "taskId": "browserslist-compat-engine", "version": "1", "name": "browserslist-compat-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/579" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bubbletea-readline-engine-v1", "taskId": "bubbletea-readline-engine", "version": "1", "name": "bubbletea-readline-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bubbletea-termgraphics-engine-v1", "taskId": "bubbletea-termgraphics-engine", "version": "1", "name": "bubbletea-termgraphics-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "buntdb-raft-consensus-v1", "taskId": "buntdb-raft-consensus", "version": "1", "name": "buntdb-raft-consensus", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bytecode-cfg-optimizer-v1", "taskId": "bytecode-cfg-optimizer", "version": "1", "name": "bytecode-cfg-optimizer", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/557" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "bytecodealliance-wasmtime-13391-v1", "taskId": "bytecodealliance-wasmtime-13391", "version": "1", "name": "bytecodealliance-wasmtime-13391", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/swepr/bytecodealliance-wasmtime-13391" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "casbin-zanzibar-engine-v1", "taskId": "casbin-zanzibar-engine", "version": "1", "name": "casbin-zanzibar-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cascadia-selectors-level4-v1", "taskId": "cascadia-selectors-level4", "version": "1", "name": "cascadia-selectors-level4", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cbor2-cddl-validation-v1", "taskId": "cbor2-cddl-validation", "version": "1", "name": "cbor2-cddl-validation", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cel-spec-conformance-v1", "taskId": "cel-spec-conformance", "version": "1", "name": "cel-spec-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "celgo-eval-dependencies-v1", "taskId": "celgo-eval-dependencies", "version": "1", "name": "celgo-eval-dependencies", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "centrifugo-pbft-engine-v1", "taskId": "centrifugo-pbft-engine", "version": "1", "name": "centrifugo-pbft-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "changesets-apidiff-engine-v1", "taskId": "changesets-apidiff-engine", "version": "1", "name": "changesets-apidiff-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/582" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chartjs-hierarchy-layout-v1", "taskId": "chartjs-hierarchy-layout", "version": "1", "name": "chartjs-hierarchy-layout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cheerio-aria-engine-v1", "taskId": "cheerio-aria-engine", "version": "1", "name": "cheerio-aria-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/542" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chessground-rules-engine-v1", "taskId": "chessground-rules-engine", "version": "1", "name": "chessground-rules-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "chevron-handlebars-v1", "taskId": "chevron-handlebars", "version": "1", "name": "chevron-handlebars", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/557" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ciliumebpf-verifier-engine-v1", "taskId": "ciliumebpf-verifier-engine", "version": "1", "name": "ciliumebpf-verifier-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/580" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "click-constraint-groups-v1", "taskId": "click-constraint-groups", "version": "1", "name": "click-constraint-groups", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cockroachdb-apd-27-v1", "taskId": "cockroachdb-apd-27", "version": "1", "name": "cockroachdb-apd-27", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/swepr/cockroachdb-apd-27" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "colly-robots-engine-v1", "taskId": "colly-robots-engine", "version": "1", "name": "colly-robots-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "concurrently-command-readiness-dependencies-13bdff1-v1", "taskId": "concurrently-command-readiness-dependencies-13bdff1", "version": "1", "name": "concurrently-command-readiness-dependencies-13bdff1", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "construct-codec-v1", "taskId": "construct-codec", "version": "1", "name": "construct-codec", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cosmic-text-missing-v1", "taskId": "cosmic-text-missing", "version": "1", "name": "cosmic-text-missing", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cron-next-fire-v1", "taskId": "cron-next-fire", "version": "1", "name": "cron-next-fire", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/74" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "crossterm-vt-interpreter-v1", "taskId": "crossterm-vt-interpreter", "version": "1", "name": "crossterm-vt-interpreter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/546" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cssparser-syntax-l3-v1", "taskId": "cssparser-syntax-l3", "version": "1", "name": "cssparser-syntax-l3", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cssutils-cssom-compute-v1", "taskId": "cssutils-cssom-compute", "version": "1", "name": "cssutils-cssom-compute", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cyclonedx-sbom-analysis-v1", "taskId": "cyclonedx-sbom-analysis", "version": "1", "name": "cyclonedx-sbom-analysis", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/559" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "cytoscape-treewidth-engine-v1", "taskId": "cytoscape-treewidth-engine", "version": "1", "name": "cytoscape-treewidth-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "datareader-sas7bdat-dta-conformance-v1", "taskId": "datareader-sas7bdat-dta-conformance", "version": "1", "name": "datareader-sas7bdat-dta-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/583" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "datefns-cldr-format-v1", "taskId": "datefns-cldr-format", "version": "1", "name": "datefns-cldr-format", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "datefns-tzdata-engine-v1", "taskId": "datefns-tzdata-engine", "version": "1", "name": "datefns-tzdata-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/581" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dayjs-edtf-engine-v1", "taskId": "dayjs-edtf-engine", "version": "1", "name": "dayjs-edtf-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dbf-xbase-conformance-v1", "taskId": "dbf-xbase-conformance", "version": "1", "name": "dbf-xbase-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/561" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ddb-outbox-eventbridge-fanout-v1", "taskId": "ddb-outbox-eventbridge-fanout", "version": "1", "name": "ddb-outbox-eventbridge-fanout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "decimal-ieee754-engine-v1", "taskId": "decimal-ieee754-engine", "version": "1", "name": "decimal-ieee754-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dep-resolver-v1", "taskId": "dep-resolver", "version": "1", "name": "dep-resolver", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/51" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "diff-patch-engine-v1", "taskId": "diff-patch-engine", "version": "1", "name": "diff-patch-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/52" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dinero-financial-math-v1", "taskId": "dinero-financial-math", "version": "1", "name": "dinero-financial-math", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "disburse-v1", "taskId": "disburse", "version": "1", "name": "disburse", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/disburse" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dissect-ese-conformance-v1", "taskId": "dissect-ese-conformance", "version": "1", "name": "dissect-ese-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dissect-ewf-conformance-v1", "taskId": "dissect-ewf-conformance", "version": "1", "name": "dissect-ewf-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dnfile-ecma335-conformance-v1", "taskId": "dnfile-ecma335-conformance", "version": "1", "name": "dnfile-ecma335-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/584" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dns-dnssec-validate-v1", "taskId": "dns-dnssec-validate", "version": "1", "name": "dns-dnssec-validate", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "docutils-rst-engine-v1", "taskId": "docutils-rst-engine", "version": "1", "name": "docutils-rst-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dulwich-openpgp-verify-v1", "taskId": "dulwich-openpgp-verify", "version": "1", "name": "dulwich-openpgp-verify", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "dxf-conformance-v1", "taskId": "dxf-conformance", "version": "1", "name": "dxf-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/561" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ebiten-ecs-engine-v1", "taskId": "ebiten-ecs-engine", "version": "1", "name": "ebiten-ecs-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "echarts-value-ordered-stacks-018ed39-v1", "taskId": "echarts-value-ordered-stacks-018ed39", "version": "1", "name": "echarts-value-ordered-stacks-018ed39", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "echo-http-signatures-v1", "taskId": "echo-http-signatures", "version": "1", "name": "echo-http-signatures", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ecr-image-scan-lifecycle-immutable-tags-replication-v1", "taskId": "ecr-image-scan-lifecycle-immutable-tags-replication", "version": "1", "name": "ecr-image-scan-lifecycle-immutable-tags-replication", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/113" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "edid-conformance-v1", "taskId": "edid-conformance", "version": "1", "name": "edid-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/563" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "efs-access-point-posix-iam-mount-target-v1", "taskId": "efs-access-point-posix-iam-mount-target", "version": "1", "name": "efs-access-point-posix-iam-mount-target", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/114" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "einops-per-axis-reductions-1f2268c-v1", "taskId": "einops-per-axis-reductions-1f2268c", "version": "1", "name": "einops-per-axis-reductions-1f2268c", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "elementpath-schematron-v1", "taskId": "elementpath-schematron", "version": "1", "name": "elementpath-schematron", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "embedded-graphics-stroke-caps-joins-59fd64b-v1", "taskId": "embedded-graphics-stroke-caps-joins-59fd64b", "version": "1", "name": "embedded-graphics-stroke-caps-joins-59fd64b", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "epub-ocf-conformance-v1", "taskId": "epub-ocf-conformance", "version": "1", "name": "epub-ocf-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/559" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "etcd-io-etcd-10693-v1", "taskId": "etcd-io-etcd-10693", "version": "1", "name": "etcd-io-etcd-10693", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/swepr/etcd-io-etcd-10693" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "etree-xpath10-engine-v1", "taskId": "etree-xpath10-engine", "version": "1", "name": "etree-xpath10-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "events-query-layer-v1", "taskId": "events-query-layer", "version": "1", "name": "events-query-layer", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/53" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "evtx-binxml-conformance-v1", "taskId": "evtx-binxml-conformance", "version": "1", "name": "evtx-binxml-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/587" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "excelize-missing-engine-v1", "taskId": "excelize-missing-engine", "version": "1", "name": "excelize-missing-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "exceljs-pivot-engine-v1", "taskId": "exceljs-pivot-engine", "version": "1", "name": "exceljs-pivot-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "exif-tiff-conformance-v1", "taskId": "exif-tiff-conformance", "version": "1", "name": "exif-tiff-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/564" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "expr-lang-expr-912-v1", "taskId": "expr-lang-expr-912", "version": "1", "name": "expr-lang-expr-912", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/swepr/expr-lang-expr-912" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fabric-text-lists-2bd4992-v1", "taskId": "fabric-text-lists-2bd4992", "version": "1", "name": "fabric-text-lists-2bd4992", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "falcon-request-body-limit-966bf9c-v1", "taskId": "falcon-request-body-limit-966bf9c", "version": "1", "name": "falcon-request-body-limit-966bf9c", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fastavro-schema-algebra-v1", "taskId": "fastavro-schema-algebra", "version": "1", "name": "fastavro-schema-algebra", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/557" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fasthttp-websocket-engine-v1", "taskId": "fasthttp-websocket-engine", "version": "1", "name": "fasthttp-websocket-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/548" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fastkml-ogc-conformance-v1", "taskId": "fastkml-ogc-conformance", "version": "1", "name": "fastkml-ogc-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/560" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fastxml-xsd-validation-v1", "taskId": "fastxml-xsd-validation", "version": "1", "name": "fastxml-xsd-validation", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fatihcolor-colorscience-engine-v1", "taskId": "fatihcolor-colorscience-engine", "version": "1", "name": "fatihcolor-colorscience-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "feedgen-conformance-v1", "taskId": "feedgen-conformance", "version": "1", "name": "feedgen-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "feedparser-normalize-v1", "taskId": "feedparser-normalize", "version": "1", "name": "feedparser-normalize", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fit-conformance-v1", "taskId": "fit-conformance", "version": "1", "name": "fit-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/561" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fix-broken-appsync-graphql-cognito-resolver-cache-leak-v1", "taskId": "fix-broken-appsync-graphql-cognito-resolver-cache-leak", "version": "1", "name": "fix-broken-appsync-graphql-cognito-resolver-cache-leak", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/115" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "flate2-brotli-decoder-v1", "taskId": "flate2-brotli-decoder", "version": "1", "name": "flate2-brotli-decoder", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "folium-mvt-engine-v1", "taskId": "folium-mvt-engine", "version": "1", "name": "folium-mvt-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fonttools-ot-shaping-v1", "taskId": "fonttools-ot-shaping", "version": "1", "name": "fonttools-ot-shaping", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fpdf2-html-layout-v1", "taskId": "fpdf2-html-layout", "version": "1", "name": "fpdf2-html-layout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fsspec-transactional-mutations-6765ff9-v1", "taskId": "fsspec-transactional-mutations-6765ff9", "version": "1", "name": "fsspec-transactional-mutations-6765ff9", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fullmoon-lua-interpreter-v1", "taskId": "fullmoon-lua-interpreter", "version": "1", "name": "fullmoon-lua-interpreter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "fzf-fmindex-engine-v1", "taskId": "fzf-fmindex-engine", "version": "1", "name": "fzf-fmindex-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "geopandas-strtree-engine-v1", "taskId": "geopandas-strtree-engine", "version": "1", "name": "geopandas-strtree-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "georinex-rinex-conformance-v1", "taskId": "georinex-rinex-conformance", "version": "1", "name": "georinex-rinex-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gerber-pcb-conformance-v1", "taskId": "gerber-pcb-conformance", "version": "1", "name": "gerber-pcb-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/562" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gettext-po-conformance-v1", "taskId": "gettext-po-conformance", "version": "1", "name": "gettext-po-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/561" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gff3-conformance-v1", "taskId": "gff3-conformance", "version": "1", "name": "gff3-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/563" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gitoxide-merge-engine-v1", "taskId": "gitoxide-merge-engine", "version": "1", "name": "gitoxide-merge-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gjson-jsonpath-rfc9535-v1", "taskId": "gjson-jsonpath-rfc9535", "version": "1", "name": "gjson-jsonpath-rfc9535", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gltf-conformance-v1", "taskId": "gltf-conformance", "version": "1", "name": "gltf-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/561" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "glue-etl-catalog-security-configuration-kms-v1", "taskId": "glue-etl-catalog-security-configuration-kms", "version": "1", "name": "glue-etl-catalog-security-configuration-kms", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/116" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "glymur-jp2-conformance-v1", "taskId": "glymur-jp2-conformance", "version": "1", "name": "glymur-jp2-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gnet-mqtt5-engine-v1", "taskId": "gnet-mqtt5-engine", "version": "1", "name": "gnet-mqtt5-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gnu-make-implicit-rule-resolver-v1", "taskId": "gnu-make-implicit-rule-resolver", "version": "1", "name": "gnu-make-implicit-rule-resolver", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/systems-spec/gnu-make-implicit-rule-resolver" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-modules-minimal-version-selection-v1", "taskId": "go-modules-minimal-version-selection", "version": "1", "name": "go-modules-minimal-version-selection", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/systems-spec/go-modules-minimal-version-selection" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-pst-pst-conformance-v1", "taskId": "go-pst-pst-conformance", "version": "1", "name": "go-pst-pst-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/587" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "go-toml-12-conformance-v1", "taskId": "go-toml-12-conformance", "version": "1", "name": "go-toml-12-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "goblin-linker-reloc-v1", "taskId": "goblin-linker-reloc", "version": "1", "name": "goblin-linker-reloc", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "goccy-go-json-366-v1", "taskId": "goccy-go-json-366", "version": "1", "name": "goccy-go-json-366", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/swepr/goccy-go-json-366" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gogit-partialclone-engine-v1", "taskId": "gogit-partialclone-engine", "version": "1", "name": "gogit-partialclone-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/579" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "goja-temporal-duration-v1", "taskId": "goja-temporal-duration", "version": "1", "name": "goja-temporal-duration", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "golanglru-cache-policies-v1", "taskId": "golanglru-cache-policies", "version": "1", "name": "golanglru-cache-policies", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "goldmark-cmark-sourcepos-v1", "taskId": "goldmark-cmark-sourcepos", "version": "1", "name": "goldmark-cmark-sourcepos", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "goldmark-latex-render-v1", "taskId": "goldmark-latex-render", "version": "1", "name": "goldmark-latex-render", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gomail-imap-engine-v1", "taskId": "gomail-imap-engine", "version": "1", "name": "gomail-imap-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gonids-suricata-rule-conformance-v1", "taskId": "gonids-suricata-rule-conformance", "version": "1", "name": "gonids-suricata-rule-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/583" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gonum-ode-suite-v1", "taskId": "gonum-ode-suite", "version": "1", "name": "gonum-ode-suite", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "goose-migration-checksums-43d2d9c-v1", "taskId": "goose-migration-checksums-43d2d9c", "version": "1", "name": "goose-migration-checksums-43d2d9c", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gopher-lua-54-semantics-v1", "taskId": "gopher-lua-54-semantics", "version": "1", "name": "gopher-lua-54-semantics", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gorillaws-tls13-engine-v1", "taskId": "gorillaws-tls13-engine", "version": "1", "name": "gorillaws-tls13-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gorm-missing-v1", "taskId": "gorm-missing", "version": "1", "name": "gorm-missing", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gorm-window-engine-v1", "taskId": "gorm-window-engine", "version": "1", "name": "gorm-window-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gpxpy-gpx-conformance-v1", "taskId": "gpxpy-gpx-conformance", "version": "1", "name": "gpxpy-gpx-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/560" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "graph-planarity-engine-v1", "taskId": "graph-planarity-engine", "version": "1", "name": "graph-planarity-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "graphology-routing-engine-v1", "taskId": "graphology-routing-engine", "version": "1", "name": "graphology-routing-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "graphql-execution-spec-v1", "taskId": "graphql-execution-spec", "version": "1", "name": "graphql-execution-spec", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "graphviz-sugiyama-layout-v1", "taskId": "graphviz-sugiyama-layout", "version": "1", "name": "graphviz-sugiyama-layout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/557" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "gtfs-realtime-conformance-v1", "taskId": "gtfs-realtime-conformance", "version": "1", "name": "gtfs-realtime-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/583" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "guice-scope-narrowing-check-bb8fb34-v1", "taskId": "guice-scope-narrowing-check-bb8fb34", "version": "1", "name": "guice-scope-narrowing-check-bb8fb34", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "handsontable-formula-engine-v1", "taskId": "handsontable-formula-engine", "version": "1", "name": "handsontable-formula-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "haralyzer-har-conformance-v1", "taskId": "haralyzer-har-conformance", "version": "1", "name": "haralyzer-har-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/560" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "helmet-csp-evaluator-v1", "taskId": "helmet-csp-evaluator", "version": "1", "name": "helmet-csp-evaluator", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hexyl-highlight-byte-ranges-6ecc29b-v1", "taskId": "hexyl-highlight-byte-ranges-6ecc29b", "version": "1", "name": "hexyl-highlight-byte-ranges-6ecc29b", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hl7-conformance-ack-v1", "taskId": "hl7-conformance-ack", "version": "1", "name": "hl7-conformance-ack", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/558" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "holidays-observance-v1", "taskId": "holidays-observance", "version": "1", "name": "holidays-observance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hono-http-cache-rfc9111-v1", "taskId": "hono-http-cache-rfc9111", "version": "1", "name": "hono-http-cache-rfc9111", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hono-range-requests-v1", "taskId": "hono-range-requests", "version": "1", "name": "hono-range-requests", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "html5ever-serialize-v1", "taskId": "html5ever-serialize", "version": "1", "name": "html5ever-serialize", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hyperfine-bench-stats-v1", "taskId": "hyperfine-bench-stats", "version": "1", "name": "hyperfine-bench-stats", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "hyperframe-hpack-h2-v1", "taskId": "hyperframe-hpack-h2", "version": "1", "name": "hyperframe-hpack-h2", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "i18next-plural-engine-v1", "taskId": "i18next-plural-engine", "version": "1", "name": "i18next-plural-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "iam-cross-account-externalid-sourcearn-v1", "taskId": "iam-cross-account-externalid-sourcearn", "version": "1", "name": "iam-cross-account-externalid-sourcearn", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "iam-permissions-boundary-ceiling-v1", "taskId": "iam-permissions-boundary-ceiling", "version": "1", "name": "iam-permissions-boundary-ceiling", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/108" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "iam-revoke-older-sessions-v1", "taskId": "iam-revoke-older-sessions", "version": "1", "name": "iam-revoke-older-sessions", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "iam-session-tag-tenant-scope-v1", "taskId": "iam-session-tag-tenant-scope", "version": "1", "name": "iam-session-tag-tenant-scope", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "icalendar-conformance-v1", "taskId": "icalendar-conformance", "version": "1", "name": "icalendar-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/567" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "icedx86-emulator-v1", "taskId": "icedx86-emulator", "version": "1", "name": "icedx86-emulator", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "iconv-lite-whatwg-mode-v1", "taskId": "iconv-lite-whatwg-mode", "version": "1", "name": "iconv-lite-whatwg-mode", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "idempotency-middleware-v1", "taskId": "idempotency-middleware", "version": "1", "name": "idempotency-middleware", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/54" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "image-quantize-dither-v1", "taskId": "image-quantize-dither", "version": "1", "name": "image-quantize-dither", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "intervaltree-allen-algebra-v1", "taskId": "intervaltree-allen-algebra", "version": "1", "name": "intervaltree-allen-algebra", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ion-binary-conformance-v1", "taskId": "ion-binary-conformance", "version": "1", "name": "ion-binary-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/566" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jcamp-jcampdx-conformance-v1", "taskId": "jcamp-jcampdx-conformance", "version": "1", "name": "jcamp-jcampdx-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/587" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jest-mutation-engine-v1", "taskId": "jest-mutation-engine", "version": "1", "name": "jest-mutation-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/580" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jiff-rrule-recurrence-v1", "taskId": "jiff-rrule-recurrence", "version": "1", "name": "jiff-rrule-recurrence", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jimp-exif-engine-v1", "taskId": "jimp-exif-engine", "version": "1", "name": "jimp-exif-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jimp-resample-v1", "taskId": "jimp-resample", "version": "1", "name": "jimp-resample", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jks-keystore-conformance-v1", "taskId": "jks-keystore-conformance", "version": "1", "name": "jks-keystore-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/565" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jmespath-compliance-v1", "taskId": "jmespath-compliance", "version": "1", "name": "jmespath-compliance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "job-scheduler-queue-v1", "taskId": "job-scheduler-queue", "version": "1", "name": "job-scheduler-queue", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/55" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jose-conformance-v1", "taskId": "jose-conformance", "version": "1", "name": "jose-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/567" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jsbarcode-qr-model2-v1", "taskId": "jsbarcode-qr-model2", "version": "1", "name": "jsbarcode-qr-model2", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/544" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jsonschema-2020-12-v1", "taskId": "jsonschema-2020-12", "version": "1", "name": "jsonschema-2020-12", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jsonschema-hyper-v1", "taskId": "jsonschema-hyper", "version": "1", "name": "jsonschema-hyper", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jsts-polygon-overlay-v1", "taskId": "jsts-polygon-overlay", "version": "1", "name": "jsts-polygon-overlay", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jte-template-fragments-a271c27-v1", "taskId": "jte-template-fragments-a271c27", "version": "1", "name": "jte-template-fragments-a271c27", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "just-parallel-scheduler-v1", "taskId": "just-parallel-scheduler", "version": "1", "name": "just-parallel-scheduler", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/579" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "jwt-jose-engine-v1", "taskId": "jwt-jose-engine", "version": "1", "name": "jwt-jose-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "keyring-shamir-engine-v1", "taskId": "keyring-shamir-engine", "version": "1", "name": "keyring-shamir-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "konva-skeleton-engine-v1", "taskId": "konva-skeleton-engine", "version": "1", "name": "konva-skeleton-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "kurbo-path-boolean-v1", "taskId": "kurbo-path-boolean", "version": "1", "name": "kurbo-path-boolean", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lage-target-timeouts-e1cfae7-v1", "taskId": "lage-target-timeouts-e1cfae7", "version": "1", "name": "lage-target-timeouts-e1cfae7", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lark-earley-sppf-v1", "taskId": "lark-earley-sppf", "version": "1", "name": "lark-earley-sppf", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "las-pointcloud-conformance-v1", "taskId": "las-pointcloud-conformance", "version": "1", "name": "las-pointcloud-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/562" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "leaflet-mapmatch-engine-v1", "taskId": "leaflet-mapmatch-engine", "version": "1", "name": "leaflet-mapmatch-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lingui-localematch-engine-v1", "taskId": "lingui-localematch-engine", "version": "1", "name": "lingui-localematch-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "linkedom-range-selection-v1", "taskId": "linkedom-range-selection", "version": "1", "name": "linkedom-range-selection", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lit-flexbox-layout-v1", "taskId": "lit-flexbox-layout", "version": "1", "name": "lit-flexbox-layout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "llm-ann-index-v1", "taskId": "llm-ann-index", "version": "1", "name": "llm-ann-index", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lnk-shortcut-conformance-v1", "taskId": "lnk-shortcut-conformance", "version": "1", "name": "lnk-shortcut-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/565" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "locust-des-engine-v1", "taskId": "locust-des-engine", "version": "1", "name": "locust-des-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lodash-persistent-engine-v1", "taskId": "lodash-persistent-engine", "version": "1", "name": "lodash-persistent-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/580" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lopdf-content-stream-v1", "taskId": "lopdf-content-stream", "version": "1", "name": "lopdf-content-stream", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "lru-ttl-cache-v1", "taskId": "lru-ttl-cache", "version": "1", "name": "lru-ttl-cache", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/56" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mailparser-emailauth-verify-v1", "taskId": "mailparser-emailauth-verify", "version": "1", "name": "mailparser-emailauth-verify", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mammoth-numbering-v1", "taskId": "mammoth-numbering", "version": "1", "name": "mammoth-numbering", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "marked-commonmark-strict-v1", "taskId": "marked-commonmark-strict", "version": "1", "name": "marked-commonmark-strict", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mathjs-polynomial-engine-v1", "taskId": "mathjs-polynomial-engine", "version": "1", "name": "mathjs-polynomial-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "matroska-ebml-conformance-v1", "taskId": "matroska-ebml-conformance", "version": "1", "name": "matroska-ebml-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/561" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "maven-version-ordering-v1", "taskId": "maven-version-ordering", "version": "1", "name": "maven-version-ordering", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/systems-spec/maven-version-ordering" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "media-chrome-timedtext-v1", "taskId": "media-chrome-timedtext", "version": "1", "name": "media-chrome-timedtext", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "meshio-fem-solver-v1", "taskId": "meshio-fem-solver", "version": "1", "name": "meshio-fem-solver", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mido-tempo-map-v1", "taskId": "mido-tempo-map", "version": "1", "name": "mido-tempo-map", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "miekgdns-mdns-engine-v1", "taskId": "miekgdns-mdns-engine", "version": "1", "name": "miekgdns-mdns-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/582" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "migrate-schema-diff-v1", "taskId": "migrate-schema-diff", "version": "1", "name": "migrate-schema-diff", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mingus-harmony-analysis-v1", "taskId": "mingus-harmony-analysis", "version": "1", "name": "mingus-harmony-analysis", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mkdocs-search-engine-v1", "taskId": "mkdocs-search-engine", "version": "1", "name": "mkdocs-search-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mobi-conformance-v1", "taskId": "mobi-conformance", "version": "1", "name": "mobi-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/563" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mosql-query-optimizer-v1", "taskId": "mosql-query-optimizer", "version": "1", "name": "mosql-query-optimizer", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mp4-isobmff-conformance-v1", "taskId": "mp4-isobmff-conformance", "version": "1", "name": "mp4-isobmff-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/567" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "msgpack-conformance-v1", "taskId": "msgpack-conformance", "version": "1", "name": "msgpack-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/564" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mutagen-id3v2-conformance-v1", "taskId": "mutagen-id3v2-conformance", "version": "1", "name": "mutagen-id3v2-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/584" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mvdan-sh-1259-v1", "taskId": "mvdan-sh-1259", "version": "1", "name": "mvdan-sh-1259", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/swepr/mvdan-sh-1259" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "mysql-binlog-conformance-v1", "taskId": "mysql-binlog-conformance", "version": "1", "name": "mysql-binlog-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/566" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "n3js-rdf-canonicalization-v1", "taskId": "n3js-rdf-canonicalization", "version": "1", "name": "n3js-rdf-canonicalization", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nalgebra-kinematics-engine-v1", "taskId": "nalgebra-kinematics-engine", "version": "1", "name": "nalgebra-kinematics-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nats-io-nats-server-8021-v1", "taskId": "nats-io-nats-server-8021", "version": "1", "name": "nats-io-nats-server-8021", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/swepr/nats-io-nats-server-8021" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "natsort-uca-collation-v1", "taskId": "natsort-uca-collation", "version": "1", "name": "natsort-uca-collation", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/571" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nbio-swim-membership-v1", "taskId": "nbio-swim-membership", "version": "1", "name": "nbio-swim-membership", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nbt-anvil-conformance-v1", "taskId": "nbt-anvil-conformance", "version": "1", "name": "nbt-anvil-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/562" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ndarray-einsum-engine-v1", "taskId": "ndarray-einsum-engine", "version": "1", "name": "ndarray-einsum-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ndarray-linalg-decomp-v1", "taskId": "ndarray-linalg-decomp", "version": "1", "name": "ndarray-linalg-decomp", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nebula-firewall-deny-rules-89178f4-v1", "taskId": "nebula-firewall-deny-rules-89178f4", "version": "1", "name": "nebula-firewall-deny-rules-89178f4", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "networkx-canonical-labeling-v1", "taskId": "networkx-canonical-labeling", "version": "1", "name": "networkx-canonical-labeling", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "networkx-community-engine-v1", "taskId": "networkx-community-engine", "version": "1", "name": "networkx-community-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/579" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "networkx-graphml-full-v1", "taskId": "networkx-graphml-full", "version": "1", "name": "networkx-graphml-full", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nltk-subword-tokenization-v1", "taskId": "nltk-subword-tokenization", "version": "1", "name": "nltk-subword-tokenization", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/543" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "nsq-channel-message-ttl-85cf10c-v1", "taskId": "nsq-channel-message-ttl-85cf10c", "version": "1", "name": "nsq-channel-message-ttl-85cf10c", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ntp-clock-discipline-v1", "taskId": "ntp-clock-discipline", "version": "1", "name": "ntp-clock-discipline", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "numbat-default-parameter-values-49fccb1-v1", "taskId": "numbat-default-parameter-values-49fccb1", "version": "1", "name": "numbat-default-parameter-values-49fccb1", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "numbigint-numbertheory-v1", "taskId": "numbigint-numbertheory", "version": "1", "name": "numbigint-numbertheory", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "occ-conditional-store-v1", "taskId": "occ-conditional-store", "version": "1", "name": "occ-conditional-store", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/93" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ofx-statement-conformance-v1", "taskId": "ofx-statement-conformance", "version": "1", "name": "ofx-statement-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/559" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "olecf-cfb-conformance-v1", "taskId": "olecf-cfb-conformance", "version": "1", "name": "olecf-cfb-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/565" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "openpyxl-condformat-v1", "taskId": "openpyxl-condformat", "version": "1", "name": "openpyxl-condformat", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "openpyxl-formula-engine-v1", "taskId": "openpyxl-formula-engine", "version": "1", "name": "openpyxl-formula-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "openpyxl-numfmt-v1", "taskId": "openpyxl-numfmt", "version": "1", "name": "openpyxl-numfmt", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "opentype-subset-v1", "taskId": "opentype-subset", "version": "1", "name": "opentype-subset", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "orb-projection-engine-v1", "taskId": "orb-projection-engine", "version": "1", "name": "orb-projection-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/547" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "orjson-jsonquery-engine-v1", "taskId": "orjson-jsonquery-engine", "version": "1", "name": "orjson-jsonquery-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "oxigraph-shacl-core-v1", "taskId": "oxigraph-shacl-core", "version": "1", "name": "oxigraph-shacl-core", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "p5-delaunay-engine-v1", "taskId": "p5-delaunay-engine", "version": "1", "name": "p5-delaunay-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "paho-mqtt-broker-v1", "taskId": "paho-mqtt-broker", "version": "1", "name": "paho-mqtt-broker", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/557" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "palette-css-color4-v1", "taskId": "palette-css-color4", "version": "1", "name": "palette-css-color4", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "paramiko-knownhosts-engine-v1", "taskId": "paramiko-knownhosts-engine", "version": "1", "name": "paramiko-knownhosts-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "parking-reservations-v1", "taskId": "parking-reservations", "version": "1", "name": "parking-reservations", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/bigcode/parking-reservations" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "parse5-html-content-model-conformance-v1", "taskId": "parse5-html-content-model-conformance", "version": "1", "name": "parse5-html-content-model-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/583" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "parse5-treediff-engine-v1", "taskId": "parse5-treediff-engine", "version": "1", "name": "parse5-treediff-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "parsimonious-packrat-v1", "taskId": "parsimonious-packrat", "version": "1", "name": "parsimonious-packrat", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "partitura-musicxml-conformance-v1", "taskId": "partitura-musicxml-conformance", "version": "1", "name": "partitura-musicxml-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/560" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "partridge-gtfs-conformance-v1", "taskId": "partridge-gtfs-conformance", "version": "1", "name": "partridge-gtfs-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/559" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "path-to-regexp-urlpattern-v1", "taskId": "path-to-regexp-urlpattern", "version": "1", "name": "path-to-regexp-urlpattern", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pcapng-block-conformance-v1", "taskId": "pcapng-block-conformance", "version": "1", "name": "pcapng-block-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/560" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pdflib-opentype-shaping-v1", "taskId": "pdflib-opentype-shaping", "version": "1", "name": "pdflib-opentype-shaping", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pe-structural-conformance-v1", "taskId": "pe-structural-conformance", "version": "1", "name": "pe-structural-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/564" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pefile-authenticode-verify-v1", "taskId": "pefile-authenticode-verify", "version": "1", "name": "pefile-authenticode-verify", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pelletier-go-toml-934-v1", "taskId": "pelletier-go-toml-934", "version": "1", "name": "pelletier-go-toml-934", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/july/swepr/pelletier-go-toml-934" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "petgraph-flow-v1", "taskId": "petgraph-flow", "version": "1", "name": "petgraph-flow", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "phonemizer-g2p-engine-v1", "taskId": "phonemizer-g2p-engine", "version": "1", "name": "phonemizer-g2p-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "phonenumbers-formatting-v1", "taskId": "phonenumbers-formatting", "version": "1", "name": "phonenumbers-formatting", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "picocli-deprecated-options-10509c0-v1", "taskId": "picocli-deprecated-options-10509c0", "version": "1", "name": "picocli-deprecated-options-10509c0", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pillow-codec-engine-v1", "taskId": "pillow-codec-engine", "version": "1", "name": "pillow-codec-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pint-ucum-engine-v1", "taskId": "pint-ucum-engine", "version": "1", "name": "pint-ucum-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pint-uncertainty-engine-v1", "taskId": "pint-uncertainty-engine", "version": "1", "name": "pint-uncertainty-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/582" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pixijs-atlas-packing-v1", "taskId": "pixijs-atlas-packing", "version": "1", "name": "pixijs-atlas-packing", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pixijs-collision-engine-v1", "taskId": "pixijs-collision-engine", "version": "1", "name": "pixijs-collision-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "png-image-conformance-v1", "taskId": "png-image-conformance", "version": "1", "name": "png-image-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/562" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pola-rs-polars-27763-v1", "taskId": "pola-rs-polars-27763", "version": "1", "name": "pola-rs-polars-27763", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/swepr/pola-rs-polars-27763" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "postcss-custom-props-calc-v1", "taskId": "postcss-custom-props-calc", "version": "1", "name": "postcss-custom-props-calc", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "prance-openapi-request-validator-v1", "taskId": "prance-openapi-request-validator", "version": "1", "name": "prance-openapi-request-validator", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/558" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "prism-incremental-parse-engine-v1", "taskId": "prism-incremental-parse-engine", "version": "1", "name": "prism-incremental-parse-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "promclient-sketch-engine-v1", "taskId": "promclient-sketch-engine", "version": "1", "name": "promclient-sketch-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "promclient-tsdb-compress-engine-v1", "taskId": "promclient-tsdb-compress-engine", "version": "1", "name": "promclient-tsdb-compress-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "prometheus-promql-eval-v1", "taskId": "prometheus-promql-eval", "version": "1", "name": "prometheus-promql-eval", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "prompttoolkit-buffer-engine-v1", "taskId": "prompttoolkit-buffer-engine", "version": "1", "name": "prompttoolkit-buffer-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "prosemirror-crdt-engine-v1", "taskId": "prosemirror-crdt-engine", "version": "1", "name": "prosemirror-crdt-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "prost-textformat-engine-v1", "taskId": "prost-textformat-engine", "version": "1", "name": "prost-textformat-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pug-compiler-v1", "taskId": "pug-compiler", "version": "1", "name": "pug-compiler", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pulp-simplex-solver-v1", "taskId": "pulp-simplex-solver", "version": "1", "name": "pulp-simplex-solver", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pyabf-abf2-conformance-v1", "taskId": "pyabf-abf2-conformance", "version": "1", "name": "pyabf-abf2-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/585" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pyais-ais-conformance-v1", "taskId": "pyais-ais-conformance", "version": "1", "name": "pyais-ais-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/584" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pyasn1-per-codec-v1", "taskId": "pyasn1-per-codec", "version": "1", "name": "pyasn1-per-codec", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pybufrkit-bufr-conformance-v1", "taskId": "pybufrkit-bufr-conformance", "version": "1", "name": "pybufrkit-bufr-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pycdlib-iso9660-conformance-v1", "taskId": "pycdlib-iso9660-conformance", "version": "1", "name": "pycdlib-iso9660-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/585" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pycparser-c-interpreter-v1", "taskId": "pycparser-c-interpreter", "version": "1", "name": "pycparser-c-interpreter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pycparser-preprocessor-engine-v1", "taskId": "pycparser-preprocessor-engine", "version": "1", "name": "pycparser-preprocessor-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/581" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pydub-dsp-engine-v1", "taskId": "pydub-dsp-engine", "version": "1", "name": "pydub-dsp-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/551" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pyelftools-dwarf-eval-v1", "taskId": "pyelftools-dwarf-eval", "version": "1", "name": "pyelftools-dwarf-eval", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pyfdt-dtb-conformance-v1", "taskId": "pyfdt-dtb-conformance", "version": "1", "name": "pyfdt-dtb-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/584" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pygments-lexer-conformance-v1", "taskId": "pygments-lexer-conformance", "version": "1", "name": "pygments-lexer-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pygments-semantic-tokens-v1", "taskId": "pygments-semantic-tokens", "version": "1", "name": "pygments-semantic-tokens", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/557" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pylatexenc-mathml-v1", "taskId": "pylatexenc-mathml", "version": "1", "name": "pylatexenc-mathml", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pyparsing-diagram-ebnf-v1", "taskId": "pyparsing-diagram-ebnf", "version": "1", "name": "pyparsing-diagram-ebnf", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pypdf-revision-forensics-v1", "taskId": "pypdf-revision-forensics", "version": "1", "name": "pypdf-revision-forensics", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pypdf-text-layout-v1", "taskId": "pypdf-text-layout", "version": "1", "name": "pypdf-text-layout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pysmi-smiv2-conformance-v1", "taskId": "pysmi-smiv2-conformance", "version": "1", "name": "pysmi-smiv2-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/591" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pysmiles-canonical-v1", "taskId": "pysmiles-canonical", "version": "1", "name": "pysmiles-canonical", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pysubs2-subtitle-conformance-v1", "taskId": "pysubs2-subtitle-conformance", "version": "1", "name": "pysubs2-subtitle-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/560" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pytest-impact-engine-v1", "taskId": "pytest-impact-engine", "version": "1", "name": "pytest-impact-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/580" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "python-constraint-global-v1", "taskId": "python-constraint-global", "version": "1", "name": "python-constraint-global", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "python-docx-tracked-changes-v1", "taskId": "python-docx-tracked-changes", "version": "1", "name": "python-docx-tracked-changes", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pythoncan-dbc-engine-v1", "taskId": "pythoncan-dbc-engine", "version": "1", "name": "pythoncan-dbc-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pythonmagic-filetype-engine-v1", "taskId": "pythonmagic-filetype-engine", "version": "1", "name": "pythonmagic-filetype-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/579" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pythonpptx-render-engine-v1", "taskId": "pythonpptx-render-engine", "version": "1", "name": "pythonpptx-render-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "pyzbar-1d-decoder-v1", "taskId": "pyzbar-1d-decoder", "version": "1", "name": "pyzbar-1d-decoder", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "quick-xml-c14n-v1", "taskId": "quick-xml-c14n", "version": "1", "name": "quick-xml-c14n", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rardecode-rar-conformance-v1", "taskId": "rardecode-rar-conformance", "version": "1", "name": "rardecode-rar-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/583" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rasn-constraints-v1", "taskId": "rasn-constraints", "version": "1", "name": "rasn-constraints", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ratatui-log-axis-scale-99168f8-v1", "taskId": "ratatui-log-axis-scale-99168f8", "version": "1", "name": "ratatui-log-axis-scale-99168f8", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rate-limiter-v1", "taskId": "rate-limiter", "version": "1", "name": "rate-limiter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/57" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rclone-delta-sync-v1", "taskId": "rclone-delta-sync", "version": "1", "name": "rclone-delta-sync", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rdflib-owlrl-engine-v1", "taskId": "rdflib-owlrl-engine", "version": "1", "name": "rdflib-owlrl-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rdflib-owlrl-reasoner-v1", "taskId": "rdflib-owlrl-reasoner", "version": "1", "name": "rdflib-owlrl-reasoner", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rdflib-shex-validate-v1", "taskId": "rdflib-shex-validate", "version": "1", "name": "rdflib-shex-validate", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "redispy-clientcache-engine-v1", "taskId": "redispy-clientcache-engine", "version": "1", "name": "redispy-clientcache-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/580" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "regexptree-exec-engine-v1", "taskId": "regexptree-exec-engine", "version": "1", "name": "regexptree-exec-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/550" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "regf-hive-conformance-v1", "taskId": "regf-hive-conformance", "version": "1", "name": "regf-hive-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/566" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "requests-charset-engine-v1", "taskId": "requests-charset-engine", "version": "1", "name": "requests-charset-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "resilient-http-client-v1", "taskId": "resilient-http-client", "version": "1", "name": "resilient-http-client", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/58" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "resp-redis-conformance-v1", "taskId": "resp-redis-conformance", "version": "1", "name": "resp-redis-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/568" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "restic-protected-snapshots-ba802d4-v1", "taskId": "restic-protected-snapshots-ba802d4", "version": "1", "name": "restic-protected-snapshots-ba802d4", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "resvg-missing-v1", "taskId": "resvg-missing", "version": "1", "name": "resvg-missing", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0194d8cd196f0e95-v1", "taskId": "review-0194d8cd196f0e95", "version": "1", "name": "tbrain-maintenance-lease-rebalance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-01f3d7dbf8742223-v1", "taskId": "review-01f3d7dbf8742223", "version": "1", "name": "Snorkel-reviewed task review-01f3d7dbf8742223", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-22T143633Z__90f4944e__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-02bd6fa6ecdbac46-v1", "taskId": "review-02bd6fa6ecdbac46", "version": "1", "name": "Snorkel-reviewed task review-02bd6fa6ecdbac46", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-03f63ebccec6b0f7-v1", "taskId": "review-03f63ebccec6b0f7", "version": "1", "name": "Snorkel-reviewed task review-03f63ebccec6b0f7", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 4, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/244" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-05be8df0a0c01932-v1", "taskId": "review-05be8df0a0c01932", "version": "1", "name": "secure-vault-relay-keyring-fix", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-062d3197efb35a99-v1", "taskId": "review-062d3197efb35a99", "version": "1", "name": "Snorkel-reviewed task review-062d3197efb35a99", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/259" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-06a9056cf93dd13b-v1", "taskId": "review-06a9056cf93dd13b", "version": "1", "name": "tideflux-redis-maintenance-workflow", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 4, "p": 0.25, "lo": 0.04558726080970055, "hi": 0.6993581574175981, "tasks": 1, "macro": 0.25, "macroLo": 0.04558726080970055, "macroHi": 0.6993581574175981, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-070ead12e193f6ca-v1", "taskId": "review-070ead12e193f6ca", "version": "1", "name": "oci-referrer-gc-import-reconcile", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0768fc24193f29cc-v1", "taskId": "review-0768fc24193f29cc", "version": "1", "name": "log-forward-relay-daemon", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-07aeefc77ef55bcf-v1", "taskId": "review-07aeefc77ef55bcf", "version": "1", "name": "Snorkel-reviewed task review-07aeefc77ef55bcf", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/173" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-087493fb0b94df25-v1", "taskId": "review-087493fb0b94df25", "version": "1", "name": "Snorkel-reviewed task review-087493fb0b94df25", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/203" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-0b35db363d26cbb2-v1", "taskId": "review-0b35db363d26cbb2", "version": "1", "name": "Snorkel-reviewed task review-0b35db363d26cbb2", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-0d3b0576690eb89c-v1", "taskId": "review-0d3b0576690eb89c", "version": "1", "name": "Snorkel-reviewed task review-0d3b0576690eb89c", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/442" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-14fc424d042a7f44-v1", "taskId": "review-14fc424d042a7f44", "version": "1", "name": "Snorkel-reviewed task review-14fc424d042a7f44", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/171" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-15bdff0ed859f5ac-v1", "taskId": "review-15bdff0ed859f5ac", "version": "1", "name": "Snorkel-reviewed task review-15bdff0ed859f5ac", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T022724Z__d668d333__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1760a912a21c4965-v1", "taskId": "review-1760a912a21c4965", "version": "1", "name": "kcr-control-plane-recovery", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-18a7ff8d596d6982-v1", "taskId": "review-18a7ff8d596d6982", "version": "1", "name": "bdm-nuget-packages-lock", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1a92429ff0332595-v1", "taskId": "review-1a92429ff0332595", "version": "1", "name": "Snorkel-reviewed task review-1a92429ff0332595", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/291" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-1c71625c68049bba-v1", "taskId": "review-1c71625c68049bba", "version": "1", "name": "Snorkel-reviewed task review-1c71625c68049bba", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/319" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-1eb3e7927e5aa06c-v1", "taskId": "review-1eb3e7927e5aa06c", "version": "1", "name": "Snorkel-reviewed task review-1eb3e7927e5aa06c", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x4", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/252" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-21ea5dbde55694e0-v1", "taskId": "review-21ea5dbde55694e0", "version": "1", "name": "Snorkel-reviewed task review-21ea5dbde55694e0", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 4, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/242" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-23b4f908-v1", "taskId": "review-23b4f908", "version": "1", "name": "Snorkel-reviewed task review-23b4f908", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": false, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded", "Reference solution fails" ] }, { "id": "review-25e6caab2ed6b707-v1", "taskId": "review-25e6caab2ed6b707", "version": "1", "name": "linux-sysctl-fleet-administration", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-2f9fd426c334ecac-v1", "taskId": "review-2f9fd426c334ecac", "version": "1", "name": "Snorkel-reviewed task review-2f9fd426c334ecac", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/396" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-34dd023351b5693a-v1", "taskId": "review-34dd023351b5693a", "version": "1", "name": "Snorkel-reviewed task review-34dd023351b5693a", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-3584a91e33f847e2-v1", "taskId": "review-3584a91e33f847e2", "version": "1", "name": "secure-vault-relay-keyring-fix", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3e0c4ece7f2bc1de-v1", "taskId": "review-3e0c4ece7f2bc1de", "version": "1", "name": "nvpl-nested-lifecycle-coordinator", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 11, "n": 15, "p": 0.7333333333333333, "lo": 0.4804956594401943, "hi": 0.8910254667430761, "tasks": 1, "macro": 0.7333333333333333, "macroLo": 0.4804956594401943, "macroHi": 0.8910254667430761, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 9, "n": 15, "p": 0.6, "lo": 0.357468301206246, "hi": 0.8017550385865841, "tasks": 1, "macro": 0.6, "macroLo": 0.357468301206246, "macroHi": 0.8017550385865841, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-3fdc31cfc07ae700-v1", "taskId": "review-3fdc31cfc07ae700", "version": "1", "name": "fix-broken-logrotate", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 13, "n": 15, "p": 0.8666666666666667, "lo": 0.6211801716767764, "hi": 0.9626387408969346, "tasks": 1, "macro": 0.8666666666666667, "macroLo": 0.6211801716767764, "macroHi": 0.9626387408969346, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 9, "n": 15, "p": 0.6, "lo": 0.357468301206246, "hi": 0.8017550385865841, "tasks": 1, "macro": 0.6, "macroLo": 0.357468301206246, "macroHi": 0.8017550385865841, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 15, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-41d8a157d1f21706-v1", "taskId": "review-41d8a157d1f21706", "version": "1", "name": "Snorkel-reviewed task review-41d8a157d1f21706", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-30T133430Z__6f728851__accept" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-448120600d4249a5-v1", "taskId": "review-448120600d4249a5", "version": "1", "name": "Snorkel-reviewed task review-448120600d4249a5", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-29T154851Z__cc2431c8__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-46585b2534e3153a-v1", "taskId": "review-46585b2534e3153a", "version": "1", "name": "Snorkel-reviewed task review-46585b2534e3153a", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/378" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-488b8e17caef3488-v1", "taskId": "review-488b8e17caef3488", "version": "1", "name": "Snorkel-reviewed task review-488b8e17caef3488", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/398" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-518f814461b5229f-v1", "taskId": "review-518f814461b5229f", "version": "1", "name": "Snorkel-reviewed task review-518f814461b5229f", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 4, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T044124Z__fd6e4b38__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-52fa9a8b1bac6661-v1", "taskId": "review-52fa9a8b1bac6661", "version": "1", "name": "Snorkel-reviewed task review-52fa9a8b1bac6661", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/135" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-533a90519e39e5e6-v1", "taskId": "review-533a90519e39e5e6", "version": "1", "name": "rust-kilnpass-optpipe-reweave", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-53ec9604222ed36f-v1", "taskId": "review-53ec9604222ed36f", "version": "1", "name": "bdm-nuget-packages-lock", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-54a9f3c205c8e092-v1", "taskId": "review-54a9f3c205c8e092", "version": "1", "name": "coverage-map-json", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-56f4287218307201-v1", "taskId": "review-56f4287218307201", "version": "1", "name": "migrate-strace-lsof-evidence-into-php", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-594623927b21bea5-v1", "taskId": "review-594623927b21bea5", "version": "1", "name": "Snorkel-reviewed task review-594623927b21bea5", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/188" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5c400b7834d5e9d6-v1", "taskId": "review-5c400b7834d5e9d6", "version": "1", "name": "mirror-quota-reclaim", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5c7325b28b8dfd55-v1", "taskId": "review-5c7325b28b8dfd55", "version": "1", "name": "Snorkel-reviewed task review-5c7325b28b8dfd55", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-29T153908Z__02b606b1__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-5cb4b7332832b796-v1", "taskId": "review-5cb4b7332832b796", "version": "1", "name": "Snorkel-reviewed task review-5cb4b7332832b796", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-5ce6defca5bf5c93-v1", "taskId": "review-5ce6defca5bf5c93", "version": "1", "name": "repair-cmake-release-bundle", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-603765a61eaee384-v1", "taskId": "review-603765a61eaee384", "version": "1", "name": "Snorkel-reviewed task review-603765a61eaee384", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/447" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-61768408fc62f713-v1", "taskId": "review-61768408fc62f713", "version": "1", "name": "go-workspace-vendor-sum-recovery-planner", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-645fab15d19ecbcd-v1", "taskId": "review-645fab15d19ecbcd", "version": "1", "name": "Snorkel-reviewed task review-645fab15d19ecbcd", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/413" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6575a3468a39a4e4-v1", "taskId": "review-6575a3468a39a4e4", "version": "1", "name": "Snorkel-reviewed task review-6575a3468a39a4e4", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/223" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6a8211413f616099-v1", "taskId": "review-6a8211413f616099", "version": "1", "name": "openvswitch-ct-zone-recycle", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-6b4fd5606d899402-v1", "taskId": "review-6b4fd5606d899402", "version": "1", "name": "Snorkel-reviewed task review-6b4fd5606d899402", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 4, "p": 0.5, "lo": 0.15003898915214953, "hi": 0.8499610108478505, "tasks": 1, "macro": 0.5, "macroLo": 0.15003898915214953, "macroHi": 0.8499610108478505, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7380e53a30145ec3-v1", "taskId": "review-7380e53a30145ec3", "version": "1", "name": "redis-failover", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-739ddcc29e07c2c7-v1", "taskId": "review-739ddcc29e07c2c7", "version": "1", "name": "Snorkel-reviewed task review-739ddcc29e07c2c7", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/316" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-74afffea97cbf88e-v1", "taskId": "review-74afffea97cbf88e", "version": "1", "name": "apt-snapshot-offline-bootstrap", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-75196631c5dc0f1f-v1", "taskId": "review-75196631c5dc0f1f", "version": "1", "name": "Snorkel-reviewed task review-75196631c5dc0f1f", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/429" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-775cb8af317b9e22-v1", "taskId": "review-775cb8af317b9e22", "version": "1", "name": "oci-referrer-gc-import-reconcile", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7995aa85261bb001-v1", "taskId": "review-7995aa85261bb001", "version": "1", "name": "Snorkel-reviewed task review-7995aa85261bb001", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-79fa7c6c78cb95b0-v1", "taskId": "review-79fa7c6c78cb95b0", "version": "1", "name": "Snorkel-reviewed task review-79fa7c6c78cb95b0", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/240" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7a3a6b81ce653ca6-v1", "taskId": "review-7a3a6b81ce653ca6", "version": "1", "name": "Snorkel-reviewed task review-7a3a6b81ce653ca6", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/284" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-7bbf32840ac8e20d-v1", "taskId": "review-7bbf32840ac8e20d", "version": "1", "name": "cargo-sigverify-core-submodule-triage", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7d574292d6fe9c35-v1", "taskId": "review-7d574292d6fe9c35", "version": "1", "name": "openvswitch-ct-zone-recycle", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-7f49e1ea5401ddaa-v1", "taskId": "review-7f49e1ea5401ddaa", "version": "1", "name": "tls-bundle-deploy-recovery", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-80099dc2c0df9adc-v1", "taskId": "review-80099dc2c0df9adc", "version": "1", "name": "isolate-cargo-host-target-artifact-cache", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-80876dcf24aaacb0-v1", "taskId": "review-80876dcf24aaacb0", "version": "1", "name": "apt-snapshot-offline-bootstrap", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-820b3f12a795d2a8-v1", "taskId": "review-820b3f12a795d2a8", "version": "1", "name": "Snorkel-reviewed task review-820b3f12a795d2a8", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/144" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8253a930561830b0-v1", "taskId": "review-8253a930561830b0", "version": "1", "name": "Snorkel-reviewed task review-8253a930561830b0", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-8261be61e4223321-v1", "taskId": "review-8261be61e4223321", "version": "1", "name": "Snorkel-reviewed task review-8261be61e4223321", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/461" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-85d7890fcb351066-v1", "taskId": "review-85d7890fcb351066", "version": "1", "name": "openssh-policy-planner", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1, needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 7, "n": 9, "p": 0.7777777777777778, "lo": 0.4525889691069889, "hi": 0.9367748928821533, "tasks": 1, "macro": 0.7777777777777778, "macroLo": 0.4525889691069889, "macroHi": 0.9367748928821533, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-89c107a748bafe50-v1", "taskId": "review-89c107a748bafe50", "version": "1", "name": "tbrain-hexdump-words", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-920b0a812f7bce15-v1", "taskId": "review-920b0a812f7bce15", "version": "1", "name": "Snorkel-reviewed task review-920b0a812f7bce15", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/190" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9260645bf8fae735-v1", "taskId": "review-9260645bf8fae735", "version": "1", "name": "Snorkel-reviewed task review-9260645bf8fae735", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/446" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-940156d658350fbb-v1", "taskId": "review-940156d658350fbb", "version": "1", "name": "Snorkel-reviewed task review-940156d658350fbb", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/426" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-99eeb6cd02bc520e-v1", "taskId": "review-99eeb6cd02bc520e", "version": "1", "name": "Snorkel-reviewed task review-99eeb6cd02bc520e", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/272" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-9d6bb34cd1455cdb-v1", "taskId": "review-9d6bb34cd1455cdb", "version": "1", "name": "Snorkel-reviewed task review-9d6bb34cd1455cdb", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-05T025017Z__a4739752__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a14988792d950989-v1", "taskId": "review-a14988792d950989", "version": "1", "name": "Snorkel-reviewed task review-a14988792d950989", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x2", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a553850bc0203a3c-v1", "taskId": "review-a553850bc0203a3c", "version": "1", "name": "tcl-rsync-snapshot-integrity-audit", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a6900450a4780650-v1", "taskId": "review-a6900450a4780650", "version": "1", "name": "Snorkel-reviewed task review-a6900450a4780650", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/283" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a6b7e9263af045ad-v1", "taskId": "review-a6b7e9263af045ad", "version": "1", "name": "healthcheck-export-fixed", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a76b8fd69c88e979-v1", "taskId": "review-a76b8fd69c88e979", "version": "1", "name": "Snorkel-reviewed task review-a76b8fd69c88e979", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/213" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-a793a6fea8e1e135-v1", "taskId": "review-a793a6fea8e1e135", "version": "1", "name": "Snorkel-reviewed task review-a793a6fea8e1e135", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/324" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a90d30d81b354411-v1", "taskId": "review-a90d30d81b354411", "version": "1", "name": "Snorkel-reviewed task review-a90d30d81b354411", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/377" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-a9575309b133f61f-v1", "taskId": "review-a9575309b133f61f", "version": "1", "name": "Snorkel-reviewed task review-a9575309b133f61f", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/143" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ac00b321b484ed1e-v1", "taskId": "review-ac00b321b484ed1e", "version": "1", "name": "Snorkel-reviewed task review-ac00b321b484ed1e", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [ "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 10, "p": 0.5, "lo": 0.236593090512564, "hi": 0.7634069094874361, "tasks": 1, "macro": 0.5, "macroLo": 0.236593090512564, "macroHi": 0.7634069094874361, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/222" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-ac9a8f9f4be4ff7a-v1", "taskId": "review-ac9a8f9f4be4ff7a", "version": "1", "name": "Snorkel-reviewed task review-ac9a8f9f4be4ff7a", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/331" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-afa3c599889fb01f-v1", "taskId": "review-afa3c599889fb01f", "version": "1", "name": "Snorkel-reviewed task review-afa3c599889fb01f", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/303" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b27bdeb938e070a2-v1", "taskId": "review-b27bdeb938e070a2", "version": "1", "name": "kernel-policy-convergence", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x3", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 4, "p": 0.75, "lo": 0.30064184258240184, "hi": 0.9544127391902995, "tasks": 1, "macro": 0.75, "macroLo": 0.30064184258240184, "macroHi": 0.9544127391902995, "labelled": 0, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b34d9ac765c4055f-v1", "taskId": "review-b34d9ac765c4055f", "version": "1", "name": "Snorkel-reviewed task review-b34d9ac765c4055f", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 0, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-06-29T135914Z__6fe110cc__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b5d04ff85d5af070-v1", "taskId": "review-b5d04ff85d5af070", "version": "1", "name": "Snorkel-reviewed task review-b5d04ff85d5af070", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/264" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-b6808f38b8c7d81d-v1", "taskId": "review-b6808f38b8c7d81d", "version": "1", "name": "repair-cmake-release-bundle", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-b71ea4a732ff3d72-v1", "taskId": "review-b71ea4a732ff3d72", "version": "1", "name": "relpack-npm10-payload-selector", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-bb804f67488709e6-v1", "taskId": "review-bb804f67488709e6", "version": "1", "name": "Snorkel-reviewed task review-bb804f67488709e6", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 2, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/290" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-bb9534d12fa0b5b4-v1", "taskId": "review-bb9534d12fa0b5b4", "version": "1", "name": "Snorkel-reviewed task review-bb9534d12fa0b5b4", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/181" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c25e85da-v1", "taskId": "review-c25e85da", "version": "1", "name": "Snorkel-reviewed task review-c25e85da", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/161" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c2826cf6941209c3-v1", "taskId": "review-c2826cf6941209c3", "version": "1", "name": "Snorkel-reviewed task review-c2826cf6941209c3", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 5, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/138" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c2fe2c21131cfbae-v1", "taskId": "review-c2fe2c21131cfbae", "version": "1", "name": "Snorkel-reviewed task review-c2fe2c21131cfbae", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot.", "Recorded concern about the reference solution or reference evidence." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/201" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-c377ee4e485f3035-v1", "taskId": "review-c377ee4e485f3035", "version": "1", "name": "was-scan-policy-cookie-drift", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c39aae8eb4c2b644-v1", "taskId": "review-c39aae8eb4c2b644", "version": "1", "name": "Snorkel-reviewed task review-c39aae8eb4c2b644", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/125" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-c58162b527cae7ca-v1", "taskId": "review-c58162b527cae7ca", "version": "1", "name": "02-go-pruned-vendor-resolver", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-cc8d0f06db175fc9-v1", "taskId": "review-cc8d0f06db175fc9", "version": "1", "name": "Snorkel-reviewed task review-cc8d0f06db175fc9", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/452" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-cddb4a1540ded144-v1", "taskId": "review-cddb4a1540ded144", "version": "1", "name": "riggingloft-host-v6b", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cdeee269f5c33766-v1", "taskId": "review-cdeee269f5c33766", "version": "1", "name": "Snorkel-reviewed task review-cdeee269f5c33766", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/286" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-cdfdca32793036c4-v1", "taskId": "review-cdfdca32793036c4", "version": "1", "name": "Snorkel-reviewed task review-cdfdca32793036c4", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 3, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 3, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": false, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/433" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d2a2f9e5dafdd6b4-v1", "taskId": "review-d2a2f9e5dafdd6b4", "version": "1", "name": "Snorkel-reviewed task review-d2a2f9e5dafdd6b4", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/423" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d4f2c5fe0aaacd40-v1", "taskId": "review-d4f2c5fe0aaacd40", "version": "1", "name": "Snorkel-reviewed task review-d4f2c5fe0aaacd40", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 1, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/239" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d5858985736fa2cf-v1", "taskId": "review-d5858985736fa2cf", "version": "1", "name": "secure-vault-relay-keyring-fix", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d58f8f66098c2ccc-v1", "taskId": "review-d58f8f66098c2ccc", "version": "1", "name": "redis-failover", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-d6fe6ff88a9e2197-v1", "taskId": "review-d6fe6ff88a9e2197", "version": "1", "name": "maven-catalog-build", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d81bce10c21c2503-v1", "taskId": "review-d81bce10c21c2503", "version": "1", "name": "Snorkel-reviewed task review-d81bce10c21c2503", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/454" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-d89645717e317e8e-v1", "taskId": "review-d89645717e317e8e", "version": "1", "name": "mirror-quota-reclaim", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e1ff7c47f9d3c3b1-v1", "taskId": "review-e1ff7c47f9d3c3b1", "version": "1", "name": "Snorkel-reviewed task review-e1ff7c47f9d3c3b1", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/450" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-e232cb1227e1324c-v1", "taskId": "review-e232cb1227e1324c", "version": "1", "name": "tbrain-log-forward-relay", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e2f972171df85b01-v1", "taskId": "review-e2f972171df85b01", "version": "1", "name": "was-scan-policy-cookie-drift", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e573927e0ebfedd8-v1", "taskId": "review-e573927e0ebfedd8", "version": "1", "name": "cpp-build-reconciliation", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-e63c197eb05a49dd-v1", "taskId": "review-e63c197eb05a49dd", "version": "1", "name": "Snorkel-reviewed task review-e63c197eb05a49dd", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/202" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e880f1ba5eef5c27-v1", "taskId": "review-e880f1ba5eef5c27", "version": "1", "name": "Snorkel-reviewed task review-e880f1ba5eef5c27", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/370" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-e8cd35c501d91d90-v1", "taskId": "review-e8cd35c501d91d90", "version": "1", "name": "Snorkel-reviewed task review-e8cd35c501d91d90", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/314" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-eaad17303f504102-v1", "taskId": "review-eaad17303f504102", "version": "1", "name": "compiler-diagnostic-correlation-framework", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ec26ed5bdf1cada9-v1", "taskId": "review-ec26ed5bdf1cada9", "version": "1", "name": "Snorkel-reviewed task review-ec26ed5bdf1cada9", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/360" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-ed8b9c86ffbee6cb-v1", "taskId": "review-ed8b9c86ffbee6cb", "version": "1", "name": ".", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/451" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-f066c9ce6a4881d8-v1", "taskId": "review-f066c9ce6a4881d8", "version": "1", "name": "task", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-06T181107Z__b0fe6c7b__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-f117a90fc4d2a39b-v1", "taskId": "review-f117a90fc4d2a39b", "version": "1", "name": "Snorkel-reviewed task review-f117a90fc4d2a39b", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/355" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f21987f8a922a460-v1", "taskId": "review-f21987f8a922a460", "version": "1", "name": "mirror-repair-resolver", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f32ec8af0f6f3bc4-v1", "taskId": "review-f32ec8af0f6f3bc4", "version": "1", "name": "bdm-pub-cache-offline-pin", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f3dc35bcbbde52e4-v1", "taskId": "review-f3dc35bcbbde52e4", "version": "1", "name": "reconcile-delegated-admin-authority-ledger", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f51ff5d5c940c908-v1", "taskId": "review-f51ff5d5c940c908", "version": "1", "name": "Snorkel-reviewed task review-f51ff5d5c940c908", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/457" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-f577f4b998484ba6-v1", "taskId": "review-f577f4b998484ba6", "version": "1", "name": "Snorkel-reviewed task review-f577f4b998484ba6", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/455" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-f65706ba03f73eed-v1", "taskId": "review-f65706ba03f73eed", "version": "1", "name": "mirror-quota-reclaim", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f6dd59def7c666de-v1", "taskId": "review-f6dd59def7c666de", "version": "1", "name": "Snorkel-reviewed task review-f6dd59def7c666de", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 4, "n": 5, "p": 0.8, "lo": 0.3755346297625253, "hi": 0.9637758913675698, "tasks": 1, "macro": 0.8, "macroLo": 0.3755346297625253, "macroHi": 0.9637758913675698, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/367" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-f86559ec1c68adc2-v1", "taskId": "review-f86559ec1c68adc2", "version": "1", "name": "Snorkel-reviewed task review-f86559ec1c68adc2", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about the instruction or evaluation contract.", "Recorded concern about verifier behavior or coverage.", "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 3, "n": 5, "p": 0.6, "lo": 0.2307242812760128, "hi": 0.882379225767352, "tasks": 1, "macro": 0.6, "macroLo": 0.2307242812760128, "macroHi": 0.882379225767352, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/197" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-fafcb14b16933697-v1", "taskId": "review-fafcb14b16933697", "version": "1", "name": "task", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/snorkel-reviewed-tasks/2026-07-07T000000Z__9cfb0331-a578-447b-9e12-5c94b99f4d65__needs-revision" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fb417542dc5a77a4-v1", "taskId": "review-fb417542dc5a77a4", "version": "1", "name": "Snorkel-reviewed task review-fb417542dc5a77a4", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [ "Recorded concern about verifier behavior or coverage." ] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/458" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "review-fdd7deb090c3286f-v1", "taskId": "review-fdd7deb090c3286f", "version": "1", "name": "Snorkel-reviewed task review-fdd7deb090c3286f", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/390" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-fe716c693a0458ce-v1", "taskId": "review-fe716c693a0458ce", "version": "1", "name": "Snorkel-reviewed task review-fe716c693a0458ce", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/192" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-deb-build-profile-resolve-v1", "taskId": "review-slug-deb-build-profile-resolve", "version": "1", "name": "deb-build-profile-resolve", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 5, "p": 0.2, "lo": 0.036224108632430085, "hi": 0.6244653702374747, "tasks": 1, "macro": 0.2, "macroLo": 0.036224108632430085, "macroHi": 0.6244653702374747, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-dispatch-registry-resume-skew-v1", "taskId": "review-slug-dispatch-registry-resume-skew", "version": "1", "name": "dispatch-registry-resume-skew", "domain": "Software engineering", "source": "repo", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: accept x1", "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 0, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-journal-link-hard-interactions-revision-v1", "taskId": "review-slug-journal-link-hard-interactions-revision", "version": "1", "name": "journal-link-hard-interactions-revision", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1", "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "review-slug-relay-wire-23-vendor-rebase-v1", "taskId": "review-slug-relay-wire-23-vendor-rebase", "version": "1", "name": "relay-wire-23-vendor-rebase", "domain": "Software engineering", "source": "repo", "tier": "contested/defective", "calibrated": false, "benchVersion": false, "curated": false, "selectionKnown": false, "quality": { "oraclePass": true, "nopPass": false, "qaVerdict": "snorkel review: needs-revision x1, accept x1", "knownDefects": [ "Documented task-quality concern; detailed evidence remains in the source snapshot." ] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/257" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [ "Known task defect recorded" ] }, { "id": "rhai-let-destructuring-950b724-v1", "taskId": "rhai-let-destructuring-950b724", "version": "1", "name": "rhai-let-destructuring-950b724", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rich-paragraph-layout-v1", "taskId": "rich-paragraph-layout", "version": "1", "name": "rich-paragraph-layout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "roxmltree-relaxng-v1", "taskId": "roxmltree-relaxng", "version": "1", "name": "roxmltree-relaxng", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rphonetic-encoders-v1", "taskId": "rphonetic-encoders", "version": "1", "name": "rphonetic-encoders", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ruff-python-interpreter-v1", "taskId": "ruff-python-interpreter", "version": "1", "name": "ruff-python-interpreter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rustbio-phylogenetics-v1", "taskId": "rustbio-phylogenetics", "version": "1", "name": "rustbio-phylogenetics", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "rustquant-option-pricing-v1", "taskId": "rustquant-option-pricing", "version": "1", "name": "rustquant-option-pricing", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "s3-lambda-ddb-pipeline-v1", "taskId": "s3-lambda-ddb-pipeline", "version": "1", "name": "s3-lambda-ddb-pipeline", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "s3-sqs-image-pipeline-kms-v1", "taskId": "s3-sqs-image-pipeline-kms", "version": "1", "name": "s3-sqs-image-pipeline-kms", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "scapy-protocol-dissect-v1", "taskId": "scapy-protocol-dissect", "version": "1", "name": "scapy-protocol-dissect", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "schedule-dag-engine-v1", "taskId": "schedule-dag-engine", "version": "1", "name": "schedule-dag-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "scrapy-readability-engine-v1", "taskId": "scrapy-readability-engine", "version": "1", "name": "scrapy-readability-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "seaborn-datetime-regression-plots-f04b6cd-v1", "taskId": "seaborn-datetime-regression-plots-f04b6cd", "version": "1", "name": "seaborn-datetime-regression-plots-f04b6cd", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "secrets-rotation-kms-v1", "taskId": "secrets-rotation-kms", "version": "1", "name": "secrets-rotation-kms", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "semver-pubgrub-resolver-v1", "taskId": "semver-pubgrub-resolver", "version": "1", "name": "semver-pubgrub-resolver", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/549" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sequelize-ledger-engine-v1", "taskId": "sequelize-ledger-engine", "version": "1", "name": "sequelize-ledger-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "session-token-verify-v1", "taskId": "session-token-verify", "version": "1", "name": "session-token-verify", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/94" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sevenzip-conformance-v1", "taskId": "sevenzip-conformance", "version": "1", "name": "sevenzip-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/563" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfn-saga-compensation-orchestrator-v1", "taskId": "sfn-saga-compensation-orchestrator", "version": "1", "name": "sfn-saga-compensation-orchestrator", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sfn-secrets-rotation-chain-v1", "taskId": "sfn-secrets-rotation-chain", "version": "1", "name": "sfn-secrets-rotation-chain", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/82" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "shiki-textmate-engine-v1", "taskId": "shiki-textmate-engine", "version": "1", "name": "shiki-textmate-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "shiro-deny-permissions-27bf094-v1", "taskId": "shiro-deny-permissions-27bf094", "version": "1", "name": "shiro-deny-permissions-27bf094", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sigma-forcelayout-engine-v1", "taskId": "sigma-forcelayout-engine", "version": "1", "name": "sigma-forcelayout-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "simplefix-fix-conformance-v1", "taskId": "simplefix-fix-conformance", "version": "1", "name": "simplefix-fix-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/591" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "simplestats-state-estimation-v1", "taskId": "simplestats-state-estimation", "version": "1", "name": "simplestats-state-estimation", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "slate-ot-engine-v1", "taskId": "slate-ot-engine", "version": "1", "name": "slate-ot-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "smartcore-gbdt-v1", "taskId": "smartcore-gbdt", "version": "1", "name": "smartcore-gbdt", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "soft-serve-protected-branches-37685d3-v1", "taskId": "soft-serve-protected-branches-37685d3", "version": "1", "name": "soft-serve-protected-branches-37685d3", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "spdx-matching-guidelines-v1", "taskId": "spdx-matching-guidelines", "version": "1", "name": "spdx-matching-guidelines", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/533" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sprs-sparse-solvers-v1", "taskId": "sprs-sparse-solvers", "version": "1", "name": "sprs-sparse-solvers", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sql-where-3vl-v1", "taskId": "sql-where-3vl", "version": "1", "name": "sql-where-3vl", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/75" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sqlc-missing-v1", "taskId": "sqlc-missing", "version": "1", "name": "sqlc-missing", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/538" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sqlglot-datalog-engine-v1", "taskId": "sqlglot-datalog-engine", "version": "1", "name": "sqlglot-datalog-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sqliteutils-btree-engine-v1", "taskId": "sqliteutils-btree-engine", "version": "1", "name": "sqliteutils-btree-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/545" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sqlparse-static-lint-v1", "taskId": "sqlparse-static-lint", "version": "1", "name": "sqlparse-static-lint", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/557" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sqlparser-exec-engine-v1", "taskId": "sqlparser-exec-engine", "version": "1", "name": "sqlparser-exec-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/534" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "srctools-bsp-conformance-v1", "taskId": "srctools-bsp-conformance", "version": "1", "name": "srctools-bsp-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "statrs-hypothesis-tests-v1", "taskId": "statrs-hypothesis-tests", "version": "1", "name": "statrs-hypothesis-tests", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "statsmodels-mcmc-engine-v1", "taskId": "statsmodels-mcmc-engine", "version": "1", "name": "statsmodels-mcmc-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "stdnum-checkdigit-algebra-v1", "taskId": "stdnum-checkdigit-algebra", "version": "1", "name": "stdnum-checkdigit-algebra", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "stomp-frame-conformance-v1", "taskId": "stomp-frame-conformance", "version": "1", "name": "stomp-frame-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/568" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "svgelements-profile-conformance-v1", "taskId": "svgelements-profile-conformance", "version": "1", "name": "svgelements-profile-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/560" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "svgo-path-geometry-v1", "taskId": "svgo-path-geometry", "version": "1", "name": "svgo-path-geometry", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "sympy-coding-theory-v1", "taskId": "sympy-coding-theory", "version": "1", "name": "sympy-coding-theory", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "syncthing-merkle-engine-v1", "taskId": "syncthing-merkle-engine", "version": "1", "name": "syncthing-merkle-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tablib-pivot-engine-v1", "taskId": "tablib-pivot-engine", "version": "1", "name": "tablib-pivot-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/541" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "taskfiles-v1", "taskId": "taskfiles", "version": "1", "name": "taskfiles", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/592" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tenacity-resilience-engine-v1", "taskId": "tenacity-resilience-engine", "version": "1", "name": "tenacity-resilience-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "terser-const-eval-v1", "taskId": "terser-const-eval", "version": "1", "name": "terser-const-eval", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "testify-approval-engine-v1", "taskId": "testify-approval-engine", "version": "1", "name": "testify-approval-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/581" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "textx-model-interpreter-v1", "taskId": "textx-model-interpreter", "version": "1", "name": "textx-model-interpreter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "thefuzz-phonetic-engine-v1", "taskId": "thefuzz-phonetic-engine", "version": "1", "name": "thefuzz-phonetic-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/586" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "threejs-slicer-engine-v1", "taskId": "threejs-slicer-engine", "version": "1", "name": "threejs-slicer-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "thrift-binary-conformance-v1", "taskId": "thrift-binary-conformance", "version": "1", "name": "thrift-binary-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/566" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tile38-sortby-field-ordering-a9f953e-v1", "taskId": "tile38-sortby-field-ordering-a9f953e", "version": "1", "name": "tile38-sortby-field-ordering-a9f953e", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/595" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tinycss2-cascade-engine-v1", "taskId": "tinycss2-cascade-engine", "version": "1", "name": "tinycss2-cascade-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tokei-complexity-engine-v1", "taskId": "tokei-complexity-engine", "version": "1", "name": "tokei-complexity-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "tone-soundfont-engine-v1", "taskId": "tone-soundfont-engine", "version": "1", "name": "tone-soundfont-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "transitions-scxml-engine-v1", "taskId": "transitions-scxml-engine", "version": "1", "name": "transitions-scxml-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "trie-router-v1", "taskId": "trie-router", "version": "1", "name": "trie-router", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/59" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "trimesh-boolean-engine-v1", "taskId": "trimesh-boolean-engine", "version": "1", "name": "trimesh-boolean-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/572" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "ttfparser-tt-hinting-v1", "taskId": "ttfparser-tt-hinting", "version": "1", "name": "ttfparser-tt-hinting", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/539" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "turf-dggs-indexing-v1", "taskId": "turf-dggs-indexing", "version": "1", "name": "turf-dggs-indexing", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/553" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "turndown-html-markdown-v1", "taskId": "turndown-html-markdown", "version": "1", "name": "turndown-html-markdown", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/540" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "typos-hunspell-engine-v1", "taskId": "typos-hunspell-engine", "version": "1", "name": "typos-hunspell-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uefi-firmware-pi-conformance-v1", "taskId": "uefi-firmware-pi-conformance", "version": "1", "name": "uefi-firmware-pi-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "unattributed-v1", "taskId": "unattributed", "version": "1", "name": "unattributed", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/531" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uncertainties-montecarlo-v1", "taskId": "uncertainties-montecarlo", "version": "1", "name": "uncertainties-montecarlo", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/554" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "undici-cookiejar-engine-v1", "taskId": "undici-cookiejar-engine", "version": "1", "name": "undici-cookiejar-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/577" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "unicode-identifier-security-v1", "taskId": "unicode-identifier-security", "version": "1", "name": "unicode-identifier-security", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/556" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uniseg-bidi-uax9-v1", "taskId": "uniseg-bidi-uax9", "version": "1", "name": "uniseg-bidi-uax9", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/532" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uniseg-uax14-engine-v1", "taskId": "uniseg-uax14-engine", "version": "1", "name": "uniseg-uax14-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/582" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "unitypy-serializedfile-conformance-v1", "taskId": "unitypy-serializedfile-conformance", "version": "1", "name": "unitypy-serializedfile-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/578" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "urfave-cli-2319-v1", "taskId": "urfave-cli-2319", "version": "1", "name": "urfave-cli-2319", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/tree/main/terminus-accepted/june/swepr/urfave-cli-2319" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "uritemplate-router-v1", "taskId": "uritemplate-router", "version": "1", "name": "uritemplate-router", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/592" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "validatorjs-uts39-engine-v1", "taskId": "validatorjs-uts39-engine", "version": "1", "name": "validatorjs-uts39-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vcf-variant-conformance-v1", "taskId": "vcf-variant-conformance", "version": "1", "name": "vcf-variant-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/561" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vegeta-slo-analytics-v1", "taskId": "vegeta-slo-analytics", "version": "1", "name": "vegeta-slo-analytics", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/575" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vexflow-harmony-engine-v1", "taskId": "vexflow-harmony-engine", "version": "1", "name": "vexflow-harmony-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/588" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "viper-config-precedence-v1", "taskId": "viper-config-precedence", "version": "1", "name": "viper-config-precedence", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "vobject-vcard-engine-v1", "taskId": "vobject-vcard-engine", "version": "1", "name": "vobject-vcard-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/537" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wal-kv-store-v1", "taskId": "wal-kv-store", "version": "1", "name": "wal-kv-store", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/60" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "warcio-conformance-v1", "taskId": "warcio-conformance", "version": "1", "name": "warcio-conformance", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/559" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "wasm-mvp-interpreter-v1", "taskId": "wasm-mvp-interpreter", "version": "1", "name": "wasm-mvp-interpreter", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/535" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "watchdog-watch-depth-limit-325508d-v1", "taskId": "watchdog-watch-depth-limit-325508d", "version": "1", "name": "watchdog-watch-depth-limit-325508d", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/596" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "watermill-saga-engine-v1", "taskId": "watermill-saga-engine", "version": "1", "name": "watermill-saga-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/574" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "weasyprint-css-module-v1", "taskId": "weasyprint-css-module", "version": "1", "name": "weasyprint-css-module", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/536" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "weasyprint-hyphenation-engine-v1", "taskId": "weasyprint-hyphenation-engine", "version": "1", "name": "weasyprint-hyphenation-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "webob-content-negotiation-v1", "taskId": "webob-content-negotiation", "version": "1", "name": "webob-content-negotiation", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/558" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "whoosh-spelling-v1", "taskId": "whoosh-spelling", "version": "1", "name": "whoosh-spelling", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/555" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "window-aggregate-store-v1", "taskId": "window-aggregate-store", "version": "1", "name": "window-aggregate-store", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/105" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "winston-grok-engine-v1", "taskId": "winston-grok-engine", "version": "1", "name": "winston-grok-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/576" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "xyflow-dag-layout-v1", "taskId": "xyflow-dag-layout", "version": "1", "name": "xyflow-dag-layout", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/573" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "yfinance-pricing-engine-v1", "taskId": "yfinance-pricing-engine", "version": "1", "name": "yfinance-pricing-engine", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": { "repo": "https://github.com/Syncrope/trial-datasets/pull/589" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "zrender-group-opacity-3491110-v1", "taskId": "zrender-group-opacity-3491110", "version": "1", "name": "zrender-group-opacity-3491110", "domain": "Software engineering", "source": "repo", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [], "links": {}, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_132026dc282647ce-v1", "taskId": "task_132026dc282647ce", "version": "1", "name": "appsync-graphql-cognito-resolver-cache-leak", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 3, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_132026dc282647ce" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_18c61d7c9cb649e6-v1", "taskId": "task_18c61d7c9cb649e6", "version": "1", "name": "ecr-image-scan-lifecycle-immutable-tags-replication", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 3, "n": 9, "p": 0.3333333333333333, "lo": 0.1205838183869109, "hi": 0.6457978644196036, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.1205838183869109, "macroHi": 0.6457978644196036, "labelled": 9, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 2, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.4444444444444444, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_18c61d7c9cb649e6" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2c29c577532f4923-v1", "taskId": "task_2c29c577532f4923", "version": "1", "name": "sfn-secrets-rotation-chain", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 1, "n": 9, "p": 0.1111111111111111, "lo": 0.019890887638544935, "hi": 0.434999705576656, "tasks": 1, "macro": 0.1111111111111111, "macroLo": 0.019890887638544935, "macroHi": 0.434999705576656, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 8, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 1 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.8888888888888888, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_2c29c577532f4923" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_337df7bf8f3649c4-v1", "taskId": "task_337df7bf8f3649c4", "version": "1", "name": "secrets-rotation-kms", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 5, "n": 9, "p": 0.5555555555555556, "lo": 0.2666512934954931, "hi": 0.8112214789023354, "tasks": 1, "macro": 0.5555555555555556, "macroLo": 0.2666512934954931, "macroHi": 0.8112214789023354, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.3333333333333333, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_337df7bf8f3649c4" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_36eca83d44a249e4-v1", "taskId": "task_36eca83d44a249e4", "version": "1", "name": "sfn-saga-compensation-orchestrator", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 4, "n": 8, "p": 0.5, "lo": 0.21521606221387757, "hi": 0.7847839377861224, "tasks": 1, "macro": 0.5, "macroLo": 0.21521606221387757, "macroHi": 0.7847839377861224, "labelled": 8, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 0, "BAD_FAILURE": 4, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_36eca83d44a249e4" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3b99e6a987b24657-v1", "taskId": "task_3b99e6a987b24657", "version": "1", "name": "efs-access-point-posix-iam-mount-target", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 3, "n": 9, "p": 0.3333333333333333, "lo": 0.1205838183869109, "hi": 0.6457978644196036, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.1205838183869109, "macroHi": 0.6457978644196036, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 1, "BAD_FAILURE": 5, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.1111111111111111, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_3b99e6a987b24657" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6fb722084d4e439d-v1", "taskId": "task_6fb722084d4e439d", "version": "1", "name": "iam-revoke-older-sessions", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 3, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.3, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_6fb722084d4e439d" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_713f5d90fdc940de-v1", "taskId": "task_713f5d90fdc940de", "version": "1", "name": "ddb-outbox-eventbridge-fanout", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 4, "n": 9, "p": 0.4444444444444444, "lo": 0.18877852109766458, "hi": 0.7333487065045069, "tasks": 1, "macro": 0.4444444444444444, "macroLo": 0.18877852109766458, "macroHi": 0.7333487065045069, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 4, "BAD_FAILURE": 1, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.4444444444444444, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_713f5d90fdc940de" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7757f12eee96413b-v1", "taskId": "task_7757f12eee96413b", "version": "1", "name": "athena-workgroup-result-encryption-cmk-enforced", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 1, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_7757f12eee96413b" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7f8d91906b9d41e2-v1", "taskId": "task_7f8d91906b9d41e2", "version": "1", "name": "glue-etl-catalog-security-configuration-kms", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 3, "n": 8, "p": 0.375, "lo": 0.13684428582359737, "hi": 0.6942576053973728, "tasks": 1, "macro": 0.375, "macroLo": 0.13684428582359737, "macroHi": 0.6942576053973728, "labelled": 8, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 4, "BAD_FAILURE": 1, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.5, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_7f8d91906b9d41e2" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_88f98c5b80a742fa-v1", "taskId": "task_88f98c5b80a742fa", "version": "1", "name": "s3-lambda-ddb-pipeline", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 2, "n": 6, "p": 0.3333333333333333, "lo": 0.09677141110578041, "hi": 0.700006684861608, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.09677141110578041, "macroHi": 0.700006684861608, "labelled": 6, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 2, "BAD_FAILURE": 2, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 4, "allValidRuns": 6, "measuredTasks": 1, "coverage": 1, "gfr": 0.3333333333333333, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_88f98c5b80a742fa" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_962354f38aaf4f00-v1", "taskId": "task_962354f38aaf4f00", "version": "1", "name": "cognito-m2m-httpapi-jwt-scope-gated", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 1, "n": 8, "p": 0.125, "lo": 0.02241749145005667, "hi": 0.4708881822128535, "tasks": 1, "macro": 0.125, "macroLo": 0.02241749145005667, "macroHi": 0.4708881822128535, "labelled": 8, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 6, "BAD_FAILURE": 1, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.75, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_962354f38aaf4f00" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_a5168465cc8c40b1-v1", "taskId": "task_a5168465cc8c40b1", "version": "1", "name": "iam-cross-account-externalid-sourcearn", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 3, "n": 9, "p": 0.3333333333333333, "lo": 0.1205838183869109, "hi": 0.6457978644196036, "tasks": 1, "macro": 0.3333333333333333, "macroLo": 0.1205838183869109, "macroHi": 0.6457978644196036, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 6, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_a5168465cc8c40b1" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_bf7726c705c94ea3-v1", "taskId": "task_bf7726c705c94ea3", "version": "1", "name": "apigw-sqs-fifo-direct-integration", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 1, "n": 8, "p": 0.125, "lo": 0.02241749145005667, "hi": 0.4708881822128535, "tasks": 1, "macro": 0.125, "macroLo": 0.02241749145005667, "macroHi": 0.4708881822128535, "labelled": 8, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.875, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_bf7726c705c94ea3" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c51675fbb49d4fc3-v1", "taskId": "task_c51675fbb49d4fc3", "version": "1", "name": "apigw-http-api-jwt-authorizer-lambda-integration", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 6, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_c51675fbb49d4fc3" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_caa085cfe6884900-v1", "taskId": "task_caa085cfe6884900", "version": "1", "name": "iam-permissions-boundary-ceiling", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 2, "n": 9, "p": 0.2222222222222222, "lo": 0.06322510711784673, "hi": 0.5474110308930111, "tasks": 1, "macro": 0.2222222222222222, "macroLo": 0.06322510711784673, "macroHi": 0.5474110308930111, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 6, "BAD_FAILURE": 1, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.6666666666666666, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_caa085cfe6884900" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d8f03df2afac4d77-v1", "taskId": "task_d8f03df2afac4d77", "version": "1", "name": "s3-sqs-image-pipeline-kms", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 2, "n": 9, "p": 0.2222222222222222, "lo": 0.06322510711784673, "hi": 0.5474110308930111, "tasks": 1, "macro": 0.2222222222222222, "macroLo": 0.06322510711784673, "macroHi": 0.5474110308930111, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 4, "BAD_FAILURE": 3, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.4444444444444444, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_d8f03df2afac4d77" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_db0863a3c7374743-v1", "taskId": "task_db0863a3c7374743", "version": "1", "name": "iam-session-tag-tenant-scope", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 2, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 1 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.5, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_db0863a3c7374743" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e71329638221445a-v1", "taskId": "task_e71329638221445a", "version": "1", "name": "ecs-fargate-secrets-kms-exec-role", "domain": "Cloud operations", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-7@@", "k": 6, "n": 7, "p": 0.8571428571428571, "lo": 0.4868721707256811, "hi": 0.9743203756552565, "tasks": 1, "macro": 0.8571428571428571, "macroLo": 0.4868721707256811, "macroHi": 0.9743203756552565, "labelled": 7, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 6, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 7, "measuredTasks": 1, "coverage": 1, "gfr": 0.14285714285714285, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_e71329638221445a" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_329fdf762b184453-v1", "taskId": "task_329fdf762b184453", "version": "1", "name": "ipl-toss-impact-analysis-r", "domain": "Data processing / data science", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 4, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_329fdf762b184453" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4a19a7bef25f463a-v1", "taskId": "task_4a19a7bef25f463a", "version": "1", "name": "product-recall-stock-price-event", "domain": "Data processing / data science", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_4a19a7bef25f463a" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4e268f9547224f39-v1", "taskId": "task_4e268f9547224f39", "version": "1", "name": "coffee-ratings-outliers", "domain": "Data processing / data science", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 1, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.5, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_4e268f9547224f39" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_bc67eaf652454e6e-v1", "taskId": "task_bc67eaf652454e6e", "version": "1", "name": "lending-club-lgd-bias-correction-r", "domain": "Data processing / data science", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 2, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_bc67eaf652454e6e" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d91a7199188d4476-v1", "taskId": "task_d91a7199188d4476", "version": "1", "name": "fedavg-federated-noniid-mnist", "domain": "Data processing / data science", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 1, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_d91a7199188d4476" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_1a9db0f83f7545c5-v1", "taskId": "task_1a9db0f83f7545c5", "version": "1", "name": "goja-temporal-duration", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 10, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_1a9db0f83f7545c5" }, "confirmed": true, "difficulty": 0, "difficultyLo": 1.1102230246251565e-16, "difficultyHi": 0.27753279986288926, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_4ff9abf6cd6c4244-v1", "taskId": "task_4ff9abf6cd6c4244", "version": "1", "name": "hono-range-requests", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_4ff9abf6cd6c4244" }, "confirmed": false, "difficulty": 1, "difficultyLo": 0.7224672001371109, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_630c6c4fd7f14dff-v1", "taskId": "task_630c6c4fd7f14dff", "version": "1", "name": "click-constraint-groups", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "medium", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 7, "n": 9, "p": 0.7777777777777778, "lo": 0.4525889691069889, "hi": 0.9367748928821533, "tasks": 1, "macro": 0.7777777777777778, "macroLo": 0.4525889691069889, "macroHi": 0.9367748928821533, "labelled": 9, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 0, "GOOD_SUCCESS": 7, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.2222222222222222, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_630c6c4fd7f14dff" }, "confirmed": false, "difficulty": 0.2222222222222222, "difficultyLo": 0.06322510711784668, "difficultyHi": 0.5474110308930111, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_88d9fbf2850f4a10-v1", "taskId": "task_88d9fbf2850f4a10", "version": "1", "name": "hono-http-cache-rfc9111", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_88d9fbf2850f4a10" }, "confirmed": false, "difficulty": 1, "difficultyLo": 0.7224672001371109, "difficultyHi": 1, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_9c7c6750cbe94197-v1", "taskId": "task_9c7c6750cbe94197", "version": "1", "name": "pypdf-revision-forensics", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 8, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_9c7c6750cbe94197" }, "confirmed": false, "difficulty": 0.8, "difficultyLo": 0.4901624715366417, "difficultyHi": 0.9433178485456248, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_a9697a1e257b4bc3-v1", "taskId": "task_a9697a1e257b4bc3", "version": "1", "name": "etree-xpath10-engine", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_a9697a1e257b4bc3" }, "confirmed": false, "difficulty": 0.7, "difficultyLo": 0.39677814746114537, "difficultyHi": 0.892208732593699, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_b1455b628b754b6d-v1", "taskId": "task_b1455b628b754b6d", "version": "1", "name": "jiff-rrule-recurrence", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "hard", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_b1455b628b754b6d" }, "confirmed": false, "difficulty": 0.7, "difficultyLo": 0.39677814746114537, "difficultyHi": 0.892208732593699, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_fde5222794724dc1-v1", "taskId": "task_fde5222794724dc1", "version": "1", "name": "python-docx-tracked-changes", "domain": "DeepSWE (long-horizon SWE)", "source": "site", "tier": "saturated", "calibrated": true, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-5@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 9, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.6-sol@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_fde5222794724dc1" }, "confirmed": false, "difficulty": 0.09999999999999998, "difficultyLo": 0.017876213095072924, "difficultyHi": 0.4041500267952385, "strongestRow": "claude-opus-5@@", "defectReasons": [] }, { "id": "task_05b9311d4ff340d6-v1", "taskId": "task_05b9311d4ff340d6", "version": "1", "name": "coprocessor-dispatcher-classmix", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_05b9311d4ff340d6" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_11d1e2d1f76046b3-v1", "taskId": "task_11d1e2d1f76046b3", "version": "1", "name": "byte-serial-round-scheduler", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_11d1e2d1f76046b3" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_380d91e4d2b546d1-v1", "taskId": "task_380d91e4d2b546d1", "version": "1", "name": "round-robin-arbiter-grant", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_380d91e4d2b546d1" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3d9485a1ffc34271-v1", "taskId": "task_3d9485a1ffc34271", "version": "1", "name": "Resynchronising serial byte receiver", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 2, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0.3333333333333333, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_3d9485a1ffc34271" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_48fce1aaaa3244f0-v1", "taskId": "task_48fce1aaaa3244f0", "version": "1", "name": "Wait-state register-file completer", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_48fce1aaaa3244f0" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4a0d096810114548-v1", "taskId": "task_4a0d096810114548", "version": "1", "name": "open-drain-command-engine", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 8, "n": 9, "p": 0.8888888888888888, "lo": 0.565000294423344, "hi": 0.980109112361455, "tasks": 1, "macro": 0.8888888888888888, "macroLo": 0.565000294423344, "macroHi": 0.980109112361455, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 7, "BAD_SUCCESS": 1 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.1111111111111111, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_4a0d096810114548" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_54682fdd72b949c2-v1", "taskId": "task_54682fdd72b949c2", "version": "1", "name": "dualmaster-membridge", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_54682fdd72b949c2" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_581c5265d77746dd-v1", "taskId": "task_581c5265d77746dd", "version": "1", "name": "Serial bit-destuff framer", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 1 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_581c5265d77746dd" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_71c61ad7fad04ea0-v1", "taskId": "task_71c61ad7fad04ea0", "version": "1", "name": "serial-break-resync", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_71c61ad7fad04ea0" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_79c6b935f4604828-v1", "taskId": "task_79c6b935f4604828", "version": "1", "name": "stream-frame-hold", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_79c6b935f4604828" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8774a275a744479e-v1", "taskId": "task_8774a275a744479e", "version": "1", "name": "Multi-cycle signed divider with a start/valid handshake", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 3, "p": 0, "lo": 0, "hi": 0.5614970317550455, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.5614970317550455, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 2, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0.6666666666666666, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_8774a275a744479e" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_949427b85b8245a8-v1", "taskId": "task_949427b85b8245a8", "version": "1", "name": "bcd-tens-rollover", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_949427b85b8245a8" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_98144ebbb02f4480-v1", "taskId": "task_98144ebbb02f4480", "version": "1", "name": "apb-lite-slave-regfile", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_98144ebbb02f4480" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_a6dbe2e590284d39-v1", "taskId": "task_a6dbe2e590284d39", "version": "1", "name": "bus-slave-abort-ack", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_a6dbe2e590284d39" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_aaa512da54ad4b7d-v1", "taskId": "task_aaa512da54ad4b7d", "version": "1", "name": "serial-receiver-framed", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_aaa512da54ad4b7d" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d012d48e446f4de3-v1", "taskId": "task_d012d48e446f4de3", "version": "1", "name": "Enable-gated streaming fold stage", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 2, "n": 2, "p": null, "lo": null, "hi": null, "tasks": 1, "macro": null, "macroLo": null, "macroHi": null, "labelled": 2, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 2, "measuredTasks": 1, "coverage": 1, "gfr": null, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_d012d48e446f4de3" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e7ea0056c1d44790-v1", "taskId": "task_e7ea0056c1d44790", "version": "1", "name": "hash-message-padder", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 9, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 9, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_e7ea0056c1d44790" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f7c0dfb137a14d48-v1", "taskId": "task_f7c0dfb137a14d48", "version": "1", "name": "Instruction-retire commit handshake", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 3, "n": 3, "p": 1, "lo": 0.43850296824495455, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.43850296824495455, "macroHi": 1, "labelled": 3, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 3 }, "harnessErrors": 0, "allValidRuns": 3, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_f7c0dfb137a14d48" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fe2f01c2236146c4-v1", "taskId": "task_fe2f01c2236146c4", "version": "1", "name": "debug-halt-step-fsm", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_fe2f01c2236146c4" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fed0a2a05a3f4895-v1", "taskId": "task_fed0a2a05a3f4895", "version": "1", "name": "page-program-suspend", "domain": "Electrical engineering (RTL)", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_fed0a2a05a3f4895" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_0a096af9acfe4df3-v1", "taskId": "task_0a096af9acfe4df3", "version": "1", "name": "camera-shake-rig", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-sonnet-4-6@@", "k": 3, "n": 6, "p": 0.5, "lo": 0.18761630648265054, "hi": 0.8123836935173494, "tasks": 1, "macro": 0.5, "macroLo": 0.18761630648265054, "macroHi": 0.8123836935173494, "labelled": 6, "classes": { "HARNESS_ERROR": 4, "GOOD_FAILURE": 1, "BAD_FAILURE": 2, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 1 }, "harnessErrors": 4, "allValidRuns": 6, "measuredTasks": 1, "coverage": 1, "gfr": 0.16666666666666666, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_0a096af9acfe4df3" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7a9a3979fb424dd5-v1", "taskId": "task_7a9a3979fb424dd5", "version": "1", "name": "crusader-sprite-assembly", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-sonnet-4-6@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 1, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 1 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.5, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_7a9a3979fb424dd5" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7cd86d3a16da4efb-v1", "taskId": "task_7cd86d3a16da4efb", "version": "1", "name": "minimap-ui", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 4, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_7cd86d3a16da4efb" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_96551755e32d4d6a-v1", "taskId": "task_96551755e32d4d6a", "version": "1", "name": "checkpoint-system", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 9, "p": 0.4444444444444444, "lo": 0.18877852109766458, "hi": 0.7333487065045069, "tasks": 1, "macro": 0.4444444444444444, "macroLo": 0.18877852109766458, "macroHi": 0.7333487065045069, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.5555555555555556, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_96551755e32d4d6a" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9bc91b79153c489d-v1", "taskId": "task_9bc91b79153c489d", "version": "1", "name": "tile-gradient-particles", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-sonnet-4-6@@", "k": 2, "n": 5, "p": 0.4, "lo": 0.11762077423264783, "hi": 0.769275718723987, "tasks": 1, "macro": 0.4, "macroLo": 0.11762077423264783, "macroHi": 0.769275718723987, "labelled": 5, "classes": { "HARNESS_ERROR": 5, "GOOD_FAILURE": 1, "BAD_FAILURE": 2, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 1 }, "harnessErrors": 5, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0.2, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_9bc91b79153c489d" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_9f64956269f1478a-v1", "taskId": "task_9f64956269f1478a", "version": "1", "name": "combo-score-system", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 6, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.3, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_9f64956269f1478a" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_cb27ed6721b34369-v1", "taskId": "task_cb27ed6721b34369", "version": "1", "name": "game-of-life-step", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 10, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 5, "n": 5, "p": 1, "lo": 0.5655175352168251, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5655175352168251, "macroHi": 1, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_cb27ed6721b34369" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e92ae7ab11414315-v1", "taskId": "task_e92ae7ab11414315", "version": "1", "name": "minimap-marker-logic", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 1, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.5, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_e92ae7ab11414315" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f0893645c8094454-v1", "taskId": "task_f0893645c8094454", "version": "1", "name": "day-night-cycle-controller", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 8, "BAD_FAILURE": 1, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_f0893645c8094454" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fb622a56a77246de-v1", "taskId": "task_fb622a56a77246de", "version": "1", "name": "car-scene-assembly", "domain": "Games", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 1, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 2 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.3, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_fb622a56a77246de" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_47a4494cf5424872-v1", "taskId": "task_47a4494cf5424872", "version": "1", "name": "reverse-engineer-decoding", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 1, "n": 10, "p": 0.1, "lo": 0.017876213095072868, "hi": 0.4041500267952385, "tasks": 1, "macro": 0.1, "macroLo": 0.017876213095072868, "macroHi": 0.4041500267952385, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 9, "BAD_FAILURE": 0, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.9, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_47a4494cf5424872" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_6b3d1e8537ee420e-v1", "taskId": "task_6b3d1e8537ee420e", "version": "1", "name": "fewlabel-ssl-fixmatch", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 8, "BAD_SUCCESS": 2 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_6b3d1e8537ee420e" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8775a27cf6614ca8-v1", "taskId": "task_8775a27cf6614ca8", "version": "1", "name": "adv-robust-pgd", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_8775a27cf6614ca8" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b5ded69410564877-v1", "taskId": "task_b5ded69410564877", "version": "1", "name": "airfoil-self-noise", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_b5ded69410564877" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b8d6f60868c74c6a-v1", "taskId": "task_b8d6f60868c74c6a", "version": "1", "name": "worst-group-spurious-dfr", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 5, "n": 9, "p": 0.5555555555555556, "lo": 0.2666512934954931, "hi": 0.8112214789023354, "tasks": 1, "macro": 0.5555555555555556, "macroLo": 0.2666512934954931, "macroHi": 0.8112214789023354, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 5, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.4444444444444444, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_b8d6f60868c74c6a" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d1332fc5fb48400d-v1", "taskId": "task_d1332fc5fb48400d", "version": "1", "name": "deepcad-canonical-equivalence", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 10, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_d1332fc5fb48400d" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ee9ca0af03a945ca-v1", "taskId": "task_ee9ca0af03a945ca", "version": "1", "name": "qat-int2-cifar", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-sonnet-4-6@@", "k": 0, "n": 5, "p": 0, "lo": 0, "hi": 0.43448246478317476, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.43448246478317476, "labelled": 5, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 5, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 5, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "claude-opus-4-8@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 8, "BAD_SUCCESS": 1 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_ee9ca0af03a945ca" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f22572227db04913-v1", "taskId": "task_f22572227db04913", "version": "1", "name": "reverse-engineer-objectives", "domain": "Machine learning", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_f22572227db04913" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2fa8d06250354a6b-v1", "taskId": "task_2fa8d06250354a6b", "version": "1", "name": "collision2d-impulse-solver", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 9, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.29914504841954404, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.29914504841954404, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 9, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_2fa8d06250354a6b" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_3a99729b47c44855-v1", "taskId": "task_3a99729b47c44855", "version": "1", "name": "pipeflow-colebrook-solver", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_3a99729b47c44855" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_485606c1eab54aff-v1", "taskId": "task_485606c1eab54aff", "version": "1", "name": "quaternion-rotation-integrator", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_485606c1eab54aff" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_b56ad3620c124d3d-v1", "taskId": "task_b56ad3620c124d3d", "version": "1", "name": "rk4-orbit-integrator", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 10, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_b56ad3620c124d3d" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ba8fd8e9c79d4fef-v1", "taskId": "task_ba8fd8e9c79d4fef", "version": "1", "name": "truss2d-solver", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 6, "n": 10, "p": 0.6, "lo": 0.31267376973365824, "hi": 0.8318196702937638, "tasks": 1, "macro": 0.6, "macroLo": 0.31267376973365824, "macroHi": 0.8318196702937638, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 4, "BAD_FAILURE": 0, "GOOD_SUCCESS": 6, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.4, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_ba8fd8e9c79d4fef" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d65d8bcc8a964600-v1", "taskId": "task_d65d8bcc8a964600", "version": "1", "name": "beam-deflection-solver", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_d65d8bcc8a964600" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e72016b2b32b4867-v1", "taskId": "task_e72016b2b32b4867", "version": "1", "name": "projectile-drag-integrator", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 9, "p": 0.2222222222222222, "lo": 0.06322510711784673, "hi": 0.5474110308930111, "tasks": 1, "macro": 0.2222222222222222, "macroLo": 0.06322510711784673, "macroHi": 0.5474110308930111, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.7777777777777778, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_e72016b2b32b4867" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_ef9037668fd04ec2-v1", "taskId": "task_ef9037668fd04ec2", "version": "1", "name": "heat1d-conduction-solver", "domain": "Mechanical engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 3, "n": 10, "p": 0.3, "lo": 0.10779126740630099, "hi": 0.6032218525388546, "tasks": 1, "macro": 0.3, "macroLo": 0.10779126740630099, "macroHi": 0.6032218525388546, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 0, "GOOD_SUCCESS": 3, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_ef9037668fd04ec2" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_13492b7eb77748ff-v1", "taskId": "task_13492b7eb77748ff", "version": "1", "name": "anova-stats", "domain": "Science / STEM", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 11, "n": 11, "p": 1, "lo": 0.7411670330319684, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.7411670330319684, "macroHi": 1, "labelled": 11, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 11, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 11, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_13492b7eb77748ff" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_49317edbfbd24c92-v1", "taskId": "task_49317edbfbd24c92", "version": "1", "name": "neonatal-drug-exposure-nlme", "domain": "Science / STEM", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 8, "p": 0.5, "lo": 0.21521606221387757, "hi": 0.7847839377861224, "tasks": 1, "macro": 0.5, "macroLo": 0.21521606221387757, "macroHi": 0.7847839377861224, "labelled": 8, "classes": { "HARNESS_ERROR": 2, "GOOD_FAILURE": 2, "BAD_FAILURE": 2, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 2, "allValidRuns": 8, "measuredTasks": 1, "coverage": 1, "gfr": 0.25, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_49317edbfbd24c92" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_7e1fa9bc50fe456f-v1", "taskId": "task_7e1fa9bc50fe456f", "version": "1", "name": "cubic-spline", "domain": "Science / STEM", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 10, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_7e1fa9bc50fe456f" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_89e4a252c8f14a98-v1", "taskId": "task_89e4a252c8f14a98", "version": "1", "name": "cg-solver", "domain": "Science / STEM", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 28, "n": 40, "p": 0.7, "lo": 0.5456998118185507, "hi": 0.8192515477025347, "tasks": 1, "macro": 0.7, "macroLo": 0.5456998118185507, "macroHi": 0.8192515477025347, "labelled": 40, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 5, "GOOD_SUCCESS": 27, "BAD_SUCCESS": 1 }, "harnessErrors": 0, "allValidRuns": 40, "measuredTasks": 1, "coverage": 1, "gfr": 0.175, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_89e4a252c8f14a98" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_d8500792374e4dfd-v1", "taskId": "task_d8500792374e4dfd", "version": "1", "name": "cholesky-solver", "domain": "Science / STEM", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 7, "BAD_FAILURE": 3, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.7, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_d8500792374e4dfd" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f652e9ed90de4703-v1", "taskId": "task_f652e9ed90de4703", "version": "1", "name": "adaptive-quadrature", "domain": "Science / STEM", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 10, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_f652e9ed90de4703" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_071f50470b4444a9-v1", "taskId": "task_071f50470b4444a9", "version": "1", "name": "simjeb-bracket-fea-mass-prediction-real", "domain": "Scientific computing", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 8, "BAD_FAILURE": 0, "GOOD_SUCCESS": 2, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.8, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_071f50470b4444a9" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_c9acf56ffbb24809-v1", "taskId": "task_c9acf56ffbb24809", "version": "1", "name": "ks-equation-1d-forecast", "domain": "Scientific computing", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_c9acf56ffbb24809" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f5fdfa4fc3934103-v1", "taskId": "task_f5fdfa4fc3934103", "version": "1", "name": "airfrans-high-reynolds-drag-extrapolation", "domain": "Scientific computing", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_f5fdfa4fc3934103" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_2627cfd299c14978-v1", "taskId": "task_2627cfd299c14978", "version": "1", "name": "diff-patch-engine", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 10, "p": 0, "lo": 0, "hi": 0.2775327998628892, "tasks": 1, "macro": 0, "macroLo": 0, "macroHi": 0.2775327998628892, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 10, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_2627cfd299c14978" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_27094c70c43a420c-v1", "taskId": "task_27094c70c43a420c", "version": "1", "name": "rate-limiter", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 2, "n": 10, "p": 0.2, "lo": 0.05668215145437519, "hi": 0.5098375284633583, "tasks": 1, "macro": 0.2, "macroLo": 0.05668215145437519, "macroHi": 0.5098375284633583, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 3, "BAD_FAILURE": 5, "GOOD_SUCCESS": 1, "BAD_SUCCESS": 1 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.3, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_27094c70c43a420c" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_4de3b9ae1ebc47ef-v1", "taskId": "task_4de3b9ae1ebc47ef", "version": "1", "name": "rust-lang-semver-305", "domain": "Software engineering", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_4de3b9ae1ebc47ef" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8505c19ed4e94507-v1", "taskId": "task_8505c19ed4e94507", "version": "1", "name": "session-token-verify", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 10, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_8505c19ed4e94507" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_8d4cba2dc1b84457-v1", "taskId": "task_8d4cba2dc1b84457", "version": "1", "name": "resilient-http-client", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 0, "n": 9, "p": 0, "lo": 2.7755575615628914e-17, "hi": 0.29914504841954404, "tasks": 1, "macro": 0, "macroLo": 2.7755575615628914e-17, "macroHi": 0.29914504841954404, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 8, "BAD_FAILURE": 1, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.8888888888888888, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_8d4cba2dc1b84457" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_bc2b4b2e92d4431e-v1", "taskId": "task_bc2b4b2e92d4431e", "version": "1", "name": "klauspost-compress-1115", "domain": "Software engineering", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_bc2b4b2e92d4431e" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_dc918c882354477d-v1", "taskId": "task_dc918c882354477d", "version": "1", "name": "evanw-esbuild-4417", "domain": "Software engineering", "source": "site", "tier": "uncalibrated", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "gpt-5.5@@", "k": 0, "n": 0, "p": null, "lo": null, "hi": null, "tasks": 0, "macro": null, "macroLo": null, "macroHi": null, "labelled": 0, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 0, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 0, "measuredTasks": 0, "coverage": 0, "gfr": null, "gfrCoverage": null, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_dc918c882354477d" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_e2cceeeb85cd4863-v1", "taskId": "task_e2cceeeb85cd4863", "version": "1", "name": "lru-cache", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-haiku-4-5@@", "k": 8, "n": 9, "p": 0.8888888888888888, "lo": 0.565000294423344, "hi": 0.980109112361455, "tasks": 1, "macro": 0.8888888888888888, "macroLo": 0.565000294423344, "macroHi": 0.980109112361455, "labelled": 9, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 8, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 9, "measuredTasks": 1, "coverage": 1, "gfr": 0.1111111111111111, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null }, { "rowId": "gpt-5.5@@", "k": 4, "n": 4, "p": 1, "lo": 0.5101091635454027, "hi": 1, "tasks": 1, "macro": 1, "macroLo": 0.5101091635454027, "macroHi": 1, "labelled": 4, "classes": { "HARNESS_ERROR": 1, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 1, "allValidRuns": 4, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": false, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_e2cceeeb85cd4863" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f658760651b04255-v1", "taskId": "task_f658760651b04255", "version": "1", "name": "occ-conditional-store", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 9, "n": 10, "p": 0.9, "lo": 0.5958499732047615, "hi": 0.9821237869049271, "tasks": 1, "macro": 0.9, "macroLo": 0.5958499732047615, "macroHi": 0.9821237869049271, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 1, "BAD_FAILURE": 0, "GOOD_SUCCESS": 9, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.1, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_f658760651b04255" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_f7066f8ab95e4dfb-v1", "taskId": "task_f7066f8ab95e4dfb", "version": "1", "name": "window-aggregate-store", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 10, "n": 10, "p": 1, "lo": 0.7224672001371107, "hi": 0.9999999999999999, "tasks": 1, "macro": 1, "macroLo": 0.7224672001371107, "macroHi": 0.9999999999999999, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 0, "BAD_FAILURE": 0, "GOOD_SUCCESS": 10, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_f7066f8ab95e4dfb" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] }, { "id": "task_fe77a704180847d4-v1", "taskId": "task_fe77a704180847d4", "version": "1", "name": "idempotency-middleware", "domain": "Software engineering", "source": "site", "tier": "legacy-only", "calibrated": false, "benchVersion": true, "curated": false, "selectionKnown": false, "quality": { "oraclePass": null, "nopPass": null, "qaVerdict": null, "knownDefects": [] }, "rows": [ { "rowId": "claude-opus-4-8@@", "k": 4, "n": 10, "p": 0.4, "lo": 0.16818032970623614, "hi": 0.6873262302663417, "tasks": 1, "macro": 0.4, "macroLo": 0.16818032970623614, "macroHi": 0.6873262302663417, "labelled": 10, "classes": { "HARNESS_ERROR": 0, "GOOD_FAILURE": 6, "BAD_FAILURE": 0, "GOOD_SUCCESS": 4, "BAD_SUCCESS": 0 }, "harnessErrors": 0, "allValidRuns": 10, "measuredTasks": 1, "coverage": 1, "gfr": 0.6, "gfrCoverage": 1, "fractional": 0, "calibrated": true, "intervalWarning": null } ], "links": { "site": "https://syncvals.syncrope.com/task/task_fe77a704180847d4" }, "confirmed": false, "difficulty": null, "difficultyLo": null, "difficultyHi": null, "strongestRow": null, "defectReasons": [] } ] }