{
  "meta": {
    "snapshotHash": "29b46d5ea53031efee082a7b9ff6f1c4535a37e21efce8924f9b05e6cdf2948f",
    "generatedAt": "2026-10-23T17:39:59+00:00",
    "sources": [
      {
        "name": "1. Delivery cohort export",
        "asOf": "2026-09-11"
      },
      {
        "name": "2. Website evaluation export",
        "asOf": "2026-08-23 (site build) / scraped 2026-09-11"
      },
      {
        "name": "3. Independent review inventory",
        "asOf": "2026-09-11"
      },
      {
        "name": "4. Independent review inventory",
        "asOf": "2026-09-12"
      },
      {
        "name": "5. Repository change inventory",
        "asOf": "2026-09-11"
      },
      {
        "name": "6. Delivery cohort export",
        "asOf": "2026-09-11"
      },
      {
        "name": "7. Repository evaluation export",
        "asOf": "2026-09-11"
      },
      {
        "name": "8. Repository evaluation export",
        "asOf": "2026-09-11"
      },
      {
        "name": "9. Repository evaluation export",
        "asOf": "2026-10-16"
      },
      {
        "name": "10. Console evaluation snapshot",
        "asOf": "2026-10-22"
      },
      {
        "name": "11. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "12. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "13. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "14. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "15. Repository evaluation export",
        "asOf": "2026-10-21"
      },
      {
        "name": "16. Independent review inventory",
        "asOf": "2026-10-21"
      },
      {
        "name": "17. Independent review inventory",
        "asOf": "2026-10-21"
      },
      {
        "name": "18. Delivery cohort export",
        "asOf": "2026-10-16"
      },
      {
        "name": "19. Independent review inventory",
        "asOf": "2026-10-14"
      },
      {
        "name": "20. Domain taxonomy inventory",
        "asOf": "2026-10-14"
      },
      {
        "name": "21. Independent review inventory",
        "asOf": "2026-10-20"
      },
      {
        "name": "22. Repository change inventory",
        "asOf": "2026-10-23"
      },
      {
        "name": "23. Delivery cohort export",
        "asOf": "2026-10-22"
      }
    ],
    "methodsVersion": "Syncvals analysis v1.2 · 2026-10-24",
    "bootstrap": {
      "replicates": 2000,
      "seed": 20261023,
      "method": "Paired task resampling, then binomial run resampling; percentile 95% interval"
    },
    "limitations": [
      "This is an observational catalog of recorded agent systems, not a balanced experiment or a representative sample of coding work.",
      "Current and legacy status comes from the frozen snapshot. No external release date or model identity was inferred.",
      "Aggregate rates use one latest nondefective version per task ID and at least three valid runs per task. Rows with fewer than three eligible tasks show counts only.",
      "Task names and display keys do not establish identity. Distinct task IDs remain distinct even if their names match.",
      "Confidence intervals describe sampling uncertainty under the stated assumptions. They do not correct selection bias, missing results, correlated task families, harness changes, or optional stopping.",
      "Two-stage empirical bootstrap intervals are approximate. Boundary cells cannot generate unobserved outcomes and may produce zero-width intervals; no certainty claim follows.",
      "The archive has no complete task-selection field. Delivery source marks a known curated proxy, while other tasks have unknown selection status.",
      "Frozen-classifier provenance is declared by the snapshot; classifier model IDs and prompt hashes are absent, so this build cannot independently attest them.",
      "Most historical effort modes, execution dates and CLI versions are missing. Unknown values remain unknown.",
      "One snapshot-valid negative reward is excluded as invalid_reward_range in analysis. The historical inventory retains original snapshot validity and trial-domain assignments for reproducibility.",
      "Classifiers explain verifier grades. HARNESS_ERROR is excluded under the inherited policy, including classifier inconsistencies; this policy can introduce outcome-dependent missingness.",
      "Duplicate trial IDs use console > site > delivery > repo precedence. Shared timestamps do not identify duplicate executions; timestamp collisions are retained.",
      "Pooled-run Wilson intervals are descriptive and assume independent Bernoulli trials. Headline comparisons use task macro averages and paired task resampling.",
      "Sign-test values are conditional sign diagnostics, not validated tests of equal model capability; unequal repetition counts can violate the fair-sign null. No superiority inference is issued.",
      "Reviews are already represented in the normalized trial labels. The site hashes the review inventory but does not adjudicate or independently establish reviewer independence.",
      "Legacy showcase trial evidence URLs have been retired because those routes return 404. Missing public evidence is shown explicitly, not replaced with a proof claim."
    ]
  },
  "figures": {
    "pairwise": [
      {
        "a": "claude-opus-4-8@@",
        "b": "gpt-5.5@@",
        "sharedTasks": 332,
        "nA": 1836,
        "nB": 1843,
        "delta": 0.0627510040160642,
        "lo": 0.001840696117804556,
        "hi": 0.11825635876840701,
        "wins": 177,
        "losses": 130,
        "ties": 25,
        "pValue": 0.008548772098080096,
        "pAdjusted": 0.01709754419616019,
        "status": "descriptive comparison; no superiority inference",
        "crossEra": false,
        "taskIds": [
          "review-0035c2516745aa41-v1",
          "review-0186fa7cb35661fb-v1",
          "review-026dfb9d2b48dec9-v1",
          "review-02c74cb81f861416-v1",
          "review-04d3d031ce54bfc2-v1",
          "review-0588ba287be732ec-v1",
          "review-05be8df0a0c01932-v1",
          "review-06a9056cf93dd13b-v1",
          "review-06d1f0750367d8fd-v1",
          "review-070ead12e193f6ca-v1",
          "review-07aeefc77ef55bcf-v1",
          "review-087493fb0b94df25-v1",
          "review-0c74fb52-v1",
          "review-0c7726a53cc27b17-v1",
          "review-0d03a77f6269e1e6-v1",
          "review-0e92905c4482e13a-v1",
          "review-0f77bf242f2ac69e-v1",
          "review-10daef0770206047-v1",
          "review-10dcfa97346ee443-v1",
          "review-10e3faa4cbb66d60-v1",
          "review-1108e91a04568719-v1",
          "review-11c6f402ab5f36a9-v1",
          "review-13ed8da0fe78942b-v1",
          "review-14fc424d042a7f44-v1",
          "review-15dc6c3847427416-v1",
          "review-1760a912a21c4965-v1",
          "review-18050a0f9d2f3a5a-v1",
          "review-186c78b61bef7d57-v1",
          "review-18abb64a7dc0675c-v1",
          "review-1c71625c68049bba-v1",
          "review-1d58c0cd74185094-v1",
          "review-1da75e480c20ded0-v1",
          "review-203f3c27ffc3b145-v1",
          "review-205fc9589670c791-v1",
          "review-207a20238cebd242-v1",
          "review-2116b096d6c849e3-v1",
          "review-21fe28c4186b8e9b-v1",
          "review-23ae94fd5fbd496f-v1",
          "review-2400093057d93f12-v1",
          "review-24f19b8f8d19a6c9-v1",
          "review-25e6caab2ed6b707-v1",
          "review-266f0a625eeee32b-v1",
          "review-27aa266b74303b06-v1",
          "review-2854e1df2e64a088-v1",
          "review-286ce4d3586d9214-v1",
          "review-28ac19f6413c6146-v1",
          "review-291baee5cfc30feb-v1",
          "review-29ce6920-v1",
          "review-2ae2f31b-v1",
          "review-2b8c21c9-v1",
          "review-2bea92e628bdc3fa-v1",
          "review-2c44ece630a3107d-v1",
          "review-2c66d77d2df99605-v1",
          "review-2cde66b5007c9853-v1",
          "review-2d1b55ffe31e8107-v1",
          "review-2d28e154db7fd568-v1",
          "review-2f214a8890a339ec-v1",
          "review-2f7b68a7fd0911fe-v1",
          "review-31a2e48327930691-v1",
          "review-33162d3c9e2c56c2-v1",
          "review-35400c11fe8d83b5-v1",
          "review-3584a91e33f847e2-v1",
          "review-3642e903d34bbcd4-v1",
          "review-36c36db89046cb22-v1",
          "review-3703682fe32fb6c9-v1",
          "review-371f0cf9ffde5c79-v1",
          "review-373909373d46cce1-v1",
          "review-38299731ba828d36-v1",
          "review-38602a4d87dd01ee-v1",
          "review-38b4f33efbafa79f-v1",
          "review-3acab3984f345e41-v1",
          "review-3bdd2a68295b770e-v1",
          "review-3c4025ecea9fef68-v1",
          "review-3c6b4144aa1bb171-v1",
          "review-3dce264b3190e928-v1",
          "review-3e0c4ece7f2bc1de-v1",
          "review-3fdc31cfc07ae700-v1",
          "review-401b796f445022e0-v1",
          "review-41080ae7e090c2e0-v1",
          "review-4117ff845d0f6436-v1",
          "review-41e27bff6fae9901-v1",
          "review-44e7db992f16c6df-v1",
          "review-4553b41e-v1",
          "review-460929c386107279-v1",
          "review-46585b2534e3153a-v1",
          "review-46777564f4cb9119-v1",
          "review-47492406-v1",
          "review-474e08262d899fc1-v1",
          "review-478e23e8fb354efa-v1",
          "review-481a52d8a80061ea-v1",
          "review-4b023432-v1",
          "review-4b19088f6b926282-v1",
          "review-4be40120934f9b09-v1",
          "review-4c4689055c972086-v1",
          "review-4cd8ddb545042876-v1",
          "review-4e3ad65b5dc08010-v1",
          "review-4f6152eb-v1",
          "review-50844bdc0b32f0b9-v1",
          "review-50880b1aeec0b1df-v1",
          "review-51661814318e4c9e-v1",
          "review-518f814461b5229f-v1",
          "review-51b8441e76d0691c-v1",
          "review-52fa9a8b1bac6661-v1",
          "review-533a90519e39e5e6-v1",
          "review-535c56e3ee975cee-v1",
          "review-5391cd51fc1d4030-v1",
          "review-53ec9604222ed36f-v1",
          "review-5483f2641d761199-v1",
          "review-54a9f3c205c8e092-v1",
          "review-54f4819d9bd34180-v1",
          "review-5500027766d80265-v1",
          "review-550d6a532a01b966-v1",
          "review-55a233f45e278dac-v1",
          "review-57881d7d5dae2c1b-v1",
          "review-58059ffc52d5e055-v1",
          "review-58133a92c9deeb38-v1",
          "review-59e1625dbd9c6ebe-v1",
          "review-59f8af9563cbe5ed-v1",
          "review-5b392f7d9604d37c-v1",
          "review-5b3a4033ef333e51-v1",
          "review-5c400b7834d5e9d6-v1",
          "review-5cb4b7332832b796-v1",
          "review-5ce6defca5bf5c93-v1",
          "review-5d5675ab0bfd64f3-v1",
          "review-5d65bdca-v1",
          "review-5f684c5d8b8bfdc6-v1",
          "review-61a7da17711a8011-v1",
          "review-644053fc049e1c96-v1",
          "review-6725958240f8f986-v1",
          "review-676705ed3a8fe260-v1",
          "review-690137fd9ef8cd31-v1",
          "review-6b4fd5606d899402-v1",
          "review-6b94f1c3c9373e1a-v1",
          "review-6c7a149682ddda41-v1",
          "review-6cde584bf6f2d6f9-v1",
          "review-6e68a5da65b3fa4b-v1",
          "review-6e6ebde46db706df-v1",
          "review-6e75d22d0a2aebc7-v1",
          "review-6e830bd5eead1e91-v1",
          "review-6ece93d97eee5d96-v1",
          "review-6f47f180a8fd4298-v1",
          "review-6f79bb218c1e97a6-v1",
          "review-709c83cb10d627bf-v1",
          "review-7236c8abe5b40afa-v1",
          "review-729db547f14398f8-v1",
          "review-7380e53a30145ec3-v1",
          "review-739ddcc29e07c2c7-v1",
          "review-7402e41286eb7cfc-v1",
          "review-74afffea97cbf88e-v1",
          "review-75196631c5dc0f1f-v1",
          "review-75627ad59aca1d13-v1",
          "review-75f58780438ea226-v1",
          "review-7688a880a0c820a2-v1",
          "review-79ddfafabc458dcf-v1",
          "review-79fa7c6c78cb95b0-v1",
          "review-79fdb8aee3e9b29f-v1",
          "review-7a1fee3b2f235c4b-v1",
          "review-7b3dbe69-v1",
          "review-7bbf32840ac8e20d-v1",
          "review-7cf6ad942044739a-v1",
          "review-7d574292d6fe9c35-v1",
          "review-7e462e5b536d2c9f-v1",
          "review-7f49e1ea5401ddaa-v1",
          "review-7f6ced1303079ede-v1",
          "review-80099dc2c0df9adc-v1",
          "review-80470c9d711756ae-v1",
          "review-8114696746ad4a99-v1",
          "review-8121e26ecf94ace5-v1",
          "review-81b445c5060dd5ee-v1",
          "review-81bf888c2712fd13-v1",
          "review-81c121087133efe2-v1",
          "review-82caf9657274b2b5-v1",
          "review-82cbb1ef69bf87a9-v1",
          "review-82d8246f6227712a-v1",
          "review-8367d115cc36c5ea-v1",
          "review-85d7890fcb351066-v1",
          "review-869b13cd00c5aa50-v1",
          "review-89c107a748bafe50-v1",
          "review-8b1b942a321c83d6-v1",
          "review-8c9ee754759701bd-v1",
          "review-8d6a0ae6-v1",
          "review-8ec6a0622cea136d-v1",
          "review-8f8e94e3de72ad68-v1",
          "review-90e933e0b6f09d7e-v1",
          "review-90ff4be3adb37352-v1",
          "review-9102053c503d260b-v1",
          "review-9206f9f198f440ed-v1",
          "review-920b0a812f7bce15-v1",
          "review-943cb1a5777d050f-v1",
          "review-94950cd7673b2085-v1",
          "review-951cc4e805caad96-v1",
          "review-95f00f0f5839ad85-v1",
          "review-96f21e2c-v1",
          "review-97b9cfdd7ce5bc62-v1",
          "review-98f43e43-v1",
          "review-993af7c619021f4c-v1",
          "review-99883b9d39b9d601-v1",
          "review-99a7b4d2238c5c9c-v1",
          "review-99eeb6cd02bc520e-v1",
          "review-9ba5a91d7ce33e0b-v1",
          "review-9bcf6a7ac7f589ce-v1",
          "review-9e0d533702efde46-v1",
          "review-a06271859ba7cab0-v1",
          "review-a14988792d950989-v1",
          "review-a1a2cdaaf525c38a-v1",
          "review-a1ec4dce1e698bca-v1",
          "review-a30f28f10efbf454-v1",
          "review-a51176485533a8ef-v1",
          "review-a553850bc0203a3c-v1",
          "review-a619d54b24f65a96-v1",
          "review-a6900450a4780650-v1",
          "review-a87234c5f684cf5f-v1",
          "review-a90d30d81b354411-v1",
          "review-a9e671f0fa608f9d-v1",
          "review-aab47bdfffc29d7d-v1",
          "review-aaef3e3a549f1422-v1",
          "review-abdddf065881e360-v1",
          "review-ac9a8f9f4be4ff7a-v1",
          "review-ae1476cb2639e72d-v1",
          "review-aea4fa357faa5976-v1",
          "review-af174d05c47c42f7-v1",
          "review-af83460793a10cc0-v1",
          "review-af9804f996324a19-v1",
          "review-afa3c599889fb01f-v1",
          "review-b0e3721bf746b9a6-v1",
          "review-b2267b7f5a55574a-v1",
          "review-b22c2eb54617ecfd-v1",
          "review-b27bdeb938e070a2-v1",
          "review-b3cec9992e9002ee-v1",
          "review-b40a75f5744fa3e4-v1",
          "review-b43f6254f82ac360-v1",
          "review-b5b89f73e450c580-v1",
          "review-b5e00c7ca2cff589-v1",
          "review-b6c5a3df723f3f36-v1",
          "review-b7c3795f60b85a6e-v1",
          "review-b81f3182545d7f8d-v1",
          "review-ba411a1299dde36a-v1",
          "review-bb804f67488709e6-v1",
          "review-bb9534d12fa0b5b4-v1",
          "review-bd836ce62bc30795-v1",
          "review-be41256a22975e12-v1",
          "review-bf1613c50e1e8a93-v1",
          "review-c060e012bd7e56d0-v1",
          "review-c07aced72086b242-v1",
          "review-c1803ab760f77a73-v1",
          "review-c24b7bdb484f393c-v1",
          "review-c255716e0a2dffe0-v1",
          "review-c25e85da-v1",
          "review-c2826cf6941209c3-v1",
          "review-c2d11d69a920097c-v1",
          "review-c377ee4e485f3035-v1",
          "review-c4adec863d17bd83-v1",
          "review-c4e36e1a39328b90-v1",
          "review-c5526eb2fe38c178-v1",
          "review-c69418079ff48fbf-v1",
          "review-c69b7fc2c3d92ce4-v1",
          "review-c7329f55bf372cce-v1",
          "review-c79614abaf886585-v1",
          "review-c8c4102c45a8d027-v1",
          "review-cab0cfc98e25a800-v1",
          "review-cb60ba39c33fd0da-v1",
          "review-cbc316323a1068d5-v1",
          "review-cc3a68b1b2a552d3-v1",
          "review-cdc910d345082988-v1",
          "review-cddb4a1540ded144-v1",
          "review-cdeee269f5c33766-v1",
          "review-d0592dae19f52448-v1",
          "review-d0881ed0c975cb52-v1",
          "review-d222c47197a601e7-v1",
          "review-d257aebc7410222f-v1",
          "review-d35c83843f557c76-v1",
          "review-d404a4dab923f55e-v1",
          "review-d4878d8f40f68160-v1",
          "review-d4eda92afd081963-v1",
          "review-d4f2c5fe0aaacd40-v1",
          "review-d58f8f66098c2ccc-v1",
          "review-d70682073ae41807-v1",
          "review-d8318208dea23ff4-v1",
          "review-d8437d87ff1eb546-v1",
          "review-d8473951f87183f2-v1",
          "review-d89645717e317e8e-v1",
          "review-d9100094-v1",
          "review-d9423bb69741de07-v1",
          "review-db431123502e492e-v1",
          "review-dd175ae76bbe080f-v1",
          "review-de1063f9ec38f973-v1",
          "review-debc3a9fb4c6e890-v1",
          "review-dfb877f9bd267d59-v1",
          "review-dfc1160e8ab6f23b-v1",
          "review-e1178bfc3fb7303f-v1",
          "review-e1a883e3a902805a-v1",
          "review-e232cb1227e1324c-v1",
          "review-e27e99f3c9d8884a-v1",
          "review-e27eb5788e04c983-v1",
          "review-e2f972171df85b01-v1",
          "review-e4ba86843400b7d4-v1",
          "review-e63c197eb05a49dd-v1",
          "review-e8166b6ee4357b77-v1",
          "review-e8bf2a5c599901e1-v1",
          "review-e8cd35c501d91d90-v1",
          "review-e90bbced5e3e5edb-v1",
          "review-ea6fde5a6a26ce26-v1",
          "review-eaad17303f504102-v1",
          "review-ead5bb42d26f0110-v1",
          "review-ec253d16b45771f7-v1",
          "review-ecfbd47c93392f7e-v1",
          "review-f0b0f7602ec3b29e-v1",
          "review-f117a90fc4d2a39b-v1",
          "review-f210ced397d4dc6e-v1",
          "review-f21987f8a922a460-v1",
          "review-f2cf0c232898ce52-v1",
          "review-f32ec8af0f6f3bc4-v1",
          "review-f3dc20fb6f7df522-v1",
          "review-f3dc35bcbbde52e4-v1",
          "review-f4c7f05235f40e77-v1",
          "review-f5218e5615888800-v1",
          "review-f595bc80e313bd6c-v1",
          "review-f6dd59def7c666de-v1",
          "review-f8b01b4eec225cd1-v1",
          "review-fc072cd1bf41b028-v1",
          "review-fcbc9a030e857334-v1",
          "review-fccb0079ec7f2459-v1",
          "review-fdd7deb090c3286f-v1",
          "review-fe716c693a0458ce-v1",
          "review-fed730d1ae9dc266-v1",
          "review-slug-chainward-x509-validation-v1",
          "review-slug-deb-build-profile-resolve-v1",
          "review-slug-dispatch-registry-resume-skew-v1",
          "review-slug-firmware-release-security-adjudication-v1",
          "review-slug-sanctuary-ledger-trust-certification-v1",
          "review-slug-tokentrust-cpp-reconcile-task-v1",
          "review-slug-vaccine-cold-chain-stochastic-plan-v1"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      },
      {
        "a": "claude-opus-5@@",
        "b": "gpt-5.6-sol@@",
        "sharedTasks": 19,
        "nA": 123,
        "nB": 125,
        "delta": 0.0935672514619883,
        "lo": -0.07894736842105264,
        "hi": 0.28304093567251454,
        "wins": 6,
        "losses": 3,
        "ties": 10,
        "pValue": 0.5078125,
        "pAdjusted": 0.5078125,
        "status": "descriptive comparison; no superiority inference",
        "crossEra": false,
        "taskIds": [
          "task_03f5f9f4f811412b-v1",
          "task_14e103e2e9454d6b-v1",
          "task_1a9db0f83f7545c5-v1",
          "task_23152389f9124dcb-v1",
          "task_40df7b0ce1654f8f-v1",
          "task_42603212f8434552-v1",
          "task_4ff9abf6cd6c4244-v1",
          "task_5c156a1be6db47fc-v1",
          "task_630c6c4fd7f14dff-v1",
          "task_6df2c2ca80cb42d4-v1",
          "task_752da4e1649540f2-v2",
          "task_79559260b7e3427d-v1",
          "task_88d9fbf2850f4a10-v1",
          "task_a9697a1e257b4bc3-v1",
          "task_bb2e200d1de14882-v2",
          "task_be13c5ae46354493-v1",
          "task_f23a5f1298c94cb9-v2",
          "task_f690cc4d68e94d74-v2",
          "task_fde5222794724dc1-v1"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      },
      {
        "a": "claude-opus-4-8@@",
        "b": "gemini-flash-latest@@",
        "sharedTasks": 3,
        "nA": 15,
        "nB": 13,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 3,
        "losses": 0,
        "ties": 0,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": false,
        "taskIds": [
          "task_14e103e2e9454d6b-v1",
          "task_5c156a1be6db47fc-v1",
          "task_be13c5ae46354493-v1"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      },
      {
        "a": "claude-opus-5@@high",
        "b": "tencent/hy4-preview@@high",
        "sharedTasks": 2,
        "nA": 10,
        "nB": 16,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 2,
        "losses": 0,
        "ties": 0,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": false,
        "taskIds": [
          "task_6be3a5abe6ca439e-v1",
          "task_e4e29638eb354f33-v1"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      },
      {
        "a": "gpt-5.5@@",
        "b": "claude-haiku-4-5@@",
        "sharedTasks": 2,
        "nA": 9,
        "nB": 19,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 1,
        "losses": 0,
        "ties": 1,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": false,
        "taskIds": [
          "task_cb27ed6721b34369-v1",
          "task_e2cceeeb85cd4863-v1"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      },
      {
        "a": "claude-opus-5@@high",
        "b": "tencent/hy4-preview@@think",
        "sharedTasks": 1,
        "nA": 5,
        "nB": 8,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 0,
        "losses": 0,
        "ties": 1,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": false,
        "taskIds": [
          "task_17499976e3a34d09-v2"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      },
      {
        "a": "tencent/hy4-preview@@think",
        "b": "claude-opus-5@@xhigh",
        "sharedTasks": 1,
        "nA": 4,
        "nB": 4,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 0,
        "losses": 1,
        "ties": 0,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": false,
        "taskIds": [
          "task_f48ea8010149425d-v1"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      },
      {
        "a": "claude-opus-4-8@@",
        "b": "claude-sonnet-4-6@@",
        "sharedTasks": 1,
        "nA": 10,
        "nB": 5,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 1,
        "losses": 0,
        "ties": 0,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": false,
        "taskIds": [
          "task_ee9ca0af03a945ca-v1"
        ],
        "warning": "Paired on identical recorded task versions. Unequal repetition counts and retrospective selection mean the sign diagnostic is not a validated test of equal model capability."
      }
    ],
    "legacyOverlap": [
      {
        "a": "claude-opus-5@@",
        "b": "claude-opus-4-8@@",
        "sharedTasks": 4,
        "nA": 20,
        "nB": 20,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 2,
        "losses": 1,
        "ties": 1,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": true,
        "taskIds": [
          "task_14e103e2e9454d6b-v1",
          "task_40df7b0ce1654f8f-v1",
          "task_5c156a1be6db47fc-v1",
          "task_be13c5ae46354493-v1"
        ],
        "warning": "Cross-era comparison: recorded agent systems differ in harnesses, dates and effort modes. No model-generation effect is identified."
      },
      {
        "a": "gpt-5.6-sol@@",
        "b": "claude-opus-4-8@@",
        "sharedTasks": 4,
        "nA": 20,
        "nB": 20,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 2,
        "losses": 0,
        "ties": 2,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": true,
        "taskIds": [
          "task_14e103e2e9454d6b-v1",
          "task_40df7b0ce1654f8f-v1",
          "task_5c156a1be6db47fc-v1",
          "task_be13c5ae46354493-v1"
        ],
        "warning": "Cross-era comparison: recorded agent systems differ in harnesses, dates and effort modes. No model-generation effect is identified."
      },
      {
        "a": "claude-opus-5@@",
        "b": "gemini-flash-latest@@",
        "sharedTasks": 3,
        "nA": 15,
        "nB": 13,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 3,
        "losses": 0,
        "ties": 0,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": true,
        "taskIds": [
          "task_14e103e2e9454d6b-v1",
          "task_5c156a1be6db47fc-v1",
          "task_be13c5ae46354493-v1"
        ],
        "warning": "Cross-era comparison: recorded agent systems differ in harnesses, dates and effort modes. No model-generation effect is identified."
      },
      {
        "a": "gpt-5.6-sol@@",
        "b": "gemini-flash-latest@@",
        "sharedTasks": 3,
        "nA": 15,
        "nB": 13,
        "delta": null,
        "lo": null,
        "hi": null,
        "wins": 3,
        "losses": 0,
        "ties": 0,
        "pValue": null,
        "pAdjusted": null,
        "status": "insufficient overlap",
        "crossEra": true,
        "taskIds": [
          "task_14e103e2e9454d6b-v1",
          "task_5c156a1be6db47fc-v1",
          "task_be13c5ae46354493-v1"
        ],
        "warning": "Cross-era comparison: recorded agent systems differ in harnesses, dates and effort modes. No model-generation effect is identified."
      }
    ],
    "validity": [
      {
        "reason": "no_reward",
        "runs": 1126
      },
      {
        "reason": "harness_error",
        "runs": 380
      },
      {
        "reason": "source_duplicate",
        "runs": 148
      },
      {
        "reason": "source_evidence_exclusion",
        "runs": 114
      },
      {
        "reason": "zero_output_tokens",
        "runs": 8
      },
      {
        "reason": "source_task_defect_exclusion",
        "runs": 8
      },
      {
        "reason": "invalid_reward_range",
        "runs": 1
      }
    ],
    "coverage": [
      {
        "domain": "Cloud operations",
        "tasks": 22,
        "calibrated": 0,
        "currentMeasured": 0,
        "legacyMeasured": 19
      },
      {
        "domain": "Data processing / data science",
        "tasks": 253,
        "calibrated": 11,
        "currentMeasured": 29,
        "legacyMeasured": 60
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "tasks": 70,
        "calibrated": 32,
        "currentMeasured": 36,
        "legacyMeasured": 15
      },
      {
        "domain": "Electrical engineering (RTL)",
        "tasks": 83,
        "calibrated": 1,
        "currentMeasured": 4,
        "legacyMeasured": 19
      },
      {
        "domain": "Games",
        "tasks": 83,
        "calibrated": 10,
        "currentMeasured": 10,
        "legacyMeasured": 40
      },
      {
        "domain": "Machine learning",
        "tasks": 324,
        "calibrated": 18,
        "currentMeasured": 36,
        "legacyMeasured": 48
      },
      {
        "domain": "Mechanical engineering",
        "tasks": 30,
        "calibrated": 0,
        "currentMeasured": 2,
        "legacyMeasured": 12
      },
      {
        "domain": "Media",
        "tasks": 1,
        "calibrated": 0,
        "currentMeasured": 1,
        "legacyMeasured": 0
      },
      {
        "domain": "Operations",
        "tasks": 5,
        "calibrated": 0,
        "currentMeasured": 1,
        "legacyMeasured": 0
      },
      {
        "domain": "Other / uncategorised",
        "tasks": 16,
        "calibrated": 0,
        "currentMeasured": 0,
        "legacyMeasured": 0
      },
      {
        "domain": "SWE-Marathon",
        "tasks": 4,
        "calibrated": 2,
        "currentMeasured": 2,
        "legacyMeasured": 0
      },
      {
        "domain": "Science / STEM",
        "tasks": 58,
        "calibrated": 1,
        "currentMeasured": 1,
        "legacyMeasured": 8
      },
      {
        "domain": "Scientific computing",
        "tasks": 224,
        "calibrated": 0,
        "currentMeasured": 2,
        "legacyMeasured": 82
      },
      {
        "domain": "Security / cyber",
        "tasks": 171,
        "calibrated": 11,
        "currentMeasured": 15,
        "legacyMeasured": 80
      },
      {
        "domain": "Snorkel-reviewed tasks (category unknown)",
        "tasks": 141,
        "calibrated": 0,
        "currentMeasured": 0,
        "legacyMeasured": 8
      },
      {
        "domain": "Software engineering",
        "tasks": 561,
        "calibrated": 8,
        "currentMeasured": 16,
        "legacyMeasured": 83
      }
    ],
    "historical": [
      {
        "domain": "Software engineering",
        "rowId": "gpt-5.5@@",
        "k": 288,
        "n": 623,
        "p": 0.4622792937399679,
        "tasks": 112
      },
      {
        "domain": "Electrical engineering (RTL)",
        "rowId": "claude-opus-4-8@@",
        "k": 62,
        "n": 130,
        "p": 0.47692307692307695,
        "tasks": 13
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "claude-opus-4-8@@",
        "k": 19,
        "n": 100,
        "p": 0.19,
        "tasks": 20
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "claude-haiku-4-5@@",
        "k": 0,
        "n": 2,
        "p": 0,
        "tasks": 1
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "claude-opus-5@@",
        "k": 74,
        "n": 223,
        "p": 0.33183856502242154,
        "tasks": 36
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "gemini-flash-latest@@",
        "k": 5,
        "n": 27,
        "p": 0.18518518518518517,
        "tasks": 8
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "gpt-5.6-sol@@",
        "k": 59,
        "n": 183,
        "p": 0.3224043715846995,
        "tasks": 27
      },
      {
        "domain": "Software engineering",
        "rowId": "claude-opus-5@@",
        "k": 51,
        "n": 89,
        "p": 0.5730337078651685,
        "tasks": 18
      },
      {
        "domain": "Security / cyber",
        "rowId": "claude-opus-5@@",
        "k": 30,
        "n": 66,
        "p": 0.45454545454545453,
        "tasks": 13
      },
      {
        "domain": "Machine learning",
        "rowId": "claude-opus-5@@",
        "k": 95,
        "n": 163,
        "p": 0.5828220858895705,
        "tasks": 37
      },
      {
        "domain": "Science / STEM",
        "rowId": "claude-opus-5@@",
        "k": 12,
        "n": 18,
        "p": 0.6666666666666666,
        "tasks": 3
      },
      {
        "domain": "Scientific computing",
        "rowId": "claude-opus-5@@",
        "k": 11,
        "n": 11,
        "p": 1,
        "tasks": 3
      },
      {
        "domain": "SWE-Marathon",
        "rowId": "claude-opus-5@@",
        "k": 6,
        "n": 16,
        "p": 0.375,
        "tasks": 4
      },
      {
        "domain": "Games",
        "rowId": "claude-opus-5@@",
        "k": 50,
        "n": 52,
        "p": 0.9615384615384616,
        "tasks": 11
      },
      {
        "domain": "Data processing / data science",
        "rowId": "claude-opus-5@@",
        "k": 7,
        "n": 15,
        "p": 0.4666666666666667,
        "tasks": 2
      },
      {
        "domain": "Electrical engineering (RTL)",
        "rowId": "claude-opus-5@@",
        "k": 11,
        "n": 11,
        "p": 1,
        "tasks": 3
      },
      {
        "domain": "Media",
        "rowId": "claude-opus-5@@",
        "k": 1,
        "n": 3,
        "p": 0.3333333333333333,
        "tasks": 2
      },
      {
        "domain": "Cloud operations",
        "rowId": "claude-opus-5@@",
        "k": 2,
        "n": 4,
        "p": 0.5,
        "tasks": 1
      },
      {
        "domain": "Data processing / data science",
        "rowId": "openai/gpt-6-astra@@",
        "k": 14,
        "n": 49,
        "p": 0.2857142857142857,
        "tasks": 12
      },
      {
        "domain": "Security / cyber",
        "rowId": "tencent/hy4-preview@@think",
        "k": 7,
        "n": 22,
        "p": 0.3181818181818182,
        "tasks": 4
      },
      {
        "domain": "Mechanical engineering",
        "rowId": "claude-opus-5@@xhigh",
        "k": 5,
        "n": 5,
        "p": 1,
        "tasks": 2
      },
      {
        "domain": "Mechanical engineering",
        "rowId": "tencent/hy4-preview@@think",
        "k": 0,
        "n": 5,
        "p": 0,
        "tasks": 2
      },
      {
        "domain": "Electrical engineering (RTL)",
        "rowId": "claude-opus-5@@xhigh",
        "k": 2,
        "n": 2,
        "p": 1,
        "tasks": 1
      },
      {
        "domain": "Electrical engineering (RTL)",
        "rowId": "tencent/hy4-preview@@think",
        "k": 2,
        "n": 2,
        "p": 1,
        "tasks": 1
      },
      {
        "domain": "Operations",
        "rowId": "claude-opus-5@@xhigh",
        "k": 0,
        "n": 3,
        "p": 0,
        "tasks": 2
      },
      {
        "domain": "Operations",
        "rowId": "tencent/hy4-preview@@think",
        "k": 0,
        "n": 9,
        "p": 0,
        "tasks": 2
      },
      {
        "domain": "Machine learning",
        "rowId": "tencent/hy4-preview@@think",
        "k": 0,
        "n": 9,
        "p": 0,
        "tasks": 1
      },
      {
        "domain": "Security / cyber",
        "rowId": "claude-opus-5@@xhigh",
        "k": 0,
        "n": 1,
        "p": 0,
        "tasks": 1
      },
      {
        "domain": "Security / cyber",
        "rowId": "claude-opus-5@@high",
        "k": 7,
        "n": 12,
        "p": 0.5833333333333334,
        "tasks": 3
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "claude-opus-5@@high",
        "k": 12,
        "n": 19,
        "p": 0.631578947368421,
        "tasks": 5
      },
      {
        "domain": "Software engineering",
        "rowId": "claude-opus-5@@high",
        "k": 0,
        "n": 5,
        "p": 0,
        "tasks": 1
      },
      {
        "domain": "Software engineering",
        "rowId": "tencent/hy4-preview@@think",
        "k": 4,
        "n": 12,
        "p": 0.3333333333333333,
        "tasks": 2
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "tencent/hy4-preview@@think",
        "k": 7,
        "n": 13,
        "p": 0.5384615384615384,
        "tasks": 3
      },
      {
        "domain": "Electrical engineering (RTL)",
        "rowId": "gpt-5.5@@",
        "k": 19,
        "n": 26,
        "p": 0.7307692307692307,
        "tasks": 7
      },
      {
        "domain": "Science / STEM",
        "rowId": "claude-opus-4-8@@",
        "k": 78,
        "n": 109,
        "p": 0.7155963302752294,
        "tasks": 10
      },
      {
        "domain": "Machine learning",
        "rowId": "claude-opus-4-8@@",
        "k": 194,
        "n": 428,
        "p": 0.4532710280373832,
        "tasks": 66
      },
      {
        "domain": "Scientific computing",
        "rowId": "claude-opus-4-8@@",
        "k": 304,
        "n": 687,
        "p": 0.44250363901018924,
        "tasks": 119
      },
      {
        "domain": "Cloud operations",
        "rowId": "claude-opus-4-7@@",
        "k": 53,
        "n": 162,
        "p": 0.3271604938271605,
        "tasks": 19
      },
      {
        "domain": "Mechanical engineering",
        "rowId": "claude-opus-4-8@@",
        "k": 30,
        "n": 114,
        "p": 0.2631578947368421,
        "tasks": 12
      },
      {
        "domain": "Games",
        "rowId": "claude-sonnet-4-6@@",
        "k": 11,
        "n": 31,
        "p": 0.3548387096774194,
        "tasks": 4
      },
      {
        "domain": "Games",
        "rowId": "claude-opus-4-8@@",
        "k": 158,
        "n": 290,
        "p": 0.5448275862068965,
        "tasks": 51
      },
      {
        "domain": "Data processing / data science",
        "rowId": "claude-opus-4-8@@",
        "k": 259,
        "n": 476,
        "p": 0.5441176470588235,
        "tasks": 92
      },
      {
        "domain": "Software engineering",
        "rowId": "claude-opus-4-8@@",
        "k": 376,
        "n": 713,
        "p": 0.5273492286115007,
        "tasks": 124
      },
      {
        "domain": "Games",
        "rowId": "claude-haiku-4-5@@",
        "k": 10,
        "n": 10,
        "p": 1,
        "tasks": 1
      },
      {
        "domain": "Games",
        "rowId": "gpt-5.5@@",
        "k": 129,
        "n": 243,
        "p": 0.5308641975308642,
        "tasks": 43
      },
      {
        "domain": "Software engineering",
        "rowId": "claude-haiku-4-5@@",
        "k": 8,
        "n": 9,
        "p": 0.8888888888888888,
        "tasks": 1
      },
      {
        "domain": "Machine learning",
        "rowId": "claude-sonnet-4-6@@",
        "k": 0,
        "n": 5,
        "p": 0,
        "tasks": 1
      },
      {
        "domain": "Data processing / data science",
        "rowId": "gpt-5.5@@",
        "k": 195,
        "n": 441,
        "p": 0.4421768707482993,
        "tasks": 87
      },
      {
        "domain": "Security / cyber",
        "rowId": "claude-opus-4-8@@",
        "k": 390,
        "n": 660,
        "p": 0.5909090909090909,
        "tasks": 128
      },
      {
        "domain": "Security / cyber",
        "rowId": "gpt-5.5@@",
        "k": 319,
        "n": 670,
        "p": 0.4761194029850746,
        "tasks": 124
      },
      {
        "domain": "Machine learning",
        "rowId": "gpt-5.5@@",
        "k": 159,
        "n": 325,
        "p": 0.48923076923076925,
        "tasks": 55
      },
      {
        "domain": "Science / STEM",
        "rowId": "gpt-5.5@@",
        "k": 9,
        "n": 20,
        "p": 0.45,
        "tasks": 4
      },
      {
        "domain": "Scientific computing",
        "rowId": "gpt-5.5@@",
        "k": 302,
        "n": 602,
        "p": 0.5016611295681063,
        "tasks": 106
      },
      {
        "domain": "Snorkel-reviewed tasks (category unknown)",
        "rowId": "claude-opus-4-8@@",
        "k": 24,
        "n": 40,
        "p": 0.6,
        "tasks": 8
      },
      {
        "domain": "Snorkel-reviewed tasks (category unknown)",
        "rowId": "gpt-5.5@@",
        "k": 21,
        "n": 40,
        "p": 0.525,
        "tasks": 8
      },
      {
        "domain": "Machine learning",
        "rowId": "claude-opus-5@@high",
        "k": 2,
        "n": 5,
        "p": 0.4,
        "tasks": 1
      },
      {
        "domain": "Operations",
        "rowId": "claude-opus-5@@high",
        "k": 2,
        "n": 5,
        "p": 0.4,
        "tasks": 1
      },
      {
        "domain": "DeepSWE (long-horizon SWE)",
        "rowId": "tencent/hy4-preview@@high",
        "k": 0,
        "n": 16,
        "p": 0,
        "tasks": 2
      },
      {
        "domain": "Data processing / data science",
        "rowId": "openai/gpt-6-astra@@medium",
        "k": 21,
        "n": 49,
        "p": 0.42857142857142855,
        "tasks": 16
      }
    ],
    "qa": {
      "duplicateTrialIdsRemoved": 0,
      "unlinkedTrials": 0,
      "invalidRewardRows": 1,
      "ignoredUnprovenExclusions": 0,
      "taskTrialDomainDisagreements": 27,
      "unknownEffortValidAgentRuns": 7885,
      "unknownDateValidAgentRuns": 6882,
      "timestampCollisionGroups": 185,
      "classifierProvenanceUnverified": 1889
    }
  }
}