{
  "meta": {
    "snapshotHash": "29b46d5ea53031efee082a7b9ff6f1c4535a37e21efce8924f9b05e6cdf2948f",
    "generatedAt": "2026-10-23T17:39:59+00:00",
    "sources": [
      {
        "name": "1. Delivery cohort export",
        "asOf": "2026-09-11"
      },
      {
        "name": "2. Website evaluation export",
        "asOf": "2026-08-23 (site build) / scraped 2026-09-11"
      },
      {
        "name": "3. Independent review inventory",
        "asOf": "2026-09-11"
      },
      {
        "name": "4. Independent review inventory",
        "asOf": "2026-09-12"
      },
      {
        "name": "5. Repository change inventory",
        "asOf": "2026-09-11"
      },
      {
        "name": "6. Delivery cohort export",
        "asOf": "2026-09-11"
      },
      {
        "name": "7. Repository evaluation export",
        "asOf": "2026-09-11"
      },
      {
        "name": "8. Repository evaluation export",
        "asOf": "2026-09-11"
      },
      {
        "name": "9. Repository evaluation export",
        "asOf": "2026-10-16"
      },
      {
        "name": "10. Console evaluation snapshot",
        "asOf": "2026-10-22"
      },
      {
        "name": "11. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "12. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "13. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "14. Independent review inventory",
        "asOf": "2026-10-22"
      },
      {
        "name": "15. Repository evaluation export",
        "asOf": "2026-10-21"
      },
      {
        "name": "16. Independent review inventory",
        "asOf": "2026-10-21"
      },
      {
        "name": "17. Independent review inventory",
        "asOf": "2026-10-21"
      },
      {
        "name": "18. Delivery cohort export",
        "asOf": "2026-10-16"
      },
      {
        "name": "19. Independent review inventory",
        "asOf": "2026-10-14"
      },
      {
        "name": "20. Domain taxonomy inventory",
        "asOf": "2026-10-14"
      },
      {
        "name": "21. Independent review inventory",
        "asOf": "2026-10-20"
      },
      {
        "name": "22. Repository change inventory",
        "asOf": "2026-10-23"
      },
      {
        "name": "23. Delivery cohort export",
        "asOf": "2026-10-22"
      }
    ],
    "methodsVersion": "Syncvals analysis v1.2 · 2026-10-24",
    "bootstrap": {
      "replicates": 2000,
      "seed": 20261023,
      "method": "Paired task resampling, then binomial run resampling; percentile 95% interval"
    },
    "limitations": [
      "This is an observational catalog of recorded agent systems, not a balanced experiment or a representative sample of coding work.",
      "Current and legacy status comes from the frozen snapshot. No external release date or model identity was inferred.",
      "Aggregate rates use one latest nondefective version per task ID and at least three valid runs per task. Rows with fewer than three eligible tasks show counts only.",
      "Task names and display keys do not establish identity. Distinct task IDs remain distinct even if their names match.",
      "Confidence intervals describe sampling uncertainty under the stated assumptions. They do not correct selection bias, missing results, correlated task families, harness changes, or optional stopping.",
      "Two-stage empirical bootstrap intervals are approximate. Boundary cells cannot generate unobserved outcomes and may produce zero-width intervals; no certainty claim follows.",
      "The archive has no complete task-selection field. Delivery source marks a known curated proxy, while other tasks have unknown selection status.",
      "Frozen-classifier provenance is declared by the snapshot; classifier model IDs and prompt hashes are absent, so this build cannot independently attest them.",
      "Most historical effort modes, execution dates and CLI versions are missing. Unknown values remain unknown.",
      "One snapshot-valid negative reward is excluded as invalid_reward_range in analysis. The historical inventory retains original snapshot validity and trial-domain assignments for reproducibility.",
      "Classifiers explain verifier grades. HARNESS_ERROR is excluded under the inherited policy, including classifier inconsistencies; this policy can introduce outcome-dependent missingness.",
      "Duplicate trial IDs use console > site > delivery > repo precedence. Shared timestamps do not identify duplicate executions; timestamp collisions are retained.",
      "Pooled-run Wilson intervals are descriptive and assume independent Bernoulli trials. Headline comparisons use task macro averages and paired task resampling.",
      "Sign-test values are conditional sign diagnostics, not validated tests of equal model capability; unequal repetition counts can violate the fair-sign null. No superiority inference is issued.",
      "Reviews are already represented in the normalized trial labels. The site hashes the review inventory but does not adjudicate or independently establish reviewer independence.",
      "Legacy showcase trial evidence URLs have been retired because those routes return 404. Missing public evidence is shown explicitly, not replaced with a proof claim."
    ]
  },
  "sourceFilesSha256": {
    "tasks.jsonl": "dc8f52be4977060f331959d713dd0ce08086376cc69e0b9fb0a90f339685db80",
    "trials.jsonl": "a92607261d09272f7199775c2ce2a5d130cb961cdbce69e622bf22308f9a7e11",
    "models.jsonl": "b62ffa5695ce314712a8c894ee565b6c23cb1c22137b97aa0fd7ba719370128e",
    "reviews.jsonl": "7521f8d2502bc74ecd07d0ae04f647010630bc5ef9ec7ada8aca8e3071594273",
    "manifest.json": "ff1ab3aa84d76d87fa9cf7d96bb60beb8197f89cdea10aaabbd906c9bce79423",
    "coverage.json": "3e81864be6ab44e6b7229b4f7830b65287f0545c803738816036e521e29bb097"
  },
  "qa": {
    "duplicateTrialIdsRemoved": 0,
    "unlinkedTrials": 0,
    "invalidRewardRows": 1,
    "ignoredUnprovenExclusions": 0,
    "taskTrialDomainDisagreements": 27,
    "unknownEffortValidAgentRuns": 7885,
    "unknownDateValidAgentRuns": 6882,
    "timestampCollisionGroups": 185,
    "classifierProvenanceUnverified": 1889
  },
  "exportPolicy": "Allowlisted scalar fields only. No raw snapshot, reviewer identity, local evidence path, account label or original free-form provenance is published."
}