tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.ls -la /workspace/app/ /workspace/
/workspace/: total 19 drwxrws--- 4 h2tagent h2tagent 4096 Jun 1 08:53 . drwxr-xr-x 28 root root 4096 Jun 1 09:33 .. -rw-rw---- 1 h2tagent h2tagent 2821 Jun 1 08:20 analysis.R drwxrws--- 2 h2tagent h2tagent 4096 Jun 1 08:53 app drwxrws--- 2 h2tagent h2tagent 4096 Jun 1 08:53 output /workspace/app/: total 424 drwxrws--- 2 h2tagent h2tagent 4096 Jun 1 08:53 . drwxrws--- 4 h2tagent h2tagent 4096 Jun 1 08:53 .. -rw-rw---- 1 h2tagent h2tagent 1725 Jun 1 08:20 dataset_manifest.json -rw-rw---- 1 h2tagent h2tagent 423783 Jun 1 08:20 lgd_workouts_source.csv
cat /workspace/app/dataset_manifest.json
{
"dataset_name": "Defaulted-loan workout panel for Loss Given Default modelling",
"source": "Calibrated to Lending Club historical loan-level disclosures of defaulted accounts and their post-default workout ledger (https://www.lendingclub.com/info/statistics.action , historical bulk files).",
"license": "Synthetic but matches the Lending Club workout schema for grading.",
"columns": {
"loan_id": "Anonymous loan identifier.",
"grade": "Lending Club grade A-G assigned at origination.",
"purpose": "Borrower-stated loan purpose.",
"home_ownership": "RENT / OWN / MORTGAGE.",
"principal_at_default": "Outstanding principal at the moment of default (USD).",
"default_month": "Months from origination to default.",
"workout_months_observed": "Months of post-default observation in the panel.",
"time_to_resolution": "Months from default to terminal resolution (recovery curve flat-lines).",
"is_resolved": "1 if workout_months_observed >= time_to_resolution; else 0 (right-censored workout).",
"recovered_amount": "Cumulative dollars recovered as of the as-of date (USD).",
"annual_inc": "Borrower annual income at origination."
},
"definitions": {
"lgd": "LGD = 1 - recovered_amount / principal_at_default , only valid when is_resolved == 1.",
"censoring": "Rows with is_resolved == 0 are right-censored: their LGD is incompletely observed. Censored rows have partial recovery, so recovered_amount understates the true terminal recovery , a naive mean of recovered/principal is biased downward, and therefore LGD = 1 - recovered/principal is biased upward (naive overestimates true LGD). Restrict to resolved workouts or model the cure mechanism."
}
}cat /workspace/analysis.R
#!/usr/bin/env Rscript
# LGD analysis , DRAFT. Audit and finish.
suppressPackageStartupMessages({
library(readr); library(dplyr); library(tidyr); library(jsonlite); library(ggplot2)
})
LGD_PATH <- Sys.getenv("LGD_PATH", "/workspace/app/lgd_workouts_source.csv")
OUT <- Sys.getenv("LGD_OUTPUT_DIR", "/workspace/output")
dir.create(OUT, showWarnings = FALSE, recursive = TRUE)
df <- read_csv(LGD_PATH, show_col_types = FALSE)
# Naive LGD over all rows (this is the biased baseline)
df <- df %>% mutate(observed_lgd = 1 - recovered_amount / principal_at_default)
naive_lgd <- mean(df$observed_lgd)
# "corrected" , but the draft uses ALL rows again, just trimming outliers
corrected_lgd <- mean(df$observed_lgd[df$observed_lgd > 0.0 & df$observed_lgd < 1.0])
# bootstrap on the wrong (full) dataset , and only 200 reps
set.seed(20260606)
boot <- replicate(200, mean(sample(df$observed_lgd, replace = TRUE)))
ci_lo <- quantile(boot, 0.025); ci_hi <- quantile(boot, 0.975)
bg <- df %>% group_by(grade) %>% summarise(corrected_lgd = mean(observed_lgd), .groups = "drop")
bp <- df %>% group_by(purpose) %>% summarise(corrected_lgd = mean(observed_lgd), .groups = "drop")
panel <- df %>% group_by(grade, purpose) %>%
summarise(n_total = n(), n_resolved = sum(is_resolved),
naive_lgd = mean(observed_lgd),
corrected_lgd = mean(observed_lgd), .groups = "drop")
write_csv(panel, file.path(OUT, "lgd_panel.csv"))
bias_diff <- naive_lgd - corrected_lgd
direction <- if (abs(bias_diff) < 0.05) "close" else if (bias_diff > 0) "materially_overestimates" else "materially_underestimates"
metrics <- list(
data_summary = list(n_total = nrow(df), n_resolved = sum(df$is_resolved),
n_censored = nrow(df) - sum(df$is_resolved),
share_resolved = mean(df$is_resolved)),
naive_lgd = list(value = naive_lgd),
corrected_lgd = list(value = corrected_lgd),
bootstrap_ci = list(ci_low = unname(ci_lo), ci_high = unname(ci_hi)),
lgd_by_grade = setNames(as.list(bg$corrected_lgd), bg$grade),
lgd_by_purpose = setNames(as.list(bp$corrected_lgd), bp$purpose),
bias_assessment = list(naive_minus_corrected = bias_diff, direction = direction)
)
write_json(metrics, file.path(OUT, "metrics.json"), auto_unbox = TRUE,
digits = 6, pretty = TRUE)
cat(paste0("Naive LGD: ", round(naive_lgd, 3),
"\nCorrected: ", round(corrected_lgd, 3),
"\nDirection: ", direction, "\n"),
file = file.path(OUT, "insights.md"))
write_json(list(figures = list()), file.path(OUT, "plot_contract.json"),
auto_unbox = TRUE, pretty = TRUE)
png(file.path(OUT, "fig1_lgd_by_grade.png"), width = 600, height = 400); plot.new(); dev.off()
png(file.path(OUT, "fig2_naive_vs_corrected.png"), width = 600, height = 400); plot.new(); dev.off()head -3 /workspace/app/lgd_workouts_source.csv; wc -l /workspace/app/lgd_workouts_source.csv
loan_id,grade,purpose,home_ownership,principal_at_default,default_month,workout_months_observed,time_to_resolution,is_resolved,recovered_amount,annual_inc LC0000000,C,small_business,MORTGAGE,9248.9,12,16,8,1,2351.97,37618.14 LC0000001,A,debt_consolidation,RENT,14442.6,10,25,15,1,7783.99,135186.3 6001 /workspace/app/lgd_workouts_source.csv
cd /workspace && Rscript -e '
df <- readr::read_csv("/workspace/app/lgd_workouts_source.csv", show_col_types=FALSE)
cat("nrow:", nrow(df), "\n")
cat("n_resolved:", sum(df$is_resolved), "\n")
cat("share_resolved:", mean(df$is_resolved), "\n")
cat("grades:", paste(sort(unique(df$grade)), collapse=","), "\n")
cat("purposes:", paste(sort(unique(df$purpose)), collapse=","), "\n")
naive <- mean(1 - df$recovered_amount / df$principal_at_default)
corr <- mean(1 - df$recovered_amount[df$is_resolved==1] / df$principal_at_default[df$is_resolved==1])
cat("naive:", naive, "\n")
cat("corrected:", corr, "\n")
cat("diff:", naive - corr, "\n")
'
nrow: 6000 n_resolved: 3185 share_resolved: 0.5308333 grades: A,B,C,D,E,F,G purposes: credit_card,debt_consolidation,home_improvement,major_purchase,other,small_business naive: 0.677057 corrected: 0.5651259 diff: 0.1119311
command: create
path: /workspace/analysis.R
file_text: #!/usr/bin/env Rscript
# LGD analysis on Lending Club defaulted-loan workout panel.
#
# The dataset mixes resolved workouts (is_resolved == 1, terminal recovery
# observed) and right-censored workouts (is_resolved == 0, partial recovery).
# Censored rows have recovered_amount BELOW their eventual terminal recovery,
# so 1 - recovered/principal is inflated on those rows. Averaging that
# ratio over all rows ("naive") therefore biases LGD UPWARD. The defensible
# ("corrected") estimator restricts to resolved workouts.
suppressPackageStartupMessages({
library(readr); library(dplyr); library(tidyr)
library(jsonlite); library(ggplot2)
})
LGD_PATH <- Sys.getenv("LGD_PATH", "/workspace/app/lgd_workouts_source.csv")
OUT <- Sys.getenv("LGD_OUTPUT_DIR", "/workspace/output")
dir.create(OUT, showWarnings = FALSE, recursive = TRUE)
df <- read_csv(LGD_PATH, show_col_types = FALSE)
# Row-level LGD ratio (definitionally valid only for is_resolved == 1, but we
# compute it on every row so we can form the biased naive mean too).
df <- df %>% mutate(lgd_row = 1 - recovered_amount / principal_at_default)
# -- Headline estimators ------------------------------------------------------
naive_lgd <- mean(df$lgd_row) # biased
resolved <- df %>% filter(is_resolved == 1)
corrected_lgd <- mean(resolved$lgd_row) # defensible
# -- Paired bootstrap over resolved loans, B = 1000 ---------------------------
B <- 1000
n_res <- nrow(resolved)
lgd_res <- resolved$lgd_row
set.seed(20260606)
boot_means <- replicate(B, mean(lgd_res[sample.int(n_res, n_res, replace = TRUE)]))
ci_lo <- unname(quantile(boot_means, 0.025))
ci_hi <- unname(quantile(boot_means, 0.975))
# -- Corrected LGD by grade and by purpose (resolved-only population) ---------
bg <- resolved %>% group_by(grade) %>% summarise(lgd = mean(lgd_row), .groups = "drop") %>% arrange(grade)
bp <- resolved %>% group_by(purpose) %>% summarise(lgd = mean(lgd_row), .groups = "drop") %>% arrange(purpose)
# -- grade x purpose panel ----------------------------------------------------
# naive_lgd = mean over every row in the cell (biased baseline).
# corrected_lgd = mean restricted to resolved rows in the cell; NaN only when
# the cell has zero resolved loans.
naive_cell <- df %>% group_by(grade, purpose) %>%
summarise(n_total = n(),
n_resolved = sum(is_resolved),
naive_lgd = mean(lgd_row),
.groups = "drop")
resolved_cell <- resolved %>% group_by(grade, purpose) %>%
summarise(corrected_lgd = mean(lgd_row), .groups = "drop")
panel <- naive_cell %>%
left_join(resolved_cell, by = c("grade", "purpose")) %>%
mutate(corrected_lgd = ifelse(n_resolved > 0, corrected_lgd, NaN)) %>%
select(grade, purpose, n_total, n_resolved, naive_lgd, corrected_lgd) %>%
arrange(grade, purpose)
write_csv(panel, file.path(OUT, "lgd_panel.csv"))
# -- Bias assessment ----------------------------------------------------------
bias_diff <- naive_lgd - corrected_lgd
direction <- if (naive_lgd < corrected_lgd - 0.05) {
"materially_underestimates"
} else if (naive_lgd > corrected_lgd + 0.05) {
"materially_overestimates"
} else {
"close"
}
# -- metrics.json -------------------------------------------------------------
metrics <- list(
data_summary = list(
n_total = nrow(df),
n_resolved = sum(df$is_resolved),
n_censored = nrow(df) - sum(df$is_resolved),
share_resolved = mean(df$is_resolved)
),
naive_lgd = list(value = naive_lgd),
corrected_lgd = list(value = corrected_lgd),
bootstrap_ci = list(ci_low = ci_lo, ci_high = ci_hi),
lgd_by_grade = setNames(as.list(bg$lgd), bg$grade),
lgd_by_purpose = setNames(as.list(bp$lgd), bp$purpose),
bias_assessment = list(
naive_minus_corrected = bias_diff,
direction = direction
)
)
write_json(metrics, file.path(OUT, "metrics.json"
…[truncated]Existing file was successfully overwritten at /workspace/analysis.R
rm -rf /workspace/output && mkdir -p /workspace/output && cd /workspace && Rscript analysis.R
Done. naive=0.6771 corrected=0.5651 CI=[0.5577, 0.5727] direction=materially_overestimates
ls -la /workspace/output/; echo "---metrics.json---"; cat /workspace/output/metrics.json
total 55
drwxr-sr-x 2 h2tagent h2tagent 4096 Jun 1 09:34 .
drwxrws--- 4 h2tagent h2tagent 4096 Jun 1 09:34 ..
-rw-r--r-- 1 h2tagent h2tagent 22864 Jun 1 09:34 fig1_lgd_by_grade.png
-rw-r--r-- 1 h2tagent h2tagent 18562 Jun 1 09:34 fig2_naive_vs_corrected.png
-rw-r--r-- 1 h2tagent h2tagent 999 Jun 1 09:34 insights.md
-rw-r--r-- 1 h2tagent h2tagent 2592 Jun 1 09:34 lgd_panel.csv
-rw-r--r-- 1 h2tagent h2tagent 807 Jun 1 09:34 metrics.json
-rw-r--r-- 1 h2tagent h2tagent 793 Jun 1 09:34 plot_contract.json
---metrics.json---
{
"data_summary": {
"n_total": 6000,
"n_resolved": 3185,
"n_censored": 2815,
"share_resolved": 0.53083333
},
"naive_lgd": {
"value": 0.677057
},
"corrected_lgd": {
"value": 0.56512588
},
"bootstrap_ci": {
"ci_low": 0.55767503,
"ci_high": 0.57265928
},
"lgd_by_grade": {
"A": 0.43529755,
"B": 0.49919225,
"C": 0.57150847,
"D": 0.65262141,
"E": 0.71929622,
"F": 0.79746682,
"G": 0.84954799
},
"lgd_by_purpose": {
"credit_card": 0.5911141,
"debt_consolidation": 0.54887667,
"home_improvement": 0.54207074,
"major_purchase": 0.61012939,
"other": 0.54548081,
"small_business": 0.6255756
},
"bias_assessment": {
"naive_minus_corrected": 0.11193112,
"direction": "materially_overestimates"
}
}echo "---plot_contract.json (head)---"; head -40 /workspace/output/plot_contract.json
echo "---lgd_panel.csv---"; head -10 /workspace/output/lgd_panel.csv
echo "---panel NaN check---"
awk -F, 'NR>1 { print $3, $4, $5, $6 }' /workspace/output/lgd_panel.csv | awk '{ if ($2 > 0 && $4 == "NaN") print "VIOLATION:", $0 }'
echo "---insights.md---"; cat /workspace/output/insights.md---plot_contract.json (head)---
{
"figures": [
{
"name": "fig1_lgd_by_grade",
"series": [
{
"name": "corrected_lgd_by_grade",
"x": [
"A",
"B",
"C",
"D",
"E",
"F",
"G"
],
"y": [
0.43529755,
0.49919225,
0.57150847,
0.65262141,
0.71929622,
0.79746682,
0.84954799
]
}
]
},
{
"name": "fig2_naive_vs_corrected",
"series": [
{
"name": "naive_vs_corrected",
"x": [
"naive",
"corrected"
],
"y": [
0.677057,
0.56512588
---lgd_panel.csv---
grade,purpose,n_total,n_resolved,naive_lgd,corrected_lgd
A,credit_card,219,136,0.5607065958509241,0.45012049865070586
A,debt_consolidation,564,297,0.5702761415442725,0.4202599413316665
A,home_improvement,114,64,0.5292722852167128,0.4073998487935503
A,major_purchase,67,35,0.6368190733531681,0.5021729573367807
A,other,90,52,0.5730028339585355,0.4161251086238681
A,small_business,86,44,0.6524850156856171,0.501025517461768
B,credit_card,321,165,0.6650274867660854,0.5529570202359294
B,debt_consolidation,796,431,0.6046227193564916,0.4831736900891306
B,home_improvement,144,72,0.5957294070774534,0.4692244006760461
---panel NaN check---
---insights.md---
# Portfolio LGD read , Lending Club defaulted-loans workout panel
Of 6000 defaulted loans in the workout panel, 3185 (53.1%) have resolved workouts and 2815 remain right-censored. The **naive** portfolio LGD (mean of `1 - recovered/principal` over every row) is **0.677**, while the **corrected** estimator that restricts to resolved workouts is **0.565** with a 95% paired-bootstrap CI of [0.558, 0.573] (B = 1000).
The naive estimate is 0.112 higher than the corrected one, so the naive baseline **materially_overestimates** the defensible LGD. This is the expected direction: censored loans have not yet reached terminal recovery, so their `recovered_amount` understates the eventual payoff and `1 - recovered/principal` is inflated. Averaging that inflated ratio across censored and resolved rows pushes the naive mean upward.
For defensible capital / provisioning use the **corrected** number; grade- and purpose-level cuts in `lgd_panel.csv` are computed on the same resolved-only basis.cd /workspace && python3 - <<'PY'
import json
m = json.load(open("/workspace/output/metrics.json"))
assert set(m) >= {"data_summary","naive_lgd","corrected_lgd","bootstrap_ci","lgd_by_grade","lgd_by_purpose","bias_assessment"}
ds = m["data_summary"]
assert ds["n_total"] == ds["n_resolved"] + ds["n_censored"]
assert abs(ds["share_resolved"] - ds["n_resolved"]/ds["n_total"]) < 1e-9
n = m["naive_lgd"]["value"]; c = m["corrected_lgd"]["value"]
assert m["bias_assessment"]["naive_minus_corrected"] == n - c or abs(m["bias_assessment"]["naive_minus_corrected"] - (n-c)) < 1e-9
expect = "materially_underestimates" if n < c - 0.05 else ("materially_overestimates" if n > c + 0.05 else "close")
assert m["bias_assessment"]["direction"] == expect
assert set("ABCDEFG") == set(m["lgd_by_grade"])
print("metrics OK; direction:", m["bias_assessment"]["direction"])
pc = json.load(open("/workspace/output/plot_contract.json"))
assert isinstance(pc["figures"], list) and len(pc["figures"])==2
names = [f["name"] for f in pc["figures"]]
assert names == ["fig1_lgd_by_grade","fig2_naive_vs_corrected"]
for f in pc["figures"]:
assert isinstance(f["series"], list) and len(f["series"])>0
print("plot_contract OK")
import struct, os
for p in ("/workspace/output/fig1_lgd_by_grade.png","/workspace/output/fig2_naive_vs_corrected.png"):
with open(p,"rb") as fh:
data = fh.read(24)
w = int.from_bytes(data[16:20],"big"); h = int.from_bytes(data[20:24],"big")
print(p, w, "x", h, "size=", os.path.getsize(p))
assert w>=200 and h>=150
PY
echo "---constraint grep---"
grep -nE 'file\.copy|file\.rename|file\.link|file\.symlink|file\.append|system\(|system2\(|shell\(|pipe\(|saveRDS|writeBin|as\.raw|sink\(|readRDS|readBin|\bload\(' /workspace/analysis.R && echo "FOUND FORBIDDEN" || echo "no forbidden calls"---constraint grep--- no forbidden calls [stderr] Traceback (most recent call last): File "<stdin>", line 6, in <module> AssertionError
python3 -c 'import json; m=json.load(open("/workspace/output/metrics.json"))["data_summary"]; print(m); print(m["n_total"], m["n_resolved"]+m["n_censored"])'{'n_total': 6000, 'n_resolved': 3185, 'n_censored': 2815, 'share_resolved': 0.53083333}
6000 6000python3 - <<'PY'
import json, os, struct
m = json.load(open("/workspace/output/metrics.json"))
ds = m["data_summary"]
print("share_resolved stored:", ds["share_resolved"], "true:", ds["n_resolved"]/ds["n_total"])
# tolerate JSON rounding
assert abs(ds["share_resolved"] - ds["n_resolved"]/ds["n_total"]) < 1e-6
n = m["naive_lgd"]["value"]; c = m["corrected_lgd"]["value"]
expect = "materially_underestimates" if n < c - 0.05 else ("materially_overestimates" if n > c + 0.05 else "close")
print("expected direction:", expect, "got:", m["bias_assessment"]["direction"])
assert m["bias_assessment"]["direction"] == expect
# bootstrap sanity
ci = m["bootstrap_ci"]
assert ci["ci_low"] < c < ci["ci_high"], (ci, c)
print("CI brackets corrected:", ci, "corrected=", c)
# panel: every cell with n_resolved > 0 must have a numeric corrected_lgd
import csv
with open("/workspace/output/lgd_panel.csv") as fh:
rows = list(csv.DictReader(fh))
print("panel rows:", len(rows))
bad = [r for r in rows if int(r["n_resolved"])>0 and r["corrected_lgd"] in ("NaN","nan","","NA")]
assert not bad, bad
nanrows = [r for r in rows if r["corrected_lgd"] in ("NaN","nan","","NA")]
print("NaN-corrected cells (should all have n_resolved==0):", len(nanrows))
for r in nanrows:
assert int(r["n_resolved"]) == 0
print("panel OK")
# PNG dims
for p in ("/workspace/output/fig1_lgd_by_grade.png","/workspace/output/fig2_naive_vs_corrected.png"):
with open(p,"rb") as fh: data = fh.read(24)
w = int.from_bytes(data[16:20],"big"); h = int.from_bytes(data[20:24],"big")
print(p, "->", w, "x", h, "size=", os.path.getsize(p))
assert w>=200 and h>=150
print("ALL OK")
PYshare_resolved stored: 0.53083333 true: 0.5308333333333334
expected direction: materially_overestimates got: materially_overestimates
CI brackets corrected: {'ci_low': 0.55767503, 'ci_high': 0.57265928} corrected= 0.56512588
panel rows: 42
NaN-corrected cells (should all have n_resolved==0): 0
panel OK
/workspace/output/fig1_lgd_by_grade.png -> 660 x 440 size= 22864
/workspace/output/fig2_naive_vs_corrected.png -> 660 x 440 size= 18562
ALL OKrm -rf /tmp/out2 && LGD_OUTPUT_DIR=/tmp/out2 Rscript /workspace/analysis.R
diff <(python3 -c 'import json;print(json.dumps(json.load(open("/workspace/output/metrics.json")),sort_keys=True,indent=2))') \
<(python3 -c 'import json;print(json.dumps(json.load(open("/tmp/out2/metrics.json")),sort_keys=True,indent=2))') && echo "REPRODUCIBLE ✓"Done. naive=0.6771 corrected=0.5651 CI=[0.5577, 0.5727] direction=materially_overestimates REPRODUCIBLE ✓
No file changes, the agent left the workspace untouched.
# SyncVals normalized verifier summary
# task: lending-club-lgd-bias-correction-r
# attempt: 8 status: completed raw_score: 1.0
# reward rule: binary: score 1.0 == 1.0
# => reward = 1.0 (PASS)
#
# Note: these tasks score via an external hidden grader; no pytest/verifier
# block is embedded in the agent transcript. The block below is the last
# sizable tool-output captured in the run (supporting evidence).
----------------------------------------------------------------------
share_resolved stored: 0.53083333 true: 0.5308333333333334
expected direction: materially_overestimates got: materially_overestimates
CI brackets corrected: {'ci_low': 0.55767503, 'ci_high': 0.57265928} corrected= 0.56512588
panel rows: 42
NaN-corrected cells (should all have n_resolved==0): 0
panel OK
/workspace/output/fig1_lgd_by_grade.png -> 660 x 440 size= 22864
/workspace/output/fig2_naive_vs_corrected.png -> 660 x 440 size= 18562
ALL OKReproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_f4104eaede6343a7. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_f4104eaede6343a7 · verifier authoritative; classifier explanatory.