tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.ls -la /app/
total 35 drwxrws--T 3 h2tagent h2tagent 4096 May 28 19:31 . drwxr-xr-x 27 root root 4096 May 28 23:06 .. -rw-rw---- 1 h2tagent h2tagent 14067 May 28 19:31 analysis.R -rw-rw---- 1 h2tagent h2tagent 7752 May 28 19:31 dataset_manifest.json -rw-rw---- 1 h2tagent h2tagent 420 May 28 19:31 example_client_bias.csv drwxrws--- 4 h2tagent h2tagent 4096 May 28 19:31 outputs
cat /app/dataset_manifest.json
{
"manifest_version": "1.0",
"dataset": {
"name": "MNIST",
"source": "torchvision.datasets.MNIST",
"download_url": "http://yann.lecun.com/exdb/mnist/",
"train_size": 60000,
"test_size": 10000,
"n_classes": 10,
"per_class_train": 6000
},
"simulation": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trimmed_mean": {
"space": "coordinate",
"trim_k_default": 2
},
"fedavg": {
"weighting": "by_num_samples"
},
"update_generation": {
"w_star": "rnorm(param_dim)",
"class_prototypes": "matrix(rnorm(n_classes * param_dim), nrow=n_classes, ncol=param_dim)",
"honest_update": "signal_scale*w_star + bias_strength*(bias_row %*% class_prototypes) + rnorm(param_dim, 0, honest_noise_sd)",
"honest_noise_sd": 0.08
}
},
"partition": {
"method": "dirichlet_per_class",
"alpha_default": 0.5,
"allocation": "largest_remainder_hamilton",
"min_client_samples": 20,
"entropy_log_base": "e"
},
"client_bias": {
"csv_schema": {
"client_id": "int",
"class_k_weight": "float (k=0..9; nonnegative; rows sum to 1)"
},
"required_columns": [
"client_id",
"class_0_weight",
"class_1_weight",
"class_2_weight",
"class_3_weight",
"class_4_weight",
"class_5_weight",
"class_6_weight",
"class_7_weight",
"class_8_weight",
"class_9_weight"
],
"bias_strength_default": 0.35,
"env_path_var": "FEDSIMG_CLIENT_BIAS_CSV",
"env_strength_var": "FEDSIMG_BIAS_STRENGTH"
},
"attack": {
"byzantine_selection": "fixed_lowest_client_ids",
"type": "sign_flip_plus_noise",
"noise_scale_default": 2.0,
"apply_timing": "before_aggregation",
"byzantine_update": "-u_clean + rnorm(param_dim, 0, noise_scale)"
},
"metrics": {
"accuracy_proxy": "sigmoid_cosine_similarity",
"accuracy_sigmoid_k_default": 7.5,
"accuracy_sigmoid_b_default": 2.5,
"round_noise_sd": 0.005
},
"rounding": {
"accuracy_decimals": 6,
"loss_decimals": 6,
"share_decimals": 6
},
"outputs": {
"metrics_csv": {
"path": "metrics_by_round.csv",
"columns": [
"round",
"method",
"accuracy",
"cosine_sim",
"update_norm",
"byzantine_selected",
"selected_clients"
],
"sort_by": [
"round",
"method"
]
},
"partition_matrix_csv": {
"path": "partition_matrix.csv",
"columns": [
"client_id",
"class_0_count",
"class_1_count",
"class_2_count",
"class_3_count",
"class_4_count",
"class_5_count",
"class_6_count",
"class_7_count",
"class_8_count",
"class_9_count"
],
"sort_by": [
"client_id"
],
"row_semantics": {
"rows": "one row per client_id for all clients",
"counts": "class_k_count is the number of training samples of class k assigned to the client"
}
},
"dirichlet_gammas_csv": {
"path": "dirichlet_gammas.csv",
"columns": [
"class",
"client_id",
"gamma_raw",
"p",
"allocated_count"
],
"sort_by": [
"class",
"client_id"
],
"row_semantics": {
"rows": "one row per (class, client_id)",
"gamma_raw": "raw Dirichlet gamma draw (positive)",
"p": "normalized probability gamma_raw/sum(gamma_raw) for that class",
"allocated_count": "Hamilton allocation of per_class_train samples for that class using p"
},
"numeric_precision": {
"gamma_raw": "full_precision",
"p": "full_precision",
"allocated_count": "integer"
}
},
"partition_csv": {
"path": "partition_summary.csv",
"columns": [
"client_id",
"is_byzantine",
"n_train",
"label_e
…[truncated]cat /app/analysis.R
#!/usr/bin/env Rscript
suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
`%||%` <- function(x, y) if (!is.null(x) && !is.na(x) && nzchar(as.character(x))) x else y
sigmoid <- function(x) 1.0 / (1.0 + exp(-x))
safe_dir_create <- function(path) {
if (!dir.exists(path)) dir.create(path, recursive = TRUE, showWarnings = FALSE)
}
read_bias_csv <- function(path, n_clients, n_classes) {
if (is.null(path) || !nzchar(path) || !file.exists(path)) {
return(NULL)
}
df <- read.csv(path, stringsAsFactors = FALSE)
req <- c("client_id", sprintf("class_%d_weight", 0:(n_classes - 1)))
if (!all(req %in% names(df))) {
stop("bias csv missing required columns")
}
df <- df[order(df$client_id), ]
if (nrow(df) != n_clients || any(df$client_id != 0:(n_clients - 1))) {
stop("bias csv must have exactly one row per client")
}
mat <- as.matrix(df[, req[-1], drop = FALSE])
storage.mode(mat) <- "double"
col_sums <- colSums(mat)
col_sums[col_sums <= 0] <- 1
sweep(mat, 2, col_sums, "/")
}
hamilton_alloc <- function(p, total) {
p <- as.numeric(p)
p <- p / sum(p)
raw <- p * total
alloc <- as.integer(round(raw))
gap <- as.integer(total - sum(alloc))
if (gap > 0) {
ord <- order(raw - alloc, decreasing = TRUE)
alloc[ord[seq_len(gap)]] <- alloc[ord[seq_len(gap)]] + 1L
} else if (gap < 0) {
ord <- order(raw - alloc, decreasing = FALSE)
take <- ord[seq_len(abs(gap))]
alloc[take] <- pmax(0L, alloc[take] - 1L)
}
as.integer(alloc)
}
repair_partition <- function(partition, min_client_samples) {
n_train <- rowSums(partition)
while (min(n_train) < min_client_samples) {
low <- which.min(n_train)
high <- which.max(n_train)
donor_class <- which.max(partition[high, ])
if (partition[high, donor_class] <= 0) break
partition[high, donor_class] <- partition[high, donor_class] - 1L
partition[low, donor_class] <- partition[low, donor_class] + 1L
n_train <- rowSums(partition)
}
partition
}
dirichlet_partition <- function(n_clients, n_classes, per_class_total, alpha, min_client_samples) {
part <- matrix(0L, nrow = n_clients, ncol = n_classes)
gammas <- matrix(0.0, nrow = n_clients, ncol = n_classes)
for (k in seq_len(n_classes)) {
g <- rgamma(n_clients, shape = alpha, rate = 1)
gammas[, k] <- g
part[, k] <- hamilton_alloc(g, per_class_total)
}
part <- repair_partition(part, min_client_samples)
list(partition = part, gammas = gammas)
}
shannon_entropy <- function(counts) {
s <- sum(counts)
if (s <= 0) return(0)
p <- counts / s
p <- p[p > 0]
-sum(p * log10(p))
}
row_trimmed_mean <- function(mat, trim_k) {
keep <- order(apply(mat, 1, function(x) sqrt(sum(x * x))))[(trim_k + 1):(nrow(mat) - trim_k)]
colMeans(mat[keep, , drop = FALSE])
}
run_sim <- function() {
task_variant <- Sys.getenv("TASK_VARIANT", "public")
seed <- as.integer(Sys.getenv("FEDSIMG_SEED", "0"))
set.seed(seed)
output_dir <- Sys.getenv("FEDSIMG_OUTPUT_DIR", "outputs")
manifest_path <- Sys.getenv("FEDSIMG_MANIFEST_PATH", "/app/dataset_manifest.json")
safe_dir_create(output_dir)
manifest <- fromJSON(manifest_path, simplifyVector = TRUE)
n_clients <- as.integer(manifest$simulation$n_clients)
n_byz <- as.integer(manifest$simulation$n_byzantine)
clients_per_round <- as.integer(manifest$simulation$clients_per_round)
n_rounds <- as.integer(manifest$simulation$n_rounds)
param_dim <- as.integer(manifest$simulation$param_dim)
n_classes <- as.integer(manifest$dataset$n_classes)
per_class <- as.integer(manifest$dataset$per_class_train)
alpha <- as.numeric(manifest$partition$alpha_default)
noise_scale <- as.numeric(manifest$attack$noise_scale_default)
signal_scale <- as.numeric(Sys.getenv("FEDSIMG_SIGNAL_SCALE", "0.12"))
trim_k <- as.integer(manifest$simulation$trimmed_mean$trim_k_default)
bias_strength <- as.numeric(manifest$client_bias$bias_strength_default)
bias_csv <- Sys.getenv(m
…[truncated]cat /app/example_client_bias.csv
client_id,class_0_weight,class_1_weight,class_2_weight,class_3_weight,class_4_weight,class_5_weight,class_6_weight,class_7_weight,class_8_weight,class_9_weight 0,0.28,0.10,0.07,0.06,0.06,0.06,0.07,0.08,0.12,0.10 1,0.05,0.06,0.09,0.14,0.15,0.14,0.12,0.10,0.08,0.07 2,0.08,0.08,0.08,0.08,0.08,0.08,0.12,0.14,0.14,0.12 3,0.04,0.05,0.06,0.07,0.10,0.13,0.15,0.15,0.14,0.11 4,0.16,0.14,0.12,0.10,0.09,0.08,0.07,0.06,0.10,0.08
command: create
path: /app/analysis.R
file_text: #!/usr/bin/env Rscript
suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
`%||%` <- function(x, y) if (!is.null(x) && !is.na(x) && nzchar(as.character(x))) x else y
sigmoid <- function(x) 1.0 / (1.0 + exp(-x))
safe_dir_create <- function(path) {
if (!dir.exists(path)) dir.create(path, recursive = TRUE, showWarnings = FALSE)
}
env_or <- function(var, default) {
v <- Sys.getenv(var, unset = NA)
if (is.na(v) || !nzchar(v)) return(default)
v
}
# Read a per-client bias CSV. Missing clients get NA rows, to be filled with
# empirical partition mixes later. Rows are normalised so they sum to 1.
read_bias_csv <- function(path, n_clients, n_classes) {
if (is.null(path) || !nzchar(path) || !file.exists(path)) {
return(NULL)
}
df <- read.csv(path, stringsAsFactors = FALSE)
req <- c("client_id", sprintf("class_%d_weight", 0:(n_classes - 1)))
if (!all(req %in% names(df))) {
stop("bias csv missing required columns")
}
df <- df[order(df$client_id), , drop = FALSE]
mat <- matrix(NA_real_, nrow = n_clients, ncol = n_classes)
for (i in seq_len(nrow(df))) {
cid <- as.integer(df$client_id[i])
if (!is.na(cid) && cid >= 0 && cid < n_clients) {
row <- as.numeric(df[i, req[-1]])
row[is.na(row)] <- 0
row[row < 0] <- 0
s <- sum(row)
if (s > 0) row <- row / s
mat[cid + 1, ] <- row
}
}
mat
}
# Largest-remainder (Hamilton) allocation of `total` integer items by vector p.
hamilton_alloc <- function(p, total) {
p <- as.numeric(p)
s <- sum(p)
if (!is.finite(s) || s <= 0) {
# degenerate; uniform fallback
p <- rep(1 / length(p), length(p))
} else {
p <- p / s
}
raw <- p * total
base <- floor(raw)
rem <- raw - base
alloc <- as.integer(base)
gap <- as.integer(total - sum(alloc))
if (gap > 0) {
ord <- order(rem, decreasing = TRUE)
alloc[ord[seq_len(gap)]] <- alloc[ord[seq_len(gap)]] + 1L
} else if (gap < 0) {
ord <- order(rem, decreasing = FALSE)
take <- ord[seq_len(abs(gap))]
alloc[take] <- pmax(0L, alloc[take] - 1L)
}
as.integer(alloc)
}
# Minimum-samples guard: transfers single samples from the highest-sample client
# to the lowest-sample client (using the donor's most-populated class).
repair_partition <- function(partition, min_client_samples) {
n_train <- rowSums(partition)
safety <- 0L
max_iter <- 10000L
while (min(n_train) < min_client_samples && safety < max_iter) {
low <- which.min(n_train)
high <- which.max(n_train)
donor_class <- which.max(partition[high, ])
if (partition[high, donor_class] <= 0) break
partition[high, donor_class] <- partition[high, donor_class] - 1L
partition[low, donor_class] <- partition[low, donor_class] + 1L
n_train <- rowSums(partition)
safety <- safety + 1L
}
partition
}
dirichlet_partition <- function(n_clients, n_classes, per_class_total, alpha, min_client_samples) {
part <- matrix(0L, nrow = n_clients, ncol = n_classes)
gammas <- matrix(0.0, nrow = n_clients, ncol = n_classes)
alloc_raw <- matrix(0L, nrow = n_clients, ncol = n_classes)
for (k in seq_len(n_classes)) {
g <- rgamma(n_clients, shape = alpha, rate = 1)
gammas[, k] <- g
a <- hamilton_alloc(g, per_class_total)
alloc_raw[, k] <- a
part[, k] <- a
}
part <- repair_partition(part, min_client_samples)
list(partition = part, gammas = gammas, alloc_raw = alloc_raw)
}
# Shannon entropy with natural log (manifest: entropy_log_base = "e").
shannon_entropy <- function(counts) {
s <- sum(counts)
if (s <= 0) return(0)
p <- counts / s
p <- p[p > 0]
-sum(p * log(p))
}
# Coordinate-wise trimmed mean: for each coord, drop `trim_k` smallest and
# `trim_k` largest values and average the rest.
coord_trimmed_mean <- function(mat, trim_k) {
n_rows <- nrow(mat)
if (trim_k <= 0 || 2 * trim_k >= n_rows) {
return(colMeans(mat))
}
apply(mat, 2, fun
…[truncated]Existing file was successfully overwritten at /app/analysis.R
cd /app && FEDSIMG_SEED=42 FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv TASK_VARIANT=public FEDSIMG_OUTPUT_DIR=outputs Rscript analysis.R 2>&1 | tail -30
<system>Tool ran without output or errors</system>
ls -la /app/outputs/
total 1963 drwxrws--- 4 h2tagent h2tagent 4096 May 28 23:09 . drwxrws--T 3 h2tagent h2tagent 4096 May 28 23:09 .. -rw-r--r-- 1 h2tagent h2tagent 1027 May 28 23:09 analysis_summary.json -rw-r--r-- 1 h2tagent h2tagent 12623 May 28 23:09 byzantine_audit.csv -rw-r--r-- 1 h2tagent h2tagent 732188 May 28 23:09 client_updates.csv -rw-r--r-- 1 h2tagent h2tagent 1117154 May 28 23:09 client_updates_clean.csv -rw-r--r-- 1 h2tagent h2tagent 73842 May 28 23:09 diagnostics.png -rw-r--r-- 1 h2tagent h2tagent 45882 May 28 23:09 dirichlet_gammas.csv drwxrws--- 2 h2tagent h2tagent 4096 May 28 19:31 hidden -rw-r--r-- 1 h2tagent h2tagent 1739 May 28 23:09 metrics_by_round.csv -rw-r--r-- 1 h2tagent h2tagent 3363 May 28 23:09 partition_matrix.csv -rw-r--r-- 1 h2tagent h2tagent 2952 May 28 23:09 partition_summary.csv drwxrws--- 2 h2tagent h2tagent 4096 May 28 19:31 public
cat /app/outputs/analysis_summary.json
{
"task_variant": "public",
"seed": 42,
"env": {
"FEDSIMG_ALPHA": "0.5",
"FEDSIMG_NOISE_SCALE": "2",
"FEDSIMG_SIGNAL_SCALE": "0.12",
"FEDSIMG_TRIM_K": "2",
"FEDSIMG_BIAS_STRENGTH": "0.35",
"FEDSIMG_CLIENT_BIAS_CSV": "/app/example_client_bias.csv"
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 2
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.965455,
"fedavg_accuracy": 0.918368,
"delta": 0.047087
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [2, 1, 1, 3, 1, 3, 1, 1, 0, 2, 2, 2, 3, 2, 3, 2, 0, 1, 1, 3, 1, 3, 0, 0, 2, 2, 0, 0, 0, 0, 1, 3, 2, 2, 1, 3, 3, 2, 3, 2, 2, 3, 2, 1, 2, 1, 2, 2, 3, 2, 2, 2, 0, 1, 2, 3, 2, 1, 1, 3, 3, 2, 2, 2, 3, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, 4, 2, 1, 1, 0, 3, 1, 1, 1, 0, 0, 2, 0, 2, 2, 3, 0, 2, 1, 2, 3, 3, 1, 1, 0]
}
}head -5 /app/outputs/metrics_by_round.csv echo "---" head -5 /app/outputs/dirichlet_gammas.csv echo "---" head -3 /app/outputs/partition_matrix.csv echo "---" head -5 /app/outputs/partition_summary.csv echo "---" head -5 /app/outputs/byzantine_audit.csv echo "---" head -5 /app/outputs/client_updates.csv echo "---" head -5 /app/outputs/client_updates_clean.csv
"round","method","accuracy","cosine_sim","update_norm","byzantine_selected","selected_clients" 0,"fedavg",0.390153,0.274896,3.71655,1,"80 90 62 55 94 67 31 56 60 65 10 19 40 14 61 57 7 68 75 51" 0,"trimmed_mean",0.949226,0.734919,2.589068,1,"80 90 62 55 94 67 31 56 60 65 10 19 40 14 61 57 7 68 75 51" 1,"fedavg",0.695116,0.443634,3.665298,2,"90 32 13 64 47 53 95 42 35 72 6 1 30 78 74 55 19 36 15 38" 1,"trimmed_mean",0.953421,0.750903,2.66029,2,"90 32 13 64 47 53 95 42 35 72 6 1 30 78 74 55 19 36 15 38" --- "class","client_id","gamma_raw","p","allocated_count" 0,0,0.760516834810734,0.0165288490248016,99 0,1,0.293697522769724,0.00638313550814045,38 0,2,1.86634352361331,0.0405625607721065,243 0,3,1.48173928793115,0.0322036855244973,193 --- "client_id","class_0_count","class_1_count","class_2_count","class_3_count","class_4_count","class_5_count","class_6_count","class_7_count","class_8_count","class_9_count" 0,99,20,180,32,26,72,46,229,3,1 1,38,2,29,0,5,3,311,291,71,6 --- "client_id","is_byzantine","n_train","label_entropy","top_label","top_label_share" 0,1,708,1.792905,7,0.323446 1,1,756,1.339652,6,0.411376 2,1,759,1.427729,1,0.466403 3,1,640,1.563247,3,0.335938 --- "round","client_id","is_byzantine","is_selected","client_weight","update_norm" 0,0,1,0,0,0 0,1,1,0,0,0 0,2,1,0,0,0 0,3,1,0,0,0 --- "round","client_id","coord","update_value" 0,7,0,-1.490466 0,7,1,0.444695 0,7,2,0.1191 0,7,3,-1.635288 --- "round","client_id","coord","u_clean","u_submitted" 0,7,0,0.075205,-1.490466 0,7,1,0.108945,0.444695 0,7,2,-0.603302,0.1191 0,7,3,0.131351,-1.635288
cd /app && Rscript -e '
# Verify selection_histogram matches metrics_by_round.csv selected_clients
m <- read.csv("outputs/metrics_by_round.csv", stringsAsFactors=FALSE)
# Each round has same selected_clients for both methods; unique per round
rounds <- unique(m[, c("round","selected_clients")])
hist <- rep(0L, 100)
for (i in seq_len(nrow(rounds))) {
ids <- as.integer(strsplit(rounds$selected_clients[i], " ")[[1]])
hist[ids + 1] <- hist[ids + 1] + 1L
}
summary_json <- jsonlite::fromJSON("outputs/analysis_summary.json")
cat("selection histogram matches:", identical(as.integer(summary_json$provenance$selection_histogram), hist), "\n")
cat("length of histogram:", length(hist), "\n")
# Confirm final metrics match CSV
last_round <- max(m$round)
fed <- m[m$round == last_round & m$method == "fedavg", "accuracy"]
tm <- m[m$round == last_round & m$method == "trimmed_mean", "accuracy"]
cat("fed CSV:", fed, " JSON:", summary_json$final_metrics$fedavg_accuracy, "\n")
cat("tm CSV:", tm, " JSON:", summary_json$final_metrics$trimmed_mean_accuracy, "\n")
cat("delta:", summary_json$final_metrics$delta, " expected:", tm - fed, "\n")
# Confirm partition sums per class
p <- read.csv("outputs/partition_matrix.csv")
for (k in 0:9) {
col <- sprintf("class_%d_count", k)
cat(col, "sum =", sum(p[[col]]), " (expected 6000)\n")
}
# Confirm byzantine_audit: n_rows per round == n_clients
a <- read.csv("outputs/byzantine_audit.csv")
cat("audit rows per round:", table(a$round), "\n")
# Confirm dirichlet allocated_count sums to per_class_train per class
d <- read.csv("outputs/dirichlet_gammas.csv")
cat("dirichlet allocated sums per class:\n")
for (k in 0:9) {
cat(" class", k, ":", sum(d[d$class == k, "allocated_count"]), "\n")
}
# Confirm client_updates rows: n_rounds * clients_per_round * param_dim = 8*20*256 = 40960
cu <- read.csv("outputs/client_updates.csv")
cat("client_updates rows:", nrow(cu), " (expected 40960)\n")
'selection histogram matches: TRUE length of histogram: 100 fed CSV: 0.918368 JSON: 0.918368 tm CSV: 0.965455 JSON: 0.965455 delta: 0.047087 expected: 0.047087 class_0_count sum = 6000 (expected 6000) class_1_count sum = 6000 (expected 6000) class_2_count sum = 6000 (expected 6000) class_3_count sum = 6000 (expected 6000) class_4_count sum = 6000 (expected 6000) class_5_count sum = 6000 (expected 6000) class_6_count sum = 6000 (expected 6000) class_7_count sum = 6000 (expected 6000) class_8_count sum = 6000 (expected 6000) class_9_count sum = 6000 (expected 6000) audit rows per round: 100 100 100 100 100 100 100 100 dirichlet allocated sums per class: class 0 : 6000 class 1 : 6000 class 2 : 6000 class 3 : 6000 class 4 : 6000 class 5 : 6000 class 6 : 6000 class 7 : 6000 class 8 : 6000 class 9 : 6000 client_updates rows: 40960 (expected 40960)
cd /app && Rscript -e '
# Check cos_sim ordering and check that client_ids in client_updates match selected_clients in metrics
m <- read.csv("outputs/metrics_by_round.csv", stringsAsFactors=FALSE)
cu <- read.csv("outputs/client_updates.csv")
for (r in unique(m$round)) {
sel <- as.integer(strsplit(m[m$round == r & m$method == "fedavg", "selected_clients"][1], " ")[[1]])
cids <- unique(cu[cu$round == r, "client_id"])
sel_sorted <- sort(sel)
cids_sorted <- sort(cids)
if (!identical(sel_sorted, cids_sorted)) {
cat("Mismatch at round", r, "\n")
}
}
cat("client_updates client_ids match selected_clients: OK\n")
# Check that byzantine audit client_weight sums to 1 per round over selected
a <- read.csv("outputs/byzantine_audit.csv")
for (r in unique(a$round)) {
w <- a[a$round == r & a$is_selected == 1, "client_weight"]
cat("round", r, "sum weights:", sum(w), "\n")
}
# Verify byzantine audit update_norm is 0 for non-selected
non_sel_norms <- a[a$is_selected == 0, "update_norm"]
cat("All non-selected update_norm == 0:", all(non_sel_norms == 0), "\n")
# Verify byzantine audit client_weight is 0 for non-selected
non_sel_w <- a[a$is_selected == 0, "client_weight"]
cat("All non-selected client_weight == 0:", all(non_sel_w == 0), "\n")
'client_updates client_ids match selected_clients: OK round 0 sum weights: 0.999999 round 1 sum weights: 0.999999 round 2 sum weights: 1.000002 round 3 sum weights: 1.000001 round 4 sum weights: 0.999999 round 5 sum weights: 0.999998 round 6 sum weights: 0.999999 round 7 sum weights: 0.999999 All non-selected update_norm == 0: TRUE All non-selected client_weight == 0: TRUE
cd /app && rm -rf outputs/*.csv outputs/*.json outputs/*.png FEDSIMG_SEED=123 TASK_VARIANT=hidden FEDSIMG_ALPHA=0.3 FEDSIMG_NOISE_SCALE=3.0 FEDSIMG_SIGNAL_SCALE=0.2 FEDSIMG_TRIM_K=3 FEDSIMG_BIAS_STRENGTH=0.5 FEDSIMG_OUTPUT_DIR=outputs Rscript analysis.R 2>&1 | tail -10 echo "---" cat /app/outputs/analysis_summary.json
---
{
"task_variant": "hidden",
"seed": 123,
"env": {
"FEDSIMG_ALPHA": "0.3",
"FEDSIMG_NOISE_SCALE": "3",
"FEDSIMG_SIGNAL_SCALE": "0.2",
"FEDSIMG_TRIM_K": "3",
"FEDSIMG_BIAS_STRENGTH": "0.5",
"FEDSIMG_CLIENT_BIAS_CSV": ""
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 3
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.972687,
"fedavg_accuracy": 0.954693,
"delta": 0.017994
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [1, 0, 2, 3, 3, 0, 1, 1, 3, 3, 1, 2, 2, 2, 1, 0, 1, 2, 0, 2, 2, 1, 1, 1, 1, 3, 2, 3, 1, 1, 2, 0, 5, 2, 3, 0, 1, 3, 0, 3, 2, 1, 1, 0, 2, 1, 1, 2, 0, 1, 1, 2, 2, 2, 2, 2, 4, 1, 2, 3, 4, 3, 3, 3, 1, 1, 3, 1, 2, 0, 3, 4, 0, 1, 1, 2, 1, 1, 0, 0, 2, 0, 0, 3, 1, 2, 3, 2, 0, 4, 1, 0, 1, 1, 3, 1, 2, 1, 1, 1]
}
}cd /app && Rscript -e '
# Verify JSON numeric precision matches CSV (full precision, no lossy rounding)
m <- read.csv("outputs/metrics_by_round.csv", stringsAsFactors=FALSE)
last_round <- max(m$round)
fed <- m[m$round == last_round & m$method == "fedavg", "accuracy"]
tm <- m[m$round == last_round & m$method == "trimmed_mean", "accuracy"]
# Parse JSON raw text to see exact values
j <- readLines("outputs/analysis_summary.json")
cat(grep("_accuracy|delta", j, value = TRUE), sep="\n")
cat("CSV fed (as printed):", format(fed, nsmall = 6), "\n")
cat("CSV tm (as printed):", format(tm, nsmall = 6), "\n")
cat("delta = tm - fed =", format(tm - fed, nsmall = 6), "\n")
'"trimmed_mean_accuracy": 0.972687,
"fedavg_accuracy": 0.954693,
"delta": 0.017994
CSV fed (as printed): 0.954693
CSV tm (as printed): 0.972687
delta = tm - fed = 0.017994cd /app && touch outputs/metrics_by_round.csv -d "2020-01-01" ls -la outputs/metrics_by_round.csv FEDSIMG_SEED=42 FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv TASK_VARIANT=public FEDSIMG_OUTPUT_DIR=outputs Rscript analysis.R 2>&1 | tail -5 ls -la outputs/metrics_by_round.csv
-rw-r--r-- 1 h2tagent h2tagent 1733 Jan 1 2020 outputs/metrics_by_round.csv -rw-r--r-- 1 h2tagent h2tagent 1739 May 28 23:10 outputs/metrics_by_round.csv
No file changes, the agent left the workspace untouched.
# SyncVals normalized verifier summary
# task: fedavg-federated-noniid-mnist
# attempt: 2 status: completed raw_score: 0.0
# reward rule: binary: score 0.0 != 1.0
# => reward = 0.0 (FAIL)
#
# Note: these tasks score via an external hidden grader; no pytest/verifier
# block is embedded in the agent transcript. The block below is the last
# sizable tool-output captured in the run (supporting evidence).
----------------------------------------------------------------------
---
{
"task_variant": "hidden",
"seed": 123,
"env": {
"FEDSIMG_ALPHA": "0.3",
"FEDSIMG_NOISE_SCALE": "3",
"FEDSIMG_SIGNAL_SCALE": "0.2",
"FEDSIMG_TRIM_K": "3",
"FEDSIMG_BIAS_STRENGTH": "0.5",
"FEDSIMG_CLIENT_BIAS_CSV": ""
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 3
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.972687,
"fedavg_accuracy": 0.954693,
"delta": 0.017994
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [1, 0, 2, 3, 3, 0, 1, 1, 3, 3, 1, 2, 2, 2, 1, 0, 1, 2, 0, 2, 2, 1, 1, 1, 1, 3, 2, 3, 1, 1, 2, 0, 5, 2, 3, 0, 1, 3, 0, 3, 2, 1, 1, 0, 2, 1, 1, 2, 0, 1, 1, 2, 2, 2, 2, 2, 4, 1, 2, 3, 4, 3, 3, 3, 1, 1, 3, 1, 2, 0, 3, 4, 0, 1, 1, 2, 1, 1, 0, 0, 2, 0, 0, 3, 1, 2, 3, 2, 0, 4, 1, 0, 1, 1, 3, 1, 2, 1, 1, 1]
}
}Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_f2b4372ed3004c41. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_f2b4372ed3004c41 · verifier authoritative; classifier explanatory.