tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.ls -la /app/
total 35 drwxrws--T 3 h2tagent h2tagent 4096 May 28 19:31 . drwxr-xr-x 27 root root 4096 May 28 23:06 .. -rw-rw---- 1 h2tagent h2tagent 14067 May 28 19:31 analysis.R -rw-rw---- 1 h2tagent h2tagent 7752 May 28 19:31 dataset_manifest.json -rw-rw---- 1 h2tagent h2tagent 420 May 28 19:31 example_client_bias.csv drwxrws--- 4 h2tagent h2tagent 4096 May 28 19:31 outputs
cat /app/dataset_manifest.json
{
"manifest_version": "1.0",
"dataset": {
"name": "MNIST",
"source": "torchvision.datasets.MNIST",
"download_url": "http://yann.lecun.com/exdb/mnist/",
"train_size": 60000,
"test_size": 10000,
"n_classes": 10,
"per_class_train": 6000
},
"simulation": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trimmed_mean": {
"space": "coordinate",
"trim_k_default": 2
},
"fedavg": {
"weighting": "by_num_samples"
},
"update_generation": {
"w_star": "rnorm(param_dim)",
"class_prototypes": "matrix(rnorm(n_classes * param_dim), nrow=n_classes, ncol=param_dim)",
"honest_update": "signal_scale*w_star + bias_strength*(bias_row %*% class_prototypes) + rnorm(param_dim, 0, honest_noise_sd)",
"honest_noise_sd": 0.08
}
},
"partition": {
"method": "dirichlet_per_class",
"alpha_default": 0.5,
"allocation": "largest_remainder_hamilton",
"min_client_samples": 20,
"entropy_log_base": "e"
},
"client_bias": {
"csv_schema": {
"client_id": "int",
"class_k_weight": "float (k=0..9; nonnegative; rows sum to 1)"
},
"required_columns": [
"client_id",
"class_0_weight",
"class_1_weight",
"class_2_weight",
"class_3_weight",
"class_4_weight",
"class_5_weight",
"class_6_weight",
"class_7_weight",
"class_8_weight",
"class_9_weight"
],
"bias_strength_default": 0.35,
"env_path_var": "FEDSIMG_CLIENT_BIAS_CSV",
"env_strength_var": "FEDSIMG_BIAS_STRENGTH"
},
"attack": {
"byzantine_selection": "fixed_lowest_client_ids",
"type": "sign_flip_plus_noise",
"noise_scale_default": 2.0,
"apply_timing": "before_aggregation",
"byzantine_update": "-u_clean + rnorm(param_dim, 0, noise_scale)"
},
"metrics": {
"accuracy_proxy": "sigmoid_cosine_similarity",
"accuracy_sigmoid_k_default": 7.5,
"accuracy_sigmoid_b_default": 2.5,
"round_noise_sd": 0.005
},
"rounding": {
"accuracy_decimals": 6,
"loss_decimals": 6,
"share_decimals": 6
},
"outputs": {
"metrics_csv": {
"path": "metrics_by_round.csv",
"columns": [
"round",
"method",
"accuracy",
"cosine_sim",
"update_norm",
"byzantine_selected",
"selected_clients"
],
"sort_by": [
"round",
"method"
]
},
"partition_matrix_csv": {
"path": "partition_matrix.csv",
"columns": [
"client_id",
"class_0_count",
"class_1_count",
"class_2_count",
"class_3_count",
"class_4_count",
"class_5_count",
"class_6_count",
"class_7_count",
"class_8_count",
"class_9_count"
],
"sort_by": [
"client_id"
],
"row_semantics": {
"rows": "one row per client_id for all clients",
"counts": "class_k_count is the number of training samples of class k assigned to the client"
}
},
"dirichlet_gammas_csv": {
"path": "dirichlet_gammas.csv",
"columns": [
"class",
"client_id",
"gamma_raw",
"p",
"allocated_count"
],
"sort_by": [
"class",
"client_id"
],
"row_semantics": {
"rows": "one row per (class, client_id)",
"gamma_raw": "raw Dirichlet gamma draw (positive)",
"p": "normalized probability gamma_raw/sum(gamma_raw) for that class",
"allocated_count": "Hamilton allocation of per_class_train samples for that class using p"
},
"numeric_precision": {
"gamma_raw": "full_precision",
"p": "full_precision",
"allocated_count": "integer"
}
},
"partition_csv": {
"path": "partition_summary.csv",
"columns": [
"client_id",
"is_byzantine",
"n_train",
"label_e
…[truncated]cat /app/analysis.R
#!/usr/bin/env Rscript
suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
`%||%` <- function(x, y) if (!is.null(x) && !is.na(x) && nzchar(as.character(x))) x else y
sigmoid <- function(x) 1.0 / (1.0 + exp(-x))
safe_dir_create <- function(path) {
if (!dir.exists(path)) dir.create(path, recursive = TRUE, showWarnings = FALSE)
}
read_bias_csv <- function(path, n_clients, n_classes) {
if (is.null(path) || !nzchar(path) || !file.exists(path)) {
return(NULL)
}
df <- read.csv(path, stringsAsFactors = FALSE)
req <- c("client_id", sprintf("class_%d_weight", 0:(n_classes - 1)))
if (!all(req %in% names(df))) {
stop("bias csv missing required columns")
}
df <- df[order(df$client_id), ]
if (nrow(df) != n_clients || any(df$client_id != 0:(n_clients - 1))) {
stop("bias csv must have exactly one row per client")
}
mat <- as.matrix(df[, req[-1], drop = FALSE])
storage.mode(mat) <- "double"
col_sums <- colSums(mat)
col_sums[col_sums <= 0] <- 1
sweep(mat, 2, col_sums, "/")
}
hamilton_alloc <- function(p, total) {
p <- as.numeric(p)
p <- p / sum(p)
raw <- p * total
alloc <- as.integer(round(raw))
gap <- as.integer(total - sum(alloc))
if (gap > 0) {
ord <- order(raw - alloc, decreasing = TRUE)
alloc[ord[seq_len(gap)]] <- alloc[ord[seq_len(gap)]] + 1L
} else if (gap < 0) {
ord <- order(raw - alloc, decreasing = FALSE)
take <- ord[seq_len(abs(gap))]
alloc[take] <- pmax(0L, alloc[take] - 1L)
}
as.integer(alloc)
}
repair_partition <- function(partition, min_client_samples) {
n_train <- rowSums(partition)
while (min(n_train) < min_client_samples) {
low <- which.min(n_train)
high <- which.max(n_train)
donor_class <- which.max(partition[high, ])
if (partition[high, donor_class] <= 0) break
partition[high, donor_class] <- partition[high, donor_class] - 1L
partition[low, donor_class] <- partition[low, donor_class] + 1L
n_train <- rowSums(partition)
}
partition
}
dirichlet_partition <- function(n_clients, n_classes, per_class_total, alpha, min_client_samples) {
part <- matrix(0L, nrow = n_clients, ncol = n_classes)
gammas <- matrix(0.0, nrow = n_clients, ncol = n_classes)
for (k in seq_len(n_classes)) {
g <- rgamma(n_clients, shape = alpha, rate = 1)
gammas[, k] <- g
part[, k] <- hamilton_alloc(g, per_class_total)
}
part <- repair_partition(part, min_client_samples)
list(partition = part, gammas = gammas)
}
shannon_entropy <- function(counts) {
s <- sum(counts)
if (s <= 0) return(0)
p <- counts / s
p <- p[p > 0]
-sum(p * log10(p))
}
row_trimmed_mean <- function(mat, trim_k) {
keep <- order(apply(mat, 1, function(x) sqrt(sum(x * x))))[(trim_k + 1):(nrow(mat) - trim_k)]
colMeans(mat[keep, , drop = FALSE])
}
run_sim <- function() {
task_variant <- Sys.getenv("TASK_VARIANT", "public")
seed <- as.integer(Sys.getenv("FEDSIMG_SEED", "0"))
set.seed(seed)
output_dir <- Sys.getenv("FEDSIMG_OUTPUT_DIR", "outputs")
manifest_path <- Sys.getenv("FEDSIMG_MANIFEST_PATH", "/app/dataset_manifest.json")
safe_dir_create(output_dir)
manifest <- fromJSON(manifest_path, simplifyVector = TRUE)
n_clients <- as.integer(manifest$simulation$n_clients)
n_byz <- as.integer(manifest$simulation$n_byzantine)
clients_per_round <- as.integer(manifest$simulation$clients_per_round)
n_rounds <- as.integer(manifest$simulation$n_rounds)
param_dim <- as.integer(manifest$simulation$param_dim)
n_classes <- as.integer(manifest$dataset$n_classes)
per_class <- as.integer(manifest$dataset$per_class_train)
alpha <- as.numeric(manifest$partition$alpha_default)
noise_scale <- as.numeric(manifest$attack$noise_scale_default)
signal_scale <- as.numeric(Sys.getenv("FEDSIMG_SIGNAL_SCALE", "0.12"))
trim_k <- as.integer(manifest$simulation$trimmed_mean$trim_k_default)
bias_strength <- as.numeric(manifest$client_bias$bias_strength_default)
bias_csv <- Sys.getenv(m
…[truncated]cat /app/example_client_bias.csv
client_id,class_0_weight,class_1_weight,class_2_weight,class_3_weight,class_4_weight,class_5_weight,class_6_weight,class_7_weight,class_8_weight,class_9_weight 0,0.28,0.10,0.07,0.06,0.06,0.06,0.07,0.08,0.12,0.10 1,0.05,0.06,0.09,0.14,0.15,0.14,0.12,0.10,0.08,0.07 2,0.08,0.08,0.08,0.08,0.08,0.08,0.12,0.14,0.14,0.12 3,0.04,0.05,0.06,0.07,0.10,0.13,0.15,0.15,0.14,0.11 4,0.16,0.14,0.12,0.10,0.09,0.08,0.07,0.06,0.10,0.08
ls /app/outputs/
hidden public
ls /app/outputs/public/ /app/outputs/hidden/ 2>/dev/null
/app/outputs/hidden/: /app/outputs/public/:
which Rscript && Rscript --version
/usr/bin/Rscript Rscript (R) version 4.3.3 (2024-02-29)
Rscript -e 'cat(suppressPackageStartupMessages({library(ggplot2); library(gridExtra); library(jsonlite); "ok"}))'ok
command: create
path: /app/analysis.R
file_text: #!/usr/bin/env Rscript
suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
`%||%` <- function(x, y) if (!is.null(x) && !is.na(x) && nzchar(as.character(x))) x else y
sigmoid <- function(x) 1.0 / (1.0 + exp(-x))
safe_dir_create <- function(path) {
if (!dir.exists(path)) dir.create(path, recursive = TRUE, showWarnings = FALSE)
}
env_or <- function(name, default) {
v <- Sys.getenv(name, unset = "")
if (!nzchar(v)) return(default)
v
}
read_bias_csv <- function(path, n_clients, n_classes) {
# Returns an n_clients x n_classes matrix of per-client class weights.
# Rows for client_ids present in the CSV are taken (and normalized to sum to 1);
# rows for clients not covered by the CSV are left as NA so that callers can
# fall back to an empirical default per-client.
bias_mat <- matrix(NA_real_, nrow = n_clients, ncol = n_classes)
if (is.null(path) || !nzchar(path) || !file.exists(path)) {
return(bias_mat)
}
df <- read.csv(path, stringsAsFactors = FALSE)
req <- c("client_id", sprintf("class_%d_weight", 0:(n_classes - 1)))
if (!all(req %in% names(df))) {
stop("bias csv missing required columns")
}
for (i in seq_len(nrow(df))) {
cid <- as.integer(df$client_id[i])
if (is.na(cid) || cid < 0 || cid >= n_clients) next
row <- as.numeric(df[i, req[-1]])
row[!is.finite(row) | row < 0] <- 0
s <- sum(row)
if (s > 0) {
row <- row / s
} else {
row <- rep(1 / n_classes, n_classes)
}
bias_mat[cid + 1, ] <- row
}
bias_mat
}
hamilton_alloc <- function(p, total) {
p <- as.numeric(p)
s <- sum(p)
if (s <= 0) {
p <- rep(1 / length(p), length(p))
} else {
p <- p / s
}
raw <- p * total
floor_alloc <- floor(raw)
remainder <- raw - floor_alloc
alloc <- as.integer(floor_alloc)
gap <- as.integer(total - sum(alloc))
if (gap > 0) {
# assign remaining units to the largest fractional remainders first
ord <- order(remainder, seq_along(remainder), decreasing = c(TRUE, FALSE),
method = "radix")
alloc[ord[seq_len(gap)]] <- alloc[ord[seq_len(gap)]] + 1L
} else if (gap < 0) {
ord <- order(remainder, seq_along(remainder), decreasing = c(FALSE, TRUE),
method = "radix")
take <- ord[seq_len(abs(gap))]
alloc[take] <- pmax(0L, alloc[take] - 1L)
}
as.integer(alloc)
}
repair_partition <- function(partition, min_client_samples) {
# Guarantees each client has at least min_client_samples by taking one
# sample from the richest client's top class and giving it to the
# neediest client. Preserves per-class totals.
n_train <- rowSums(partition)
while (min(n_train) < min_client_samples) {
low <- which.min(n_train)
high <- which.max(n_train)
donor_class <- which.max(partition[high, ])
if (partition[high, donor_class] <= 0) break
partition[high, donor_class] <- partition[high, donor_class] - 1L
partition[low, donor_class] <- partition[low, donor_class] + 1L
n_train <- rowSums(partition)
}
partition
}
dirichlet_partition <- function(n_clients, n_classes, per_class_total, alpha, min_client_samples) {
part <- matrix(0L, nrow = n_clients, ncol = n_classes)
gammas <- matrix(0.0, nrow = n_clients, ncol = n_classes)
for (k in seq_len(n_classes)) {
g <- rgamma(n_clients, shape = alpha, rate = 1)
gammas[, k] <- g
part[, k] <- hamilton_alloc(g, per_class_total)
}
part <- repair_partition(part, min_client_samples)
list(partition = part, gammas = gammas)
}
shannon_entropy <- function(counts) {
s <- sum(counts)
if (s <= 0) return(0)
p <- counts / s
p <- p[p > 0]
-sum(p * log(p)) # natural log per manifest (entropy_log_base "e")
}
coord_trimmed_mean <- function(mat, trim_k) {
# Coordinate-wise trimmed mean: for each coordinate, sort values across
# clients and drop trim_k smallest and trim_k largest before averaging.
apply(mat, 2, functi
…[truncated]Existing file was successfully overwritten at /app/analysis.R
cd /app && rm -rf outputs_test && mkdir -p outputs_test && \ TASK_VARIANT=public FEDSIMG_SEED=42 FEDSIMG_OUTPUT_DIR=outputs_test \ FEDSIMG_ALPHA=0.5 FEDSIMG_NOISE_SCALE=2.0 FEDSIMG_SIGNAL_SCALE=0.12 \ FEDSIMG_TRIM_K=2 FEDSIMG_BIAS_STRENGTH=0.35 \ FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv \ Rscript analysis.R 2>&1 | tail -30
<system>Tool ran without output or errors</system>
ls -la /app/outputs_test/
total 1952 drwxr-sr-x 2 h2tagent h2tagent 4096 May 28 23:09 . drwxrws--T 4 h2tagent h2tagent 4096 May 28 23:09 .. -rw-r--r-- 1 h2tagent h2tagent 1016 May 28 23:09 analysis_summary.json -rw-r--r-- 1 h2tagent h2tagent 12612 May 28 23:09 byzantine_audit.csv -rw-r--r-- 1 h2tagent h2tagent 731196 May 28 23:09 client_updates.csv -rw-r--r-- 1 h2tagent h2tagent 1115104 May 28 23:09 client_updates_clean.csv -rw-r--r-- 1 h2tagent h2tagent 74639 May 28 23:09 diagnostics.png -rw-r--r-- 1 h2tagent h2tagent 45882 May 28 23:09 dirichlet_gammas.csv -rw-r--r-- 1 h2tagent h2tagent 1737 May 28 23:09 metrics_by_round.csv -rw-r--r-- 1 h2tagent h2tagent 3363 May 28 23:09 partition_matrix.csv -rw-r--r-- 1 h2tagent h2tagent 2952 May 28 23:09 partition_summary.csv
cat /app/outputs_test/metrics_by_round.csv
"round","method","accuracy","cosine_sim","update_norm","byzantine_selected","selected_clients" 0,"fedavg",0.701539,0.44856,4.016918,1,"80 90 62 55 94 67 31 56 60 65 10 19 40 14 61 57 7 68 75 51" 0,"trimmed_mean",0.946524,0.727073,2.572666,1,"80 90 62 55 94 67 31 56 60 65 10 19 40 14 61 57 7 68 75 51" 1,"fedavg",0.772213,0.496612,3.54306,2,"90 32 13 64 47 53 95 42 35 72 6 1 30 78 74 55 19 36 15 38" 1,"trimmed_mean",0.945395,0.72637,2.607558,2,"90 32 13 64 47 53 95 42 35 72 6 1 30 78 74 55 19 36 15 38" 2,"fedavg",0.854487,0.574747,3.056391,1,"96 3 64 33 76 50 18 75 11 17 80 40 92 25 47 21 60 63 14 42" 2,"trimmed_mean",0.950662,0.725882,2.678662,1,"96 3 64 33 76 50 18 75 11 17 80 40 92 25 47 21 60 63 14 42" 3,"fedavg",0.89949,0.614974,3.138203,2,"19 9 48 31 24 54 0 58 93 34 70 76 89 14 55 59 88 97 36 41" 3,"trimmed_mean",0.957684,0.73116,2.564068,2,"19 9 48 31 24 54 0 58 93 34 70 76 89 14 55 59 88 97 36 41" 4,"fedavg",0.916335,0.648344,3.191474,2,"98 10 77 25 41 80 35 44 86 59 32 46 4 48 38 31 5 56 92 37" 4,"trimmed_mean",0.947596,0.733307,2.55824,2,"98 10 77 25 41 80 35 44 86 59 32 46 4 48 38 31 5 56 92 37" 5,"fedavg",0.931644,0.673158,3.361221,3,"66 21 54 20 62 9 12 36 45 39 69 95 5 88 96 0 90 46 70 71" 5,"trimmed_mean",0.952317,0.738967,2.641233,3,"66 21 54 20 62 9 12 36 45 39 69 95 5 88 96 0 90 46 70 71" 6,"fedavg",0.937543,0.684333,3.40599,2,"12 64 44 3 60 43 39 82 49 83 50 66 75 86 5 21 13 41 95 15" 6,"trimmed_mean",0.955575,0.739989,2.595111,2,"12 64 44 3 60 43 39 82 49 83 50 66 75 86 5 21 13 41 95 15" 7,"fedavg",0.92827,0.677745,3.689732,2,"35 59 11 2 61 96 33 51 63 49 75 38 81 12 24 37 94 48 3 89" 7,"trimmed_mean",0.960252,0.739615,2.507281,2,"35 59 11 2 61 96 33 51 63 49 75 38 81 12 24 37 94 48 3 89"
cat /app/outputs_test/analysis_summary.json
{
"task_variant": "public",
"seed": 42,
"env": {
"FEDSIMG_ALPHA": 0.5,
"FEDSIMG_NOISE_SCALE": 2,
"FEDSIMG_SIGNAL_SCALE": 0.12,
"FEDSIMG_TRIM_K": 2,
"FEDSIMG_BIAS_STRENGTH": 0.35,
"FEDSIMG_CLIENT_BIAS_CSV": "/app/example_client_bias.csv"
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 2
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.960252,
"fedavg_accuracy": 0.92827,
"delta": 0.031982
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [2, 1, 1, 3, 1, 3, 1, 1, 0, 2, 2, 2, 3, 2, 3, 2, 0, 1, 1, 3, 1, 3, 0, 0, 2, 2, 0, 0, 0, 0, 1, 3, 2, 2, 1, 3, 3, 2, 3, 2, 2, 3, 2, 1, 2, 1, 2, 2, 3, 2, 2, 2, 0, 1, 2, 3, 2, 1, 1, 3, 3, 2, 2, 2, 3, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, 4, 2, 1, 1, 0, 3, 1, 1, 1, 0, 0, 2, 0, 2, 2, 3, 0, 2, 1, 2, 3, 3, 1, 1, 0]
}
}head -5 /app/outputs_test/partition_summary.csv echo "---" head -5 /app/outputs_test/byzantine_audit.csv echo "---" head -5 /app/outputs_test/partition_matrix.csv echo "---" head -5 /app/outputs_test/dirichlet_gammas.csv echo "---" head -3 /app/outputs_test/client_updates.csv echo "---" head -3 /app/outputs_test/client_updates_clean.csv
"client_id","is_byzantine","n_train","label_entropy","top_label","top_label_share" 0,1,708,1.792905,7,0.323446 1,1,756,1.339652,6,0.411376 2,1,759,1.427729,1,0.466403 3,1,640,1.563247,3,0.335938 --- "round","client_id","is_byzantine","is_selected","client_weight","update_norm" 0,0,1,0,0,0 0,1,1,0,0,0 0,2,1,0,0,0 0,3,1,0,0,0 --- "client_id","class_0_count","class_1_count","class_2_count","class_3_count","class_4_count","class_5_count","class_6_count","class_7_count","class_8_count","class_9_count" 0,99,20,180,32,26,72,46,229,3,1 1,38,2,29,0,5,3,311,291,71,6 2,243,354,1,35,11,20,53,21,16,5 3,193,2,8,215,49,130,11,1,28,3 --- "class","client_id","gamma_raw","p","allocated_count" 0,0,0.760516834810734,0.0165288490248016,99 0,1,0.293697522769724,0.00638313550814045,38 0,2,1.86634352361331,0.0405625607721065,243 0,3,1.48173928793115,0.0322036855244973,193 --- "round","client_id","coord","update_value" 0,7,0,-1.571876 0,7,1,0.292317 --- "round","client_id","coord","u_clean","u_submitted" 0,7,0,0.156615,-1.571876 0,7,1,0.261323,0.292317
cd /app && Rscript -e '
pm <- read.csv("outputs_test/partition_matrix.csv")
# Row sums should all be >= 20 (min_client_samples)
rsum <- rowSums(pm[, -1])
cat("Min n_train:", min(rsum), " Max:", max(rsum), "\n")
# Col sums should equal 6000 each (per_class_train)
csum <- colSums(pm[, -1])
cat("Col sums:", csum, "\n")
# Dirichlet gammas p per class sums to 1; allocated_count per class sums to 6000
dg <- read.csv("outputs_test/dirichlet_gammas.csv")
for (k in 0:9) {
sub <- dg[dg$class == k, ]
cat("Class", k, ": sum(p)=", sum(sub$p), "sum(alloc)=", sum(sub$allocated_count), "\n")
}
# Byzantine audit: sum of client_weight among selected per round
ba <- read.csv("outputs_test/byzantine_audit.csv")
for (r in 0:7) {
w <- ba$client_weight[ba$round == r & ba$is_selected == 1]
cat("Round", r, "num_selected=", length(w), "sum(weight)=", sum(w), "\n")
}
# Byzantine update_norm should be > 0 only for selected
sel_norms <- ba$update_norm[ba$is_selected == 1]
unsel_norms <- ba$update_norm[ba$is_selected == 0]
cat("Min selected norm:", min(sel_norms), "Max unselected:", max(unsel_norms), "\n")
'
Min n_train: 125 Max: 1199 Col sums: 6000 6000 6000 6000 6000 6000 6000 6000 6000 6000 Class 0 : sum(p)= 1 sum(alloc)= 6000 Class 1 : sum(p)= 1 sum(alloc)= 6000 Class 2 : sum(p)= 1 sum(alloc)= 6000 Class 3 : sum(p)= 1 sum(alloc)= 6000 Class 4 : sum(p)= 1 sum(alloc)= 6000 Class 5 : sum(p)= 1 sum(alloc)= 6000 Class 6 : sum(p)= 1 sum(alloc)= 6000 Class 7 : sum(p)= 1 sum(alloc)= 6000 Class 8 : sum(p)= 1 sum(alloc)= 6000 Class 9 : sum(p)= 1 sum(alloc)= 6000 Round 0 num_selected= 20 sum(weight)= 0.999999 Round 1 num_selected= 20 sum(weight)= 0.999999 Round 2 num_selected= 20 sum(weight)= 1.000002 Round 3 num_selected= 20 sum(weight)= 1.000001 Round 4 num_selected= 20 sum(weight)= 0.999999 Round 5 num_selected= 20 sum(weight)= 0.999998 Round 6 num_selected= 20 sum(weight)= 0.999999 Round 7 num_selected= 20 sum(weight)= 0.999999 Min selected norm: 3.002057 Max unselected: 0
cd /app && Rscript -e '
# Check that selection_histogram in summary matches what the audit says.
library(jsonlite)
s <- fromJSON("outputs_test/analysis_summary.json")
ba <- read.csv("outputs_test/byzantine_audit.csv")
sel_count <- as.integer(tapply(ba$is_selected, ba$client_id, sum))
# Compare
cat("summary sel_hist sum:", sum(s$provenance$selection_histogram), "\n")
cat("audit sel sum:", sum(ba$is_selected), "\n")
cat("clients_per_round * n_rounds:", 20*8, "\n")
cat("Equal?", all(s$provenance$selection_histogram == sel_count), "\n")
cat("Length sel_hist:", length(s$provenance$selection_histogram), "\n")
# Verify final metric values match CSV
mc <- read.csv("outputs_test/metrics_by_round.csv")
final_r <- max(mc$round)
tm_val <- mc$accuracy[mc$method=="trimmed_mean" & mc$round == final_r]
fed_val <- mc$accuracy[mc$method=="fedavg" & mc$round == final_r]
cat("CSV final tm:", tm_val, "JSON:", s$final_metrics$trimmed_mean_accuracy, "\n")
cat("CSV final fed:", fed_val, "JSON:", s$final_metrics$fedavg_accuracy, "\n")
cat("JSON delta:", s$final_metrics$delta, "computed:", tm_val - fed_val, "\n")
'
summary sel_hist sum: 160 audit sel sum: 160 clients_per_round * n_rounds: 160 Equal? TRUE Length sel_hist: 100 CSV final tm: 0.960252 JSON: 0.960252 CSV final fed: 0.92827 JSON: 0.92827 JSON delta: 0.031982 computed: 0.031982
cd /app && Rscript -e '
# Ensure client_updates.csv client_id matches selected_clients per round
mc <- read.csv("outputs_test/metrics_by_round.csv")
cu <- read.csv("outputs_test/client_updates.csv")
for (r in unique(mc$round)) {
sel <- sort(as.integer(strsplit(trimws(mc$selected_clients[mc$method=="fedavg" & mc$round==r]), "\\s+")[[1]]))
actual <- sort(unique(cu$client_id[cu$round==r]))
if (!identical(sel, actual)) {
cat("MISMATCH at round", r, "\n")
}
}
cat("All rounds consistent\n")
# Ensure that client_updates.csv and client_updates_clean.csv are the same rows
cc <- read.csv("outputs_test/client_updates_clean.csv")
cat("cu nrows:", nrow(cu), "cc nrows:", nrow(cc), "\n")
cat("Coords per client per round:", nrow(cu) / (8*20), "\n") # should be 256
# Check u_submitted in clean matches update_value in updates
cat("update_value ~ u_submitted agree?:",
all(abs(cu$update_value - cc$u_submitted) < 1e-6), "\n")
# Check byzantine attack: for byz client, u_submitted should differ from u_clean
cat("Byz sample: cid=0 at round 0:\n")
byz_rows <- cc[cc$client_id == 7 & cc$round == 0, ]
cat("head client 7 (honest), diff:", max(abs(byz_rows$u_clean - byz_rows$u_submitted)), "\n")
# any byz selected this round?
sel_r0 <- sort(as.integer(strsplit(trimws(mc$selected_clients[mc$method=="fedavg" & mc$round==0]), "\\s+")[[1]]))
cat("selected round 0:", sel_r0, "\n")
byz_sel <- intersect(sel_r0, 0:9)
cat("byz in round 0:", byz_sel, "\n")
if (length(byz_sel) > 0) {
br <- cc[cc$client_id == byz_sel[1] & cc$round == 0, ]
cat("For byz client", byz_sel[1], "max abs(u_clean - u_submitted):", max(abs(br$u_clean - br$u_submitted)), "\n")
cat("head:\n")
print(head(br))
}
'
All rounds consistent cu nrows: 40960 cc nrows: 40960 Coords per client per round: 256 update_value ~ u_submitted agree?: TRUE Byz sample: cid=0 at round 0: head client 7 (honest), diff: 6.753126 selected round 0: 7 10 14 19 31 40 51 55 56 57 60 61 62 65 67 68 75 80 90 94 byz in round 0: 7 For byz client 7 max abs(u_clean - u_submitted): 6.753126 head: round client_id coord u_clean u_submitted 1 0 7 0 0.156615 -1.571876 2 0 7 1 0.261323 0.292317 3 0 7 2 -0.181832 -0.302370 4 0 7 3 0.312278 -1.816215 5 0 7 4 0.079616 -2.732872 6 0 7 5 0.169914 -0.097772
cd /app && Rscript -e '
cc <- read.csv("outputs_test/client_updates_clean.csv")
# Honest client: 80 is selected in round 0 and NOT in 0:9 (byz range)
br <- cc[cc$client_id == 80 & cc$round == 0, ]
cat("For honest client 80 max abs(u_clean - u_submitted):", max(abs(br$u_clean - br$u_submitted)), "\n")
cat("head:\n")
print(head(br))
'
For honest client 80 max abs(u_clean - u_submitted): 0
head:
round client_id coord u_clean u_submitted
4353 0 80 0 0.047176 0.047176
4354 0 80 1 0.238698 0.238698
4355 0 80 2 -0.332335 -0.332335
4356 0 80 3 0.017771 0.017771
4357 0 80 4 0.152773 0.152773
4358 0 80 5 0.065671 0.065671cd /app && rm -rf outputs_test2 && mkdir -p outputs_test2 && \ TASK_VARIANT=hidden FEDSIMG_SEED=7 FEDSIMG_OUTPUT_DIR=outputs_test2 \ FEDSIMG_ALPHA=0.8 FEDSIMG_NOISE_SCALE=1.5 FEDSIMG_SIGNAL_SCALE=0.1 \ FEDSIMG_TRIM_K=3 FEDSIMG_BIAS_STRENGTH=0.5 \ Rscript analysis.R 2>&1 | tail -20 && ls outputs_test2/
analysis_summary.json byzantine_audit.csv client_updates.csv client_updates_clean.csv diagnostics.png dirichlet_gammas.csv metrics_by_round.csv partition_matrix.csv partition_summary.csv
cat /app/outputs_test2/analysis_summary.json
{
"task_variant": "hidden",
"seed": 7,
"env": {
"FEDSIMG_ALPHA": 0.8,
"FEDSIMG_NOISE_SCALE": 1.5,
"FEDSIMG_SIGNAL_SCALE": 0.1,
"FEDSIMG_TRIM_K": 3,
"FEDSIMG_BIAS_STRENGTH": 0.5,
"FEDSIMG_CLIENT_BIAS_CSV": ""
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 3
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.870042,
"fedavg_accuracy": 0.843212,
"delta": 0.02683
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [0, 3, 1, 2, 4, 1, 2, 4, 2, 3, 0, 1, 1, 2, 1, 1, 2, 1, 1, 0, 2, 0, 3, 4, 2, 2, 1, 0, 3, 1, 3, 4, 0, 1, 0, 3, 2, 0, 2, 0, 1, 2, 3, 0, 0, 1, 3, 4, 3, 2, 1, 3, 2, 1, 4, 1, 0, 1, 2, 1, 1, 0, 1, 5, 1, 2, 1, 1, 3, 1, 3, 1, 1, 4, 1, 0, 1, 0, 3, 2, 1, 3, 1, 1, 0, 2, 0, 2, 2, 1, 3, 1, 0, 5, 1, 3, 1, 0, 1, 0]
}
}cd /app && Rscript -e '
# Run-consistency check
library(jsonlite)
s <- fromJSON("outputs_test2/analysis_summary.json")
ba <- read.csv("outputs_test2/byzantine_audit.csv")
# Verify sel_hist
sel_count <- as.integer(tapply(ba$is_selected, ba$client_id, sum))
cat("Matches audit?", all(s$provenance$selection_histogram == sel_count), "\n")
# Verify no bias CSV env output path provenance
cat("Env keys:", names(s$env), "\n")
# partition_matrix sums
pm <- read.csv("outputs_test2/partition_matrix.csv")
rsum <- rowSums(pm[, -1])
cat("min n_train:", min(rsum), "\n")
csum <- colSums(pm[, -1])
cat("col sums (all 6000?):", all(csum == 6000), "\n")
'
Matches audit? TRUE Env keys: FEDSIMG_ALPHA FEDSIMG_NOISE_SCALE FEDSIMG_SIGNAL_SCALE FEDSIMG_TRIM_K FEDSIMG_BIAS_STRENGTH FEDSIMG_CLIENT_BIAS_CSV min n_train: 149 col sums (all 6000?): TRUE
cd /app && rm -rf outputs_t3 && mkdir -p outputs_t3 && \ FEDSIMG_SEED=0 FEDSIMG_OUTPUT_DIR=outputs_t3 \ Rscript analysis.R 2>&1 | tail -10 && ls outputs_t3/ 2>/dev/null
analysis_summary.json byzantine_audit.csv client_updates.csv client_updates_clean.csv diagnostics.png dirichlet_gammas.csv metrics_by_round.csv partition_matrix.csv partition_summary.csv
cd /app && cat outputs_t3/analysis_summary.json | head -15
{
"task_variant": "public",
"seed": 0,
"env": {
"FEDSIMG_ALPHA": 0.5,
"FEDSIMG_NOISE_SCALE": 2,
"FEDSIMG_SIGNAL_SCALE": 0.12,
"FEDSIMG_TRIM_K": 2,
"FEDSIMG_BIAS_STRENGTH": 0.35,
"FEDSIMG_CLIENT_BIAS_CSV": ""
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,cd /app && Rscript -e '
library(jsonlite)
# Take the raw strings to see they match exactly
s_text <- readLines("outputs_test/analysis_summary.json")
m_text <- readLines("outputs_test/metrics_by_round.csv")
# Extract values
cat("metrics_csv tm final line:\n")
final_tm_line <- tail(m_text[grepl("trimmed_mean", m_text)], 1)
cat(final_tm_line, "\n")
cat("JSON tm value:\n")
cat(s_text[grep("trimmed_mean_accuracy", s_text)], "\n")
cat("metrics_csv fed final line:\n")
final_fed_line <- tail(m_text[grepl("\"fedavg\"", m_text)], 1)
cat(final_fed_line, "\n")
cat("JSON fed value:\n")
cat(s_text[grep("fedavg_accuracy", s_text)], "\n")
'
metrics_csv tm final line:
7,"trimmed_mean",0.960252,0.739615,2.507281,2,"35 59 11 2 61 96 33 51 63 49 75 38 81 12 24 37 94 48 3 89"
JSON tm value:
"trimmed_mean_accuracy": 0.960252,
metrics_csv fed final line:
7,"fedavg",0.92827,0.677745,3.689732,2,"35 59 11 2 61 96 33 51 63 49 75 38 81 12 24 37 94 48 3 89"
JSON fed value:
"fedavg_accuracy": 0.92827,cd /app && Rscript -e '
library(jsonlite)
s <- fromJSON("outputs_test/analysis_summary.json")
# exact values
cat(sprintf("tm=%.17f fed=%.17f delta=%.17f diff=%.17f\n",
s$final_metrics$trimmed_mean_accuracy,
s$final_metrics$fedavg_accuracy,
s$final_metrics$delta,
s$final_metrics$trimmed_mean_accuracy - s$final_metrics$fedavg_accuracy))
'
tm=0.96025199999999999 fed=0.92827000000000004 delta=0.03198200000000000 diff=0.03198199999999995
grep -E "accuracy|delta" /app/outputs_test/analysis_summary.json
"trimmed_mean_accuracy": 0.960252,
"fedavg_accuracy": 0.92827,
"delta": 0.031982cd /app && Rscript -e '
x <- 0.960252 - 0.92827
cat(sprintf("%.17f\n", x))
# Check if x equals 0.031982 in R terms
cat("is it 0.031982?", identical(x, 0.031982), "\n")
print(x == 0.031982)
'
0.03198199999999995 is it 0.031982? FALSE [1] FALSE
cd /app && Rscript -e '
cat(sprintf("%.20f\n", 0.031982))
cat(sprintf("%.20f\n", 0.960252 - 0.92827))
'
0.03198199999999999654 0.03198199999999995491
cd /app && Rscript -e '
library(jsonlite)
x <- 0.960252 - 0.92827
cat("val:", sprintf("%.20f", x), "\n")
cat("json digits=NA:\n")
cat(toJSON(x, auto_unbox=TRUE, digits=NA), "\n")
cat("json digits=15:\n")
cat(toJSON(x, auto_unbox=TRUE, digits=15), "\n")
cat("json default:\n")
cat(toJSON(x, auto_unbox=TRUE), "\n")
'
val: 0.03198199999999995491 json digits=NA: 0.031982 json digits=15: 0.03198199999999995 json default: 0.032
cd /app && Rscript -e '
# Check if scientific notation appears in written CSVs
cu <- readLines("outputs_test/client_updates.csv", n=100)
cat("Sci notation in client_updates.csv?", any(grepl("e[+-]", cu, ignore.case=TRUE)), "\n")
# Find any scientific
sci_lines <- grep("e[+-]", cu, ignore.case=TRUE, value=TRUE)
if (length(sci_lines) > 0) print(head(sci_lines, 5))
cu <- readLines("outputs_test/client_updates.csv")
sci_all <- grep("e[+-]", cu, ignore.case=TRUE, value=TRUE)
cat("Total sci lines:", length(sci_all), "\n")
if (length(sci_all) > 0) print(head(sci_all, 3))
# Check dirichlet_gammas too (gamma_raw is full precision)
dg <- readLines("outputs_test/dirichlet_gammas.csv", n=10)
print(dg)
'
Sci notation in client_updates.csv? FALSE Total sci lines: 16 [1] "0,10,206,-7.3e-05" "0,51,81,7e-04" "0,62,245,8.7e-05" [1] "\"class\",\"client_id\",\"gamma_raw\",\"p\",\"allocated_count\"" [2] "0,0,0.760516834810734,0.0165288490248016,99" [3] "0,1,0.293697522769724,0.00638313550814045,38" [4] "0,2,1.86634352361331,0.0405625607721065,243" [5] "0,3,1.48173928793115,0.0322036855244973,193" [6] "0,4,0.213423103156904,0.00463847490156219,28" [7] "0,5,0.21113732015893,0.00458879636671902,28" [8] "0,6,2.18518396319137e-05,4.74921450331816e-07,0" [9] "0,7,7.53977505366017e-05,1.63867251635434e-06,0" [10] "0,8,1.50888420258649,0.0327936451093378,197"
command: str_replace
path: /app/analysis.R
old_str: suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
new_str: suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
# Keep numeric CSV output in plain decimal form (no scientific notation)
options(scipen = 999)The file /app/analysis.R has been edited successfully.
cd /app && rm -rf outputs_test && mkdir -p outputs_test && \ TASK_VARIANT=public FEDSIMG_SEED=42 FEDSIMG_OUTPUT_DIR=outputs_test \ FEDSIMG_ALPHA=0.5 FEDSIMG_NOISE_SCALE=2.0 FEDSIMG_SIGNAL_SCALE=0.12 \ FEDSIMG_TRIM_K=2 FEDSIMG_BIAS_STRENGTH=0.35 \ FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv \ Rscript analysis.R 2>&1 | tail -5 && \ head -3 outputs_test/dirichlet_gammas.csv && echo --- && \ head -3 outputs_test/client_updates.csv && echo --- && \ awk -F, 'NR==1 || $4 ~ /e/ || $4 ~ /E/' outputs_test/client_updates.csv | head -5
"class","client_id","gamma_raw","p","allocated_count" 0,0,0.760516834810734,0.0165288490248016,99 0,1,0.293697522769724,0.00638313550814045,38 --- "round","client_id","coord","update_value" 0,7,0,-1.571876 0,7,1,0.292317 --- "round","client_id","coord","update_value"
cd /app && awk -F, 'NR>1 && ($3 ~ /e/ || $3 ~ /E/ || $4 ~ /e/ || $4 ~ /E/)' outputs_test/dirichlet_gammas.csv | head -3
echo "---"
# Verify reading each CSV does not fail
cd /app && Rscript -e '
for (f in c("metrics_by_round.csv","dirichlet_gammas.csv","partition_matrix.csv","partition_summary.csv","byzantine_audit.csv","client_updates.csv","client_updates_clean.csv")) {
df <- read.csv(file.path("outputs_test", f))
cat(f, "rows=", nrow(df), "cols=", ncol(df), "\n")
}'--- metrics_by_round.csv rows= 16 cols= 7 dirichlet_gammas.csv rows= 1000 cols= 5 partition_matrix.csv rows= 100 cols= 11 partition_summary.csv rows= 100 cols= 6 byzantine_audit.csv rows= 800 cols= 6 client_updates.csv rows= 40960 cols= 4 client_updates_clean.csv rows= 40960 cols= 5
cat /app/outputs_test/analysis_summary.json | head -30
{
"task_variant": "public",
"seed": 42,
"env": {
"FEDSIMG_ALPHA": 0.5,
"FEDSIMG_NOISE_SCALE": 2,
"FEDSIMG_SIGNAL_SCALE": 0.12,
"FEDSIMG_TRIM_K": 2,
"FEDSIMG_BIAS_STRENGTH": 0.35,
"FEDSIMG_CLIENT_BIAS_CSV": "/app/example_client_bias.csv"
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 2
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.960252,
"fedavg_accuracy": 0.92827,
"delta": 0.031982
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [2, 1, 1, 3, 1, 3, 1, 1, 0, 2, 2, 2, 3, 2, 3, 2, 0, 1, 1, 3, 1, 3, 0, 0, 2, 2, 0, 0, 0, 0, 1, 3, 2, 2, 1, 3, 3, 2, 3, 2, 2, 3, 2, 1, 2, 1, 2, 2, 3, 2, 2, 2, 0, 1, 2, 3, 2, 1, 1, 3, 3, 2, 2, 2, 3, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, 4, 2, 1, 1, 0, 3, 1, 1, 1, 0, 0, 2, 0, 2, 2, 3, 0, 2, 1, 2, 3, 3, 1, 1, 0]
}
}cd /app && rm -rf outputs_test outputs_test2 outputs_t3 # Final sanity: ensure the default env with no overrides also works rm -rf outputs Rscript analysis.R 2>&1 | tail -5 ls outputs/ echo "---" head -3 outputs/metrics_by_round.csv
analysis_summary.json byzantine_audit.csv client_updates.csv client_updates_clean.csv diagnostics.png dirichlet_gammas.csv metrics_by_round.csv partition_matrix.csv partition_summary.csv --- "round","method","accuracy","cosine_sim","update_norm","byzantine_selected","selected_clients" 0,"fedavg",0.175285,0.125977,4.946046,4,"79 48 73 32 72 39 50 67 86 20 56 97 83 54 34 7 6 4 1 82" 0,"trimmed_mean",0.931524,0.681359,2.665407,4,"79 48 73 32 72 39 50 67 86 20 56 97 83 54 34 7 6 4 1 82"
cd /app && rm -rf outputs && mkdir -p outputs ls -la /app/
total 39 drwxrws--T 3 h2tagent h2tagent 4096 May 28 23:12 . drwxr-xr-x 27 root root 4096 May 28 23:06 .. -rw-rw---- 1 h2tagent h2tagent 18583 May 28 23:11 analysis.R -rw-rw---- 1 h2tagent h2tagent 7752 May 28 19:31 dataset_manifest.json -rw-rw---- 1 h2tagent h2tagent 420 May 28 19:31 example_client_bias.csv drwxr-sr-x 2 h2tagent h2tagent 4096 May 28 23:12 outputs
cat /app/analysis.R | wc -l
488
No file changes, the agent left the workspace untouched.
# SyncVals normalized verifier summary # task: fedavg-federated-noniid-mnist # attempt: 7 status: completed raw_score: 0.0 # reward rule: binary: score 0.0 != 1.0 # => reward = 0.0 (FAIL) # # Note: these tasks score via an external hidden grader; no pytest/verifier # block is embedded in the agent transcript. The block below is the last # sizable tool-output captured in the run (supporting evidence). ---------------------------------------------------------------------- total 39 drwxrws--T 3 h2tagent h2tagent 4096 May 28 23:12 . drwxr-xr-x 27 root root 4096 May 28 23:06 .. -rw-rw---- 1 h2tagent h2tagent 18583 May 28 23:11 analysis.R -rw-rw---- 1 h2tagent h2tagent 7752 May 28 19:31 dataset_manifest.json -rw-rw---- 1 h2tagent h2tagent 420 May 28 19:31 example_client_bias.csv drwxr-sr-x 2 h2tagent h2tagent 4096 May 28 23:12 outputs
Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_0117515a305843ac. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_0117515a305843ac · verifier authoritative; classifier explanatory.