tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.ls -la /app/
total 35 drwxrws--T 3 h2tagent h2tagent 4096 May 28 19:31 . drwxr-xr-x 27 root root 4096 May 28 23:06 .. -rw-rw---- 1 h2tagent h2tagent 14067 May 28 19:31 analysis.R -rw-rw---- 1 h2tagent h2tagent 7752 May 28 19:31 dataset_manifest.json -rw-rw---- 1 h2tagent h2tagent 420 May 28 19:31 example_client_bias.csv drwxrws--- 4 h2tagent h2tagent 4096 May 28 19:31 outputs
cat /app/dataset_manifest.json
{
"manifest_version": "1.0",
"dataset": {
"name": "MNIST",
"source": "torchvision.datasets.MNIST",
"download_url": "http://yann.lecun.com/exdb/mnist/",
"train_size": 60000,
"test_size": 10000,
"n_classes": 10,
"per_class_train": 6000
},
"simulation": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trimmed_mean": {
"space": "coordinate",
"trim_k_default": 2
},
"fedavg": {
"weighting": "by_num_samples"
},
"update_generation": {
"w_star": "rnorm(param_dim)",
"class_prototypes": "matrix(rnorm(n_classes * param_dim), nrow=n_classes, ncol=param_dim)",
"honest_update": "signal_scale*w_star + bias_strength*(bias_row %*% class_prototypes) + rnorm(param_dim, 0, honest_noise_sd)",
"honest_noise_sd": 0.08
}
},
"partition": {
"method": "dirichlet_per_class",
"alpha_default": 0.5,
"allocation": "largest_remainder_hamilton",
"min_client_samples": 20,
"entropy_log_base": "e"
},
"client_bias": {
"csv_schema": {
"client_id": "int",
"class_k_weight": "float (k=0..9; nonnegative; rows sum to 1)"
},
"required_columns": [
"client_id",
"class_0_weight",
"class_1_weight",
"class_2_weight",
"class_3_weight",
"class_4_weight",
"class_5_weight",
"class_6_weight",
"class_7_weight",
"class_8_weight",
"class_9_weight"
],
"bias_strength_default": 0.35,
"env_path_var": "FEDSIMG_CLIENT_BIAS_CSV",
"env_strength_var": "FEDSIMG_BIAS_STRENGTH"
},
"attack": {
"byzantine_selection": "fixed_lowest_client_ids",
"type": "sign_flip_plus_noise",
"noise_scale_default": 2.0,
"apply_timing": "before_aggregation",
"byzantine_update": "-u_clean + rnorm(param_dim, 0, noise_scale)"
},
"metrics": {
"accuracy_proxy": "sigmoid_cosine_similarity",
"accuracy_sigmoid_k_default": 7.5,
"accuracy_sigmoid_b_default": 2.5,
"round_noise_sd": 0.005
},
"rounding": {
"accuracy_decimals": 6,
"loss_decimals": 6,
"share_decimals": 6
},
"outputs": {
"metrics_csv": {
"path": "metrics_by_round.csv",
"columns": [
"round",
"method",
"accuracy",
"cosine_sim",
"update_norm",
"byzantine_selected",
"selected_clients"
],
"sort_by": [
"round",
"method"
]
},
"partition_matrix_csv": {
"path": "partition_matrix.csv",
"columns": [
"client_id",
"class_0_count",
"class_1_count",
"class_2_count",
"class_3_count",
"class_4_count",
"class_5_count",
"class_6_count",
"class_7_count",
"class_8_count",
"class_9_count"
],
"sort_by": [
"client_id"
],
"row_semantics": {
"rows": "one row per client_id for all clients",
"counts": "class_k_count is the number of training samples of class k assigned to the client"
}
},
"dirichlet_gammas_csv": {
"path": "dirichlet_gammas.csv",
"columns": [
"class",
"client_id",
"gamma_raw",
"p",
"allocated_count"
],
"sort_by": [
"class",
"client_id"
],
"row_semantics": {
"rows": "one row per (class, client_id)",
"gamma_raw": "raw Dirichlet gamma draw (positive)",
"p": "normalized probability gamma_raw/sum(gamma_raw) for that class",
"allocated_count": "Hamilton allocation of per_class_train samples for that class using p"
},
"numeric_precision": {
"gamma_raw": "full_precision",
"p": "full_precision",
"allocated_count": "integer"
}
},
"partition_csv": {
"path": "partition_summary.csv",
"columns": [
"client_id",
"is_byzantine",
"n_train",
"label_e
…[truncated]cat /app/analysis.R
#!/usr/bin/env Rscript
suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
`%||%` <- function(x, y) if (!is.null(x) && !is.na(x) && nzchar(as.character(x))) x else y
sigmoid <- function(x) 1.0 / (1.0 + exp(-x))
safe_dir_create <- function(path) {
if (!dir.exists(path)) dir.create(path, recursive = TRUE, showWarnings = FALSE)
}
read_bias_csv <- function(path, n_clients, n_classes) {
if (is.null(path) || !nzchar(path) || !file.exists(path)) {
return(NULL)
}
df <- read.csv(path, stringsAsFactors = FALSE)
req <- c("client_id", sprintf("class_%d_weight", 0:(n_classes - 1)))
if (!all(req %in% names(df))) {
stop("bias csv missing required columns")
}
df <- df[order(df$client_id), ]
if (nrow(df) != n_clients || any(df$client_id != 0:(n_clients - 1))) {
stop("bias csv must have exactly one row per client")
}
mat <- as.matrix(df[, req[-1], drop = FALSE])
storage.mode(mat) <- "double"
col_sums <- colSums(mat)
col_sums[col_sums <= 0] <- 1
sweep(mat, 2, col_sums, "/")
}
hamilton_alloc <- function(p, total) {
p <- as.numeric(p)
p <- p / sum(p)
raw <- p * total
alloc <- as.integer(round(raw))
gap <- as.integer(total - sum(alloc))
if (gap > 0) {
ord <- order(raw - alloc, decreasing = TRUE)
alloc[ord[seq_len(gap)]] <- alloc[ord[seq_len(gap)]] + 1L
} else if (gap < 0) {
ord <- order(raw - alloc, decreasing = FALSE)
take <- ord[seq_len(abs(gap))]
alloc[take] <- pmax(0L, alloc[take] - 1L)
}
as.integer(alloc)
}
repair_partition <- function(partition, min_client_samples) {
n_train <- rowSums(partition)
while (min(n_train) < min_client_samples) {
low <- which.min(n_train)
high <- which.max(n_train)
donor_class <- which.max(partition[high, ])
if (partition[high, donor_class] <= 0) break
partition[high, donor_class] <- partition[high, donor_class] - 1L
partition[low, donor_class] <- partition[low, donor_class] + 1L
n_train <- rowSums(partition)
}
partition
}
dirichlet_partition <- function(n_clients, n_classes, per_class_total, alpha, min_client_samples) {
part <- matrix(0L, nrow = n_clients, ncol = n_classes)
gammas <- matrix(0.0, nrow = n_clients, ncol = n_classes)
for (k in seq_len(n_classes)) {
g <- rgamma(n_clients, shape = alpha, rate = 1)
gammas[, k] <- g
part[, k] <- hamilton_alloc(g, per_class_total)
}
part <- repair_partition(part, min_client_samples)
list(partition = part, gammas = gammas)
}
shannon_entropy <- function(counts) {
s <- sum(counts)
if (s <= 0) return(0)
p <- counts / s
p <- p[p > 0]
-sum(p * log10(p))
}
row_trimmed_mean <- function(mat, trim_k) {
keep <- order(apply(mat, 1, function(x) sqrt(sum(x * x))))[(trim_k + 1):(nrow(mat) - trim_k)]
colMeans(mat[keep, , drop = FALSE])
}
run_sim <- function() {
task_variant <- Sys.getenv("TASK_VARIANT", "public")
seed <- as.integer(Sys.getenv("FEDSIMG_SEED", "0"))
set.seed(seed)
output_dir <- Sys.getenv("FEDSIMG_OUTPUT_DIR", "outputs")
manifest_path <- Sys.getenv("FEDSIMG_MANIFEST_PATH", "/app/dataset_manifest.json")
safe_dir_create(output_dir)
manifest <- fromJSON(manifest_path, simplifyVector = TRUE)
n_clients <- as.integer(manifest$simulation$n_clients)
n_byz <- as.integer(manifest$simulation$n_byzantine)
clients_per_round <- as.integer(manifest$simulation$clients_per_round)
n_rounds <- as.integer(manifest$simulation$n_rounds)
param_dim <- as.integer(manifest$simulation$param_dim)
n_classes <- as.integer(manifest$dataset$n_classes)
per_class <- as.integer(manifest$dataset$per_class_train)
alpha <- as.numeric(manifest$partition$alpha_default)
noise_scale <- as.numeric(manifest$attack$noise_scale_default)
signal_scale <- as.numeric(Sys.getenv("FEDSIMG_SIGNAL_SCALE", "0.12"))
trim_k <- as.integer(manifest$simulation$trimmed_mean$trim_k_default)
bias_strength <- as.numeric(manifest$client_bias$bias_strength_default)
bias_csv <- Sys.getenv(m
…[truncated]cat /app/example_client_bias.csv
client_id,class_0_weight,class_1_weight,class_2_weight,class_3_weight,class_4_weight,class_5_weight,class_6_weight,class_7_weight,class_8_weight,class_9_weight 0,0.28,0.10,0.07,0.06,0.06,0.06,0.07,0.08,0.12,0.10 1,0.05,0.06,0.09,0.14,0.15,0.14,0.12,0.10,0.08,0.07 2,0.08,0.08,0.08,0.08,0.08,0.08,0.12,0.14,0.14,0.12 3,0.04,0.05,0.06,0.07,0.10,0.13,0.15,0.15,0.14,0.11 4,0.16,0.14,0.12,0.10,0.09,0.08,0.07,0.06,0.10,0.08
ls /app/outputs/ 2>/dev/null || echo "empty"
hidden public
ls -la /app/outputs/public /app/outputs/hidden 2>/dev/null
/app/outputs/hidden: total 8 drwxrws--- 2 h2tagent h2tagent 4096 May 28 19:31 . drwxrws--- 4 h2tagent h2tagent 4096 May 28 19:31 .. /app/outputs/public: total 8 drwxrws--- 2 h2tagent h2tagent 4096 May 28 19:31 . drwxrws--- 4 h2tagent h2tagent 4096 May 28 19:31 ..
command: create
path: /app/analysis.R
file_text: #!/usr/bin/env Rscript
suppressPackageStartupMessages({
library(ggplot2)
library(gridExtra)
library(jsonlite)
})
`%||%` <- function(x, y) if (!is.null(x) && !is.na(x) && nzchar(as.character(x))) x else y
sigmoid <- function(x) 1.0 / (1.0 + exp(-x))
safe_dir_create <- function(path) {
if (!dir.exists(path)) dir.create(path, recursive = TRUE, showWarnings = FALSE)
}
env_num <- function(name, default) {
v <- Sys.getenv(name, "")
if (!nzchar(v)) return(as.numeric(default))
x <- suppressWarnings(as.numeric(v))
if (is.na(x)) as.numeric(default) else x
}
env_int <- function(name, default) {
v <- Sys.getenv(name, "")
if (!nzchar(v)) return(as.integer(default))
x <- suppressWarnings(as.integer(v))
if (is.na(x)) as.integer(default) else x
}
read_bias_csv <- function(path, n_clients, n_classes) {
if (is.null(path) || !nzchar(path) || !file.exists(path)) {
return(NULL)
}
df <- read.csv(path, stringsAsFactors = FALSE)
req <- c("client_id", sprintf("class_%d_weight", 0:(n_classes - 1)))
if (!all(req %in% names(df))) {
stop("bias csv missing required columns")
}
# Start with per-client weights: default uniform (1/n_classes)
mat <- matrix(1.0 / n_classes, nrow = n_clients, ncol = n_classes)
# Overwrite rows for clients present in the CSV
for (i in seq_len(nrow(df))) {
cid <- as.integer(df$client_id[i])
if (cid < 0 || cid >= n_clients) next
row_vals <- as.numeric(df[i, req[-1]])
s <- sum(row_vals)
if (!is.finite(s) || s <= 0) next
mat[cid + 1L, ] <- row_vals / s
}
mat
}
hamilton_alloc <- function(p, total) {
p <- as.numeric(p)
s <- sum(p)
if (!is.finite(s) || s <= 0) {
n <- length(p)
q <- rep(1.0 / n, n)
} else {
q <- p / s
}
raw <- q * total
floored <- as.integer(floor(raw))
gap <- as.integer(total - sum(floored))
if (gap > 0) {
frac <- raw - floored
ord <- order(frac, decreasing = TRUE)
floored[ord[seq_len(gap)]] <- floored[ord[seq_len(gap)]] + 1L
} else if (gap < 0) {
# unusual given floor semantics, but fall back safely
frac <- raw - floored
ord <- order(frac, decreasing = FALSE)
take <- ord[seq_len(abs(gap))]
floored[take] <- pmax(0L, floored[take] - 1L)
}
as.integer(floored)
}
repair_partition <- function(partition, min_client_samples) {
n_train <- rowSums(partition)
safety <- 0L
max_iter <- as.integer(nrow(partition)) * as.integer(ncol(partition)) * 10L + 1000L
while (min(n_train) < min_client_samples) {
safety <- safety + 1L
if (safety > max_iter) break
low <- which.min(n_train)
high <- which.max(n_train)
donor_class <- which.max(partition[high, ])
if (partition[high, donor_class] <= 0) break
partition[high, donor_class] <- partition[high, donor_class] - 1L
partition[low, donor_class] <- partition[low, donor_class] + 1L
n_train <- rowSums(partition)
}
partition
}
dirichlet_partition <- function(n_clients, n_classes, per_class_total, alpha, min_client_samples) {
part <- matrix(0L, nrow = n_clients, ncol = n_classes)
gammas <- matrix(0.0, nrow = n_clients, ncol = n_classes)
alloc_pre <- matrix(0L, nrow = n_clients, ncol = n_classes)
for (k in seq_len(n_classes)) {
g <- rgamma(n_clients, shape = alpha, rate = 1)
gammas[, k] <- g
a <- hamilton_alloc(g, per_class_total)
alloc_pre[, k] <- a
part[, k] <- a
}
part <- repair_partition(part, min_client_samples)
list(partition = part, gammas = gammas, alloc_pre = alloc_pre)
}
shannon_entropy <- function(counts) {
s <- sum(counts)
if (s <= 0) return(0)
p <- counts / s
p <- p[p > 0]
-sum(p * log(p)) # natural log
}
coord_trimmed_mean <- function(mat, trim_k) {
# Coordinate-wise trimmed mean: for each column, sort values across rows,
# drop the smallest trim_k and largest trim_k, average the rest.
n <- nrow(mat)
if (trim_k < 0) trim_k <- 0L
if (2L * trim_k >= n) {
return(apply(mat, 2, medi
…[truncated]Existing file was successfully overwritten at /app/analysis.R
cd /app && rm -rf outputs/public/* && \ TASK_VARIANT=public \ FEDSIMG_SEED=42 \ FEDSIMG_OUTPUT_DIR=outputs/public \ FEDSIMG_MANIFEST_PATH=/app/dataset_manifest.json \ FEDSIMG_ALPHA=0.5 \ FEDSIMG_NOISE_SCALE=2.0 \ FEDSIMG_SIGNAL_SCALE=0.12 \ FEDSIMG_TRIM_K=2 \ FEDSIMG_BIAS_STRENGTH=0.35 \ FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv \ Rscript /app/analysis.R 2>&1 | tail -30
<system>Tool ran without output or errors</system>
ls -la /app/outputs/public/
total 1955 drwxrws--- 2 h2tagent h2tagent 4096 May 28 23:09 . drwxrws--- 4 h2tagent h2tagent 4096 May 28 19:31 .. -rw-r--r-- 1 h2tagent h2tagent 1017 May 28 23:09 analysis_summary.json -rw-r--r-- 1 h2tagent h2tagent 12622 May 28 23:09 byzantine_audit.csv -rw-r--r-- 1 h2tagent h2tagent 732247 May 28 23:09 client_updates.csv -rw-r--r-- 1 h2tagent h2tagent 1117299 May 28 23:09 client_updates_clean.csv -rw-r--r-- 1 h2tagent h2tagent 73953 May 28 23:09 diagnostics.png -rw-r--r-- 1 h2tagent h2tagent 45882 May 28 23:09 dirichlet_gammas.csv -rw-r--r-- 1 h2tagent h2tagent 1740 May 28 23:09 metrics_by_round.csv -rw-r--r-- 1 h2tagent h2tagent 3363 May 28 23:09 partition_matrix.csv -rw-r--r-- 1 h2tagent h2tagent 2952 May 28 23:09 partition_summary.csv
cat /app/outputs/public/analysis_summary.json
{
"task_variant": "public",
"seed": 42,
"env": {
"FEDSIMG_ALPHA": 0.5,
"FEDSIMG_NOISE_SCALE": 2,
"FEDSIMG_SIGNAL_SCALE": 0.12,
"FEDSIMG_TRIM_K": 2,
"FEDSIMG_BIAS_STRENGTH": 0.35,
"FEDSIMG_CLIENT_BIAS_CSV": "/app/example_client_bias.csv"
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 2
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.965518,
"fedavg_accuracy": 0.916737,
"delta": 0.048781
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [2, 1, 1, 3, 1, 3, 1, 1, 0, 2, 2, 2, 3, 2, 3, 2, 0, 1, 1, 3, 1, 3, 0, 0, 2, 2, 0, 0, 0, 0, 1, 3, 2, 2, 1, 3, 3, 2, 3, 2, 2, 3, 2, 1, 2, 1, 2, 2, 3, 2, 2, 2, 0, 1, 2, 3, 2, 1, 1, 3, 3, 2, 2, 2, 3, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, 4, 2, 1, 1, 0, 3, 1, 1, 1, 0, 0, 2, 0, 2, 2, 3, 0, 2, 1, 2, 3, 3, 1, 1, 0]
}
}head -5 /app/outputs/public/metrics_by_round.csv echo "---" head -5 /app/outputs/public/partition_summary.csv echo "---" head -5 /app/outputs/public/byzantine_audit.csv echo "---" head -3 /app/outputs/public/client_updates.csv echo "---" head -3 /app/outputs/public/client_updates_clean.csv echo "---" head -3 /app/outputs/public/dirichlet_gammas.csv echo "---" head -3 /app/outputs/public/partition_matrix.csv
"round","method","accuracy","cosine_sim","update_norm","byzantine_selected","selected_clients" 0,"fedavg",0.386048,0.272595,3.732997,1,"80 90 62 55 94 67 31 56 60 65 10 19 40 14 61 57 7 68 75 51" 0,"trimmed_mean",0.952078,0.743687,2.588454,1,"80 90 62 55 94 67 31 56 60 65 10 19 40 14 61 57 7 68 75 51" 1,"fedavg",0.683623,0.436469,3.630707,2,"90 32 13 64 47 53 95 42 35 72 6 1 30 78 74 55 19 36 15 38" 1,"trimmed_mean",0.953833,0.752278,2.613005,2,"90 32 13 64 47 53 95 42 35 72 6 1 30 78 74 55 19 36 15 38" --- "client_id","is_byzantine","n_train","label_entropy","top_label","top_label_share" 0,1,708,1.792905,7,0.323446 1,1,756,1.339652,6,0.411376 2,1,759,1.427729,1,0.466403 3,1,640,1.563247,3,0.335938 --- "round","client_id","is_byzantine","is_selected","client_weight","update_norm" 0,0,1,0,0,0 0,1,1,0,0,0 0,2,1,0,0,0 0,3,1,0,0,0 --- "round","client_id","coord","update_value" 0,7,0,-1.575337 0,7,1,0.50494 --- "round","client_id","coord","u_clean","u_submitted" 0,7,0,0.160076,-1.575337 0,7,1,0.0487,0.50494 --- "class","client_id","gamma_raw","p","allocated_count" 0,0,0.760516834810734,0.0165288490248016,99 0,1,0.293697522769724,0.00638313550814045,38 --- "client_id","class_0_count","class_1_count","class_2_count","class_3_count","class_4_count","class_5_count","class_6_count","class_7_count","class_8_count","class_9_count" 0,99,20,180,32,26,72,46,229,3,1 1,38,2,29,0,5,3,311,291,71,6
cd /app && python3 -c "
import pandas as pd
import json, math
m = pd.read_csv('outputs/public/metrics_by_round.csv')
print('=== metrics shape ===')
print(m.shape, list(m.columns))
print('=== final round ===')
final_r = m['round'].max()
f = m[(m['round']==final_r) & (m['method']=='fedavg')].iloc[0]
t = m[(m['round']==final_r) & (m['method']=='trimmed_mean')].iloc[0]
print('fedavg acc:', f['accuracy'], 'tm acc:', t['accuracy'], 'delta:', t['accuracy']-f['accuracy'])
print('=== summary ===')
with open('outputs/public/analysis_summary.json') as fh:
s = json.load(fh)
print('json trimmed_mean_accuracy:', s['final_metrics']['trimmed_mean_accuracy'])
print('json fedavg_accuracy:', s['final_metrics']['fedavg_accuracy'])
print('json delta:', s['final_metrics']['delta'])
# Check that values match exactly
assert s['final_metrics']['trimmed_mean_accuracy'] == t['accuracy']
assert s['final_metrics']['fedavg_accuracy'] == f['accuracy']
assert math.isclose(s['final_metrics']['delta'], t['accuracy'] - f['accuracy'], abs_tol=1e-12)
print('All matches')
# Check selection_histogram length
print('sel_hist len:', len(s['provenance']['selection_histogram']))
print('sum sel_hist:', sum(s['provenance']['selection_histogram']))
print('expected (clients_per_round * n_rounds):', s['simulation_meta']['clients_per_round'] * s['simulation_meta']['n_rounds'])
"
=== metrics shape === (16, 7) ['round', 'method', 'accuracy', 'cosine_sim', 'update_norm', 'byzantine_selected', 'selected_clients'] === final round === fedavg acc: 0.916737 tm acc: 0.965518 delta: 0.04878099999999996 === summary === json trimmed_mean_accuracy: 0.965518 json fedavg_accuracy: 0.916737 json delta: 0.048781 All matches sel_hist len: 100 sum sel_hist: 160 expected (clients_per_round * n_rounds): 160
cd /app && Rscript -e '
tm <- 0.965518
fed <- 0.916737
delta <- tm - fed
cat("delta raw:", format(delta, digits = 22), "\n")
cat("delta:", delta, "\n")
library(jsonlite)
s <- list(x = delta)
cat(toJSON(s, digits = NA, auto_unbox = TRUE), "\n")
cat(toJSON(s, digits = 8, auto_unbox = TRUE), "\n")
'
delta raw: 0.04878099999999996327915
delta: 0.048781
{"x":0.048781}
{"x":0.048781}python3 -c "
import json
with open('/app/outputs/public/analysis_summary.json') as f:
s = json.load(f)
tm = s['final_metrics']['trimmed_mean_accuracy']
fed = s['final_metrics']['fedavg_accuracy']
delta = s['final_metrics']['delta']
print('tm:', repr(tm))
print('fed:', repr(fed))
print('delta:', repr(delta))
print('tm - fed:', repr(tm - fed))
print('exact match:', delta == (tm - fed))
"
tm: 0.965518 fed: 0.916737 delta: 0.048781 tm - fed: 0.04878099999999996 exact match: False
cd /app && Rscript -e '
library(jsonlite)
# Try various approaches to full-precision output
delta_raw <- 0.965518 - 0.916737
cat("raw:", sprintf("%.20f", delta_raw), "\n")
s <- list(x = delta_raw)
cat("digits=NA:", toJSON(s, digits = NA, auto_unbox = TRUE), "\n")
cat("digits=17:", toJSON(s, digits = 17, auto_unbox = TRUE), "\n")
cat("digits=15:", toJSON(s, digits = 15, auto_unbox = TRUE), "\n")
# What about writing via formatted strings?
m <- sprintf("%.17g", delta_raw)
cat("sprintf:", m, "\n")
'
raw: 0.04878099999999996328
digits=NA: {"x":0.048781}
digits=17: {"x":0.048780999999999963}
digits=15: {"x":0.04878099999999996}
sprintf: 0.048780999999999963cd /app && Rscript -e '
library(jsonlite)
x <- 0.965518
cat("sprintf %.17g:", sprintf("%.17g", x), "\n")
s <- list(
acc = 0.965518,
fed = 0.916737,
delta = 0.965518 - 0.916737
)
cat("digits=17:", toJSON(s, digits = 17, auto_unbox = TRUE), "\n")
'
sprintf %.17g: 0.96551799999999999
digits=17: {"acc":0.96551799999999999,"fed":0.91673700000000002,"delta":0.048780999999999963}cd /app && Rscript -e '
library(jsonlite)
s <- list(
acc = 0.965518,
fed = 0.916737,
delta = 0.965518 - 0.916737
)
write_json(s, "/tmp/test.json", digits = 17, auto_unbox = TRUE, pretty = TRUE)
'
cat /tmp/test.json
echo
echo "---"
python3 -c "
import json
with open('/tmp/test.json') as f:
d = json.load(f)
print('acc:', repr(d['acc']))
print('fed:', repr(d['fed']))
print('delta:', repr(d['delta']))
print('acc - fed:', repr(d['acc'] - d['fed']))
print('delta == acc-fed?', d['delta'] == d['acc'] - d['fed'])
"
{
"acc": 0.96551799999999999,
"fed": 0.91673700000000002,
"delta": 0.048780999999999963
}
---
acc: 0.965518
fed: 0.916737
delta: 0.04878099999999996
acc - fed: 0.04878099999999996
delta == acc-fed? Truecommand: str_replace
path: /app/analysis.R
old_str: write_json(
summary,
file.path(output_dir, manifest$outputs$summary_json$path),
pretty = TRUE,
auto_unbox = TRUE,
digits = NA # preserve full numeric precision for values that should match CSV
)
new_str: write_json(
summary,
file.path(output_dir, manifest$outputs$summary_json$path),
pretty = TRUE,
auto_unbox = TRUE,
digits = 17 # full IEEE-754 round-trip precision so JSON floats match CSV floats
)The file /app/analysis.R has been edited successfully.
cd /app && rm -rf outputs/public/* && \ TASK_VARIANT=public \ FEDSIMG_SEED=42 \ FEDSIMG_OUTPUT_DIR=outputs/public \ FEDSIMG_MANIFEST_PATH=/app/dataset_manifest.json \ FEDSIMG_ALPHA=0.5 \ FEDSIMG_NOISE_SCALE=2.0 \ FEDSIMG_SIGNAL_SCALE=0.12 \ FEDSIMG_TRIM_K=2 \ FEDSIMG_BIAS_STRENGTH=0.35 \ FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv \ Rscript /app/analysis.R 2>&1 cat /app/outputs/public/analysis_summary.json | head -40
{
"task_variant": "public",
"seed": 42,
"env": {
"FEDSIMG_ALPHA": 0.5,
"FEDSIMG_NOISE_SCALE": 2,
"FEDSIMG_SIGNAL_SCALE": 0.12,
"FEDSIMG_TRIM_K": 2,
"FEDSIMG_BIAS_STRENGTH": 0.34999999999999998,
"FEDSIMG_CLIENT_BIAS_CSV": "/app/example_client_bias.csv"
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 2
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.96551799999999999,
"fedavg_accuracy": 0.91673700000000002,
"delta": 0.048780999999999963
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [2, 1, 1, 3, 1, 3, 1, 1, 0, 2, 2, 2, 3, 2, 3, 2, 0, 1, 1, 3, 1, 3, 0, 0, 2, 2, 0, 0, 0, 0, 1, 3, 2, 2, 1, 3, 3, 2, 3, 2, 2, 3, 2, 1, 2, 1, 2, 2, 3, 2, 2, 2, 0, 1, 2, 3, 2, 1, 1, 3, 3, 2, 2, 2, 3, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, 4, 2, 1, 1, 0, 3, 1, 1, 1, 0, 0, 2, 0, 2, 2, 3, 0, 2, 1, 2, 3, 3, 1, 1, 0]
}
}python3 -c "
import json
with open('/app/outputs/public/analysis_summary.json') as f:
s = json.load(f)
print(repr(s['env']['FEDSIMG_BIAS_STRENGTH']))
print(repr(s['env']['FEDSIMG_NOISE_SCALE']))
print(repr(s['env']['FEDSIMG_ALPHA']))
print('0.35 match:', s['env']['FEDSIMG_BIAS_STRENGTH'] == 0.35)
print('2.0 match:', s['env']['FEDSIMG_NOISE_SCALE'] == 2.0)
"0.35 2 0.5 0.35 match: True 2.0 match: True
cd /app && python3 -c "
import pandas as pd
import json
# check partition_matrix
pm = pd.read_csv('outputs/public/partition_matrix.csv')
print('partition_matrix shape:', pm.shape, list(pm.columns))
row_sums = pm.iloc[:, 1:].sum(axis=1)
print('row sum range:', row_sums.min(), row_sums.max())
col_sums = pm.iloc[:, 1:].sum(axis=0)
print('col sums (should each be 6000):', col_sums.tolist())
print('\n=== partition_summary ===')
ps = pd.read_csv('outputs/public/partition_summary.csv')
print(ps.shape, list(ps.columns))
print(ps.head())
print('\n=== dirichlet_gammas ===')
g = pd.read_csv('outputs/public/dirichlet_gammas.csv')
print(g.shape, list(g.columns))
# Check allocated_count sums to per_class (6000) per class
print('per-class alloc sum:', g.groupby('class')['allocated_count'].sum().tolist())
print('\n=== byzantine_audit ===')
b = pd.read_csv('outputs/public/byzantine_audit.csv')
print(b.shape, list(b.columns))
# Selected count per round should equal clients_per_round (20)
print('selected per round:', b.groupby('round')['is_selected'].sum().tolist())
# Byzantine IDs should be 0..9
print('is_byzantine for client 0-9:', b[b['round']==0]['is_byzantine'].head(10).tolist())
print('is_byzantine for client 90-99:', b[b['round']==0]['is_byzantine'].tail(10).tolist())
# Check client_weights sum to 1 per round
cw = b[b['is_selected']==1].groupby('round')['client_weight'].sum()
print('per-round client_weight sum:', cw.tolist())
# Check update_norm is 0 for not selected
print('update_norm for not selected (should all be 0):', (b.loc[b['is_selected']==0, 'update_norm'] != 0).sum())
print('\n=== client_updates ===')
cu = pd.read_csv('outputs/public/client_updates.csv')
print(cu.shape, list(cu.columns))
# Expected rows: clients_per_round * n_rounds * param_dim = 20*8*256
print('expected rows:', 20*8*256)
"
partition_matrix shape: (100, 11) ['client_id', 'class_0_count', 'class_1_count', 'class_2_count', 'class_3_count', 'class_4_count', 'class_5_count', 'class_6_count', 'class_7_count', 'class_8_count', 'class_9_count'] row sum range: 125 1199 col sums (should each be 6000): [6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000] === partition_summary === (100, 6) ['client_id', 'is_byzantine', 'n_train', 'label_entropy', 'top_label', 'top_label_share'] client_id is_byzantine n_train label_entropy top_label top_label_share 0 0 1 708 1.792905 7 0.323446 1 1 1 756 1.339652 6 0.411376 2 2 1 759 1.427729 1 0.466403 3 3 1 640 1.563247 3 0.335938 4 4 1 462 1.935098 8 0.313853 === dirichlet_gammas === (1000, 5) ['class', 'client_id', 'gamma_raw', 'p', 'allocated_count'] per-class alloc sum: [6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000] === byzantine_audit === (800, 6) ['round', 'client_id', 'is_byzantine', 'is_selected', 'client_weight', 'update_norm'] selected per round: [20, 20, 20, 20, 20, 20, 20, 20] is_byzantine for client 0-9: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] is_byzantine for client 90-99: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] per-round client_weight sum: [0.999999, 0.999999, 1.000002, 1.000001, 0.999999, 0.999998, 0.9999990000000001, 0.9999990000000001] update_norm for not selected (should all be 0): 0 === client_updates === (40960, 4) ['round', 'client_id', 'coord', 'update_value'] expected rows: 40960
cd /app && python3 -c "
import pandas as pd
import json
import numpy as np
# Verify selection_histogram matches CSV
m = pd.read_csv('outputs/public/metrics_by_round.csv')
with open('outputs/public/analysis_summary.json') as f:
s = json.load(f)
# Parse selected_clients in round+fedavg rows
fed = m[m['method']=='fedavg'].sort_values('round')
counts = np.zeros(100, dtype=int)
for _, row in fed.iterrows():
for cid in row['selected_clients'].split():
counts[int(cid)] += 1
hist = s['provenance']['selection_histogram']
print('histograms match:', counts.tolist() == hist)
# Check cosine_sim increasing generally
print('fedavg cosine_sim:', fed['cosine_sim'].tolist())
tm = m[m['method']=='trimmed_mean'].sort_values('round')
print('tm cosine_sim:', tm['cosine_sim'].tolist())
print('tm accuracy:', tm['accuracy'].tolist())
print('fedavg accuracy:', fed['accuracy'].tolist())
"
histograms match: True fedavg cosine_sim: [0.272595, 0.436469, 0.559204, 0.600394, 0.618796, 0.629878, 0.647836, 0.655808] tm cosine_sim: [0.743687, 0.752278, 0.755374, 0.754868, 0.756184, 0.75443, 0.755545, 0.756807] tm accuracy: [0.952078, 0.953833, 0.960185, 0.965211, 0.954774, 0.957102, 0.960354, 0.965518] fedavg accuracy: [0.386048, 0.683623, 0.839806, 0.888504, 0.897229, 0.906551, 0.918248, 0.916737]
cd /app && rm -rf outputs/hidden/* && \ TASK_VARIANT=hidden \ FEDSIMG_SEED=17 \ FEDSIMG_OUTPUT_DIR=outputs/hidden \ FEDSIMG_MANIFEST_PATH=/app/dataset_manifest.json \ FEDSIMG_ALPHA=0.3 \ FEDSIMG_NOISE_SCALE=3.0 \ FEDSIMG_SIGNAL_SCALE=0.15 \ FEDSIMG_TRIM_K=3 \ FEDSIMG_BIAS_STRENGTH=0.5 \ Rscript /app/analysis.R 2>&1 | tail -20 echo "---" ls /app/outputs/hidden/ echo "---" cat /app/outputs/hidden/analysis_summary.json | head -40
---
analysis_summary.json
byzantine_audit.csv
client_updates.csv
client_updates_clean.csv
diagnostics.png
dirichlet_gammas.csv
metrics_by_round.csv
partition_matrix.csv
partition_summary.csv
---
{
"task_variant": "hidden",
"seed": 17,
"env": {
"FEDSIMG_ALPHA": 0.29999999999999999,
"FEDSIMG_NOISE_SCALE": 3,
"FEDSIMG_SIGNAL_SCALE": 0.14999999999999999,
"FEDSIMG_TRIM_K": 3,
"FEDSIMG_BIAS_STRENGTH": 0.5,
"FEDSIMG_CLIENT_BIAS_CSV": ""
},
"simulation_meta": {
"n_clients": 100,
"n_byzantine": 10,
"clients_per_round": 20,
"n_rounds": 8,
"param_dim": 256,
"trim_k": 3
},
"byzantine_client_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"final_metrics": {
"trimmed_mean_accuracy": 0.91554199999999997,
"fedavg_accuracy": 0.88021799999999994,
"delta": 0.035324000000000022
},
"provenance": {
"manifest_md5": "475adf00a65314160823670bdaef093c",
"selection_histogram": [2, 1, 1, 1, 4, 1, 3, 1, 1, 3, 2, 1, 1, 1, 2, 0, 1, 3, 1, 2, 3, 1, 1, 4, 2, 0, 2, 1, 3, 2, 1, 3, 0, 1, 2, 0, 2, 4, 1, 3, 1, 2, 0, 1, 1, 2, 0, 1, 2, 2, 1, 3, 2, 0, 1, 1, 3, 1, 1, 2, 2, 2, 2, 1, 4, 0, 1, 0, 1, 2, 0, 2, 1, 2, 2, 0, 3, 0, 1, 2, 3, 4, 3, 2, 3, 0, 0, 1, 1, 2, 2, 2, 1, 2, 1, 4, 1, 1, 2, 2]
}
}cd /app && python3 -c "
import pandas as pd, json
m = pd.read_csv('outputs/hidden/metrics_by_round.csv')
with open('outputs/hidden/analysis_summary.json') as f:
s = json.load(f)
# Sanity checks:
tm = m[m['method']=='trimmed_mean'].sort_values('round').iloc[-1]['accuracy']
fed = m[m['method']=='fedavg'].sort_values('round').iloc[-1]['accuracy']
print('CSV tm:', repr(tm))
print('CSV fed:', repr(fed))
print('JSON tm:', repr(s['final_metrics']['trimmed_mean_accuracy']))
print('JSON fed:', repr(s['final_metrics']['fedavg_accuracy']))
print('JSON delta:', repr(s['final_metrics']['delta']))
print('CSV tm - fed:', repr(tm - fed))
print('exact match tm:', tm == s['final_metrics']['trimmed_mean_accuracy'])
print('exact match fed:', fed == s['final_metrics']['fedavg_accuracy'])
print('exact match delta:', (tm - fed) == s['final_metrics']['delta'])
"
CSV tm: 0.915542 CSV fed: 0.880218 JSON tm: 0.915542 JSON fed: 0.880218 JSON delta: 0.03532400000000002 CSV tm - fed: 0.03532400000000002 exact match tm: True exact match fed: True exact match delta: True
cd /app && python3 -c "
import pandas as pd
cu_clean = pd.read_csv('outputs/public/client_updates_clean.csv')
cu = pd.read_csv('outputs/public/client_updates.csv')
# Pick a byzantine client (0-9) that was selected
with open('outputs/public/metrics_by_round.csv') as f:
pass
audit = pd.read_csv('outputs/public/byzantine_audit.csv')
byz_selected = audit[(audit['is_byzantine']==1) & (audit['is_selected']==1)]
print('byz selected sample:')
print(byz_selected.head())
# Pick the first selected byzantine in round 0
r, cid = byz_selected.iloc[0]['round'], byz_selected.iloc[0]['client_id']
clean_rows = cu_clean[(cu_clean['round']==r) & (cu_clean['client_id']==cid)].sort_values('coord')
sub_rows = cu[(cu['round']==r) & (cu['client_id']==cid)].sort_values('coord')
# Check: u_submitted = -u_clean + noise
diff = sub_rows['update_value'].values + clean_rows['u_clean'].values
print('\\nbyzantine noise stats (u_sub + u_clean):')
print('mean:', diff.mean(), 'std:', diff.std())
# Pick an honest selected client
hon_selected = audit[(audit['is_byzantine']==0) & (audit['is_selected']==1)]
r2, cid2 = hon_selected.iloc[0]['round'], hon_selected.iloc[0]['client_id']
clean2 = cu_clean[(cu_clean['round']==r2) & (cu_clean['client_id']==cid2)].sort_values('coord')
sub2 = cu[(cu['round']==r2) & (cu['client_id']==cid2)].sort_values('coord')
# For honest, u_submitted should == u_clean
print('\\nhonest: u_sub == u_clean?')
print('max abs diff:', (sub2['update_value'].values - clean2['u_clean'].values).__abs__().max())
print('u_submitted col in clean == u_sub?')
print('max abs diff:', (clean2['u_submitted'].values - sub2['update_value'].values).__abs__().max())
"
byz selected sample:
round client_id is_byzantine is_selected client_weight update_norm
7 0 7 1 1 0.097258 33.971819
101 1 1 1 1 0.067979 34.645467
106 1 6 1 1 0.047208 30.915869
203 2 3 1 1 0.055335 31.458959
300 3 0 1 1 0.061608 31.777368
byzantine noise stats (u_sub + u_clean):
mean: 0.007285574218749952 std: 2.096880836382072
honest: u_sub == u_clean?
max abs diff: 0.0
u_submitted col in clean == u_sub?
max abs diff: 0.0cd /app && python3 -c "
import pandas as pd
import numpy as np
from math import log
# Verify entropy uses natural log
pm = pd.read_csv('outputs/public/partition_matrix.csv')
ps = pd.read_csv('outputs/public/partition_summary.csv')
for i in range(5):
counts = pm.iloc[i, 1:].values.astype(float)
s = counts.sum()
p = counts[counts > 0] / s
ent_natural = -(p * np.log(p)).sum()
ent_log10 = -(p * np.log10(p)).sum()
print(f'client {i}: CSV={ps.iloc[i][\"label_entropy\"]:.6f}, natural={ent_natural:.6f}, log10={ent_log10:.6f}')
"
client 0: CSV=1.792905, natural=1.792905, log10=0.778649 client 1: CSV=1.339652, natural=1.339652, log10=0.581803 client 2: CSV=1.427729, natural=1.427729, log10=0.620055 client 3: CSV=1.563247, natural=1.563247, log10=0.678910 client 4: CSV=1.935098, natural=1.935098, log10=0.840402
cd /app && python3 -c "
import pandas as pd
pm = pd.read_csv('outputs/public/partition_matrix.csv')
ps = pd.read_csv('outputs/public/partition_summary.csv')
for i in range(5):
counts = pm.iloc[i, 1:].values
top = counts.max()
total = counts.sum()
expected_share = top/total
actual = ps.iloc[i]['top_label_share']
top_label = counts.argmax()
print(f'client {i}: top={top}, total={total}, expected_share={expected_share:.6f}, CSV={actual}, top_label={top_label} (CSV={ps.iloc[i][\"top_label\"]})')
"
client 0: top=229, total=708, expected_share=0.323446, CSV=0.323446, top_label=7 (CSV=7.0) client 1: top=311, total=756, expected_share=0.411376, CSV=0.411376, top_label=6 (CSV=6.0) client 2: top=354, total=759, expected_share=0.466403, CSV=0.466403, top_label=1 (CSV=1.0) client 3: top=215, total=640, expected_share=0.335938, CSV=0.335938, top_label=3 (CSV=3.0) client 4: top=145, total=462, expected_share=0.313853, CSV=0.313853, top_label=8 (CSV=8.0)
cd /app && python3 -c "
import pandas as pd
import numpy as np
m = pd.read_csv('outputs/public/metrics_by_round.csv')
cu = pd.read_csv('outputs/public/client_updates.csv')
# For round 0, get all selected client updates and compute trimmed mean
r = 0
tm_k = 2
updates_r0 = cu[cu['round']==r].pivot(index='client_id', columns='coord', values='update_value').values
print('update matrix shape:', updates_r0.shape)
# Coordinate-wise trimmed mean
sorted_vals = np.sort(updates_r0, axis=0)
trimmed = sorted_vals[tm_k:-tm_k, :].mean(axis=0)
tm_norm = np.sqrt(np.sum(trimmed**2))
print(f'computed tm_update_norm: {tm_norm:.6f}')
csv_norm = m[(m['round']==0) & (m['method']=='trimmed_mean')]['update_norm'].iloc[0]
print(f'CSV update_norm: {csv_norm}')
"
update matrix shape: (20, 256) computed tm_update_norm: 2.588454 CSV update_norm: 2.588454
cd /app && python3 -c "
import pandas as pd
import numpy as np
m = pd.read_csv('outputs/public/metrics_by_round.csv')
cu = pd.read_csv('outputs/public/client_updates.csv')
audit = pd.read_csv('outputs/public/byzantine_audit.csv')
r = 0
updates_r0 = cu[cu['round']==r].pivot(index='client_id', columns='coord', values='update_value')
client_ids = updates_r0.index.tolist()
vals = updates_r0.values
# Get FedAvg weights from audit
aud_r = audit[audit['round']==r]
weights = []
for cid in client_ids:
w = aud_r[aud_r['client_id']==cid]['client_weight'].iloc[0]
weights.append(w)
weights = np.array(weights)
print('sum of weights:', weights.sum())
# FedAvg update
fed = (vals * weights[:, None]).sum(axis=0)
fed_norm = np.sqrt(np.sum(fed**2))
print('computed fed_norm:', fed_norm)
csv_fed_norm = m[(m['round']==0) & (m['method']=='fedavg')]['update_norm'].iloc[0]
print('CSV fed_norm:', csv_fed_norm)
"
sum of weights: 0.999999 computed fed_norm: 3.7329892935464737 CSV fed_norm: 3.732997
cd /app && rm -rf outputs/public/* outputs/hidden/*
# Run 1
TASK_VARIANT=public FEDSIMG_SEED=42 FEDSIMG_OUTPUT_DIR=outputs/public \
FEDSIMG_ALPHA=0.5 FEDSIMG_NOISE_SCALE=2.0 FEDSIMG_SIGNAL_SCALE=0.12 \
FEDSIMG_TRIM_K=2 FEDSIMG_BIAS_STRENGTH=0.35 \
FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv \
Rscript /app/analysis.R 2>&1
md5sum /app/outputs/public/*.csv /app/outputs/public/*.json
# Run 2 (same inputs)
mkdir -p /tmp/out2
TASK_VARIANT=public FEDSIMG_SEED=42 FEDSIMG_OUTPUT_DIR=/tmp/out2 \
FEDSIMG_ALPHA=0.5 FEDSIMG_NOISE_SCALE=2.0 FEDSIMG_SIGNAL_SCALE=0.12 \
FEDSIMG_TRIM_K=2 FEDSIMG_BIAS_STRENGTH=0.35 \
FEDSIMG_CLIENT_BIAS_CSV=/app/example_client_bias.csv \
Rscript /app/analysis.R 2>&1
echo "--- Check md5 match ---"
diff <(md5sum /app/outputs/public/*.csv /app/outputs/public/*.json | awk '{print $1}') \
<(md5sum /tmp/out2/*.csv /tmp/out2/*.json | awk '{print $1}') && echo "REPRODUCIBLE"6fcd9ac0eea746c4e95ba835a571c05c /app/outputs/public/byzantine_audit.csv 68fadf68f1cddc376814b35e70fae6ac /app/outputs/public/client_updates.csv c0eb9626651d55c41f5f9d946fa42d3f /app/outputs/public/client_updates_clean.csv a6500f23bf8536df80eac7f83a58745b /app/outputs/public/dirichlet_gammas.csv 77a0f62dbff4e5f5b5dd38dfb206f6f5 /app/outputs/public/metrics_by_round.csv 2bdf9a61151395a670f4b12479d276f5 /app/outputs/public/partition_matrix.csv 9ca4e5062d77fa48b3b0a89132798506 /app/outputs/public/partition_summary.csv 8ab42596bb3ad7136b88ef54c39f8adc /app/outputs/public/analysis_summary.json --- Check md5 match --- REPRODUCIBLE
cd /app && TASK_VARIANT=hidden FEDSIMG_SEED=17 FEDSIMG_OUTPUT_DIR=outputs/hidden \
FEDSIMG_ALPHA=0.3 FEDSIMG_NOISE_SCALE=3.0 FEDSIMG_SIGNAL_SCALE=0.15 \
FEDSIMG_TRIM_K=3 FEDSIMG_BIAS_STRENGTH=0.5 \
Rscript /app/analysis.R 2>&1
ls /app/outputs/hidden/
python3 -c "
import json
with open('/app/outputs/hidden/analysis_summary.json') as f:
s = json.load(f)
# Ensure all required keys present
req = ['task_variant','seed','env','simulation_meta','byzantine_client_ids','final_metrics','provenance']
for k in req:
assert k in s, k
env_req = ['FEDSIMG_ALPHA','FEDSIMG_NOISE_SCALE','FEDSIMG_SIGNAL_SCALE','FEDSIMG_TRIM_K','FEDSIMG_BIAS_STRENGTH','FEDSIMG_CLIENT_BIAS_CSV']
for k in env_req:
assert k in s['env'], k
sim_req = ['n_clients','n_byzantine','clients_per_round','n_rounds','param_dim','trim_k']
for k in sim_req:
assert k in s['simulation_meta'], k
fm_req = ['trimmed_mean_accuracy','fedavg_accuracy','delta']
for k in fm_req:
assert k in s['final_metrics'], k
prov_req = ['manifest_md5','selection_histogram']
for k in prov_req:
assert k in s['provenance'], k
print('All required keys present.')
print('trim_k:', s['simulation_meta']['trim_k'])
print('final_metrics:', s['final_metrics'])
"
analysis_summary.json
byzantine_audit.csv
client_updates.csv
client_updates_clean.csv
diagnostics.png
dirichlet_gammas.csv
metrics_by_round.csv
partition_matrix.csv
partition_summary.csv
All required keys present.
trim_k: 3
final_metrics: {'trimmed_mean_accuracy': 0.915542, 'fedavg_accuracy': 0.880218, 'delta': 0.03532400000000002}cd /app && Rscript -e '
# Syntax check only
source("/app/analysis.R", echo = FALSE)
' 2>&1 | head -5<system>Tool ran without output or errors</system>
# Finally, verify the diagnostics.png got generated properly file /app/outputs/public/diagnostics.png /app/outputs/hidden/diagnostics.png
[stderr] bash: file: command not found
ls -la /app/outputs/public/diagnostics.png /app/outputs/hidden/diagnostics.png head -c 8 /app/outputs/public/diagnostics.png | od -c | head -1
-rw-r--r-- 1 h2tagent h2tagent 76001 May 28 23:11 /app/outputs/hidden/diagnostics.png -rw-r--r-- 1 h2tagent h2tagent 73953 May 28 23:11 /app/outputs/public/diagnostics.png 0000000 211 P N G \r \n 032 \n
No file changes, the agent left the workspace untouched.
# SyncVals normalized verifier summary # task: fedavg-federated-noniid-mnist # attempt: 9 status: completed raw_score: 0.0 # reward rule: binary: score 0.0 != 1.0 # => reward = 0.0 (FAIL) # # Note: these tasks score via an external hidden grader; no pytest/verifier # block is embedded in the agent transcript. The block below is the last # sizable tool-output captured in the run (supporting evidence). ---------------------------------------------------------------------- -rw-r--r-- 1 h2tagent h2tagent 76001 May 28 23:11 /app/outputs/hidden/diagnostics.png -rw-r--r-- 1 h2tagent h2tagent 73953 May 28 23:11 /app/outputs/public/diagnostics.png 0000000 211 P N G \r \n 032 \n
Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_830367e508684f9e. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_830367e508684f9e · verifier authoritative; classifier explanatory.