tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.Check environment and tools
ls /app && echo "---" && which aws python3 jq zip curl && echo "---" && aws --version && python3 --version
--- /usr/local/bin/aws /opt/venv/bin/python3 /usr/bin/jq /usr/bin/zip /usr/bin/curl --- aws-cli/2.34.33 Python/3.14.4 Linux/6.10.14-linuxkit exe/aarch64.ubuntu.24 Python 3.12.3 [stdout] --- /usr/local/bin/aws /opt/venv/bin/python3 /usr/bin/jq /usr/bin/zip /usr/bin/curl --- aws-cli/2.34.33 Python/3.14.4 Linux/6.10.14-linuxkit exe/aarch64.ubuntu.24 Python 3.12.3
Check LocalStack health
curl -s http://localstack:4566/_localstack/health | jq .
{
"services": {
"acm": "disabled",
"apigateway": "disabled",
"cloudformation": "disabled",
"cloudwatch": "disabled",
"config": "disabled",
"dynamodb": "disabled",
"dynamodbstreams": "disabled",
"ec2": "disabled",
"es": "disabled",
"events": "available",
"firehose": "disabled",
"iam": "available",
"kinesis": "disabled",
"kms": "available",
"lambda": "available",
"logs": "available",
"opensearch": "disabled",
"redshift": "disabled",
"resource-groups": "disabled",
"resourcegroupstaggingapi": "disabled",
"route53": "disabled",
"route53resolver": "disabled",
"s3": "available",
"s3control": "disabled",
"scheduler": "disabled",
"secretsmanager": "available",
"ses": "disabled",
"sns": "disabled",
"sqs": "disabled",
"ssm": "disabled",
"stepfunctions": "available",
"sts": "available",
"support": "disabled",
"swf": "disabled",
"transcribe": "disabled"
},
"edition": "community",
"version": "4.13.1"
}
[stdout]
{
"services": {
"acm": "disabled",
"apigateway": "disabled",
"cloudformation": "disabled",
"cloudwatch": "disabled",
"config": "disabled",
"dynamodb": "disabled",
"dynamodbstreams": "disabled",
"ec2": "disabled",
"es": "disabled",
"events": "available",
"firehose": "disabled",
"iam": "available",
"kinesis": "disabled",
"kms": "available",
"lambda": "available",
"logs": "available",
"opensearch": "disabled",
"redshift": "disabled",
"resource-groups": "disabled",
"resourcegroupstaggingapi": "disabled",
"route53": "disabled",
"route53resolver": "disabled",
"s3": "available",
"s3control": "disabled",
"scheduler": "disabled",
"secretsmanager": "available",
"ses": "disabled",
"sns": "disabled",
"sqs": "disabled",
"ssm": "disabled",
"stepfunctions": "available",
"sts": "available",
"support": "disabled",
"swf": "disabled",
"transcribe": "disabled"
},
"edition": "community",
"version": "4.13.1"
}Create build directory structure
mkdir -p /app/build/rotator /app/build/notifier /app/policies /app/state-machines && ls /app
build policies state-machines [stdout] build policies state-machines
import json
import os
import secrets
import string
import uuid
import boto3
ENDPOINT = os.environ["AWS_ENDPOINT_URL"]
REGION = os.environ.get("AWS_REGION", "us-east-1")
SECRET_ID = os.environ["SECRET_ID"]
_sm = boto3.client("secretsmanager", endpoint_url=ENDPOINT, region_name=REGION)
def _generate(existing):
alphabet = string.ascii_letters + string.digits + "-_"
while True:
candidate = "".join(secrets.choice(alphabet) for _ in range(40))
if candidate != existing:
return candidate
def handler(event, context):
current_plain = None
try:
current = _sm.get_secret_value(SecretId=SECRET_ID, VersionStage="AWSCURRENT")
current_plain = json.loads(current["SecretString"]).get("password")
except Exception:
current_plain = None
new_password = _generate(current_plain)
new_value = json.dumps({"password": new_password})
_sm.put_secret_value(
SecretId=SECRET_ID,
ClientRequestToken=str(uuid.uuid4()),
SecretString=new_value,
VersionStages=["AWSCURRENT"],
)
return {"status": "rotated", "secret": SECRET_ID}
import json
def handler(event, context):
detail = event.get("detail", {}) if isinstance(event, dict) else {}
print("notified: " + json.dumps(detail))
return {"status": "ok"}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"Service": "lambda.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"Service": "states.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
#!/usr/bin/env bash
set -euo pipefail
export AWS_PAGER=""
export AWS_DEFAULT_REGION=us-east-1
ENDPOINT="http://localstack:4566"
AWS="aws --endpoint-url $ENDPOINT --region us-east-1"
ACCOUNT=$($AWS sts get-caller-identity --query Account --output text)
REGION=us-east-1
echo "account=$ACCOUNT"
BUILD=/app/build
POL=/app/policies
SMDIR=/app/state-machines
# ------------------------------------------------------------------
# 1. KMS customer-managed key + alias
# ------------------------------------------------------------------
echo ">>> KMS key"
KEY_POLICY=$(cat <<JSON
{
"Version": "2012-10-17",
"Id": "app-rotation-key",
"Statement": [
{
"Sid": "RootAccountAdmin",
"Effect": "Allow",
"Principal": {"AWS": "arn:aws:iam::${ACCOUNT}:root"},
"Action": "kms:*",
"Resource": "*"
}
]
}
JSON
)
KEY_ID=$($AWS kms create-key \
--description "app rotation CMK" \
--key-usage ENCRYPT_DECRYPT \
--key-spec SYMMETRIC_DEFAULT \
--policy "$KEY_POLICY" \
--query 'KeyMetadata.KeyId' --output text)
KEY_ARN="arn:aws:kms:${REGION}:${ACCOUNT}:key/${KEY_ID}"
echo "key_id=$KEY_ID"
$AWS kms create-alias --alias-name alias/app-rotation-key --target-key-id "$KEY_ID" >/dev/null
echo "alias=alias/app-rotation-key -> $KEY_ID"
# ------------------------------------------------------------------
# 2. Secret encrypted with the CMK
# ------------------------------------------------------------------
echo ">>> Secret"
INITIAL_VALUE='{"password":"initial-placeholder"}'
$AWS secretsmanager create-secret \
--name app/db/password \
--description "app db password" \
--kms-key-id alias/app-rotation-key \
--secret-string "$INITIAL_VALUE" >/dev/null
SECRET_ARN=$($AWS secretsmanager describe-secret --secret-id app/db/password --query ARN --output text)
echo "secret_arn=$SECRET_ARN"
# ------------------------------------------------------------------
# 3. IAM roles: lambda roles + state machine roles
# ------------------------------------------------------------------
echo ">>> IAM roles"
# --- rotator lambda role ---
$AWS iam create-role \
--role-name secrets-rotator-role \
--assume-role-policy-document "file://${POL}/trust-lambda.json" >/dev/null
ROTATOR_POLICY=$(cat <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/lambda/secrets-rotator:*"
},
{
"Sid": "SecretRotate",
"Effect": "Allow",
"Action": [
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:DescribeSecret",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "${SECRET_ARN}"
},
{
"Sid": "KmsForSecret",
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": "${KEY_ARN}"
}
]
}
JSON
)
$AWS iam put-role-policy \
--role-name secrets-rotator-role \
--policy-name secrets-rotator-inline \
--policy-document "$ROTATOR_POLICY" >/dev/null
ROTATOR_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/secrets-rotator-role"
# --- notifier lambda role ---
$AWS iam create-role \
--role-name notification-lambda-role \
--assume-role-policy-document "file://${POL}/trust-lambda.json" >/dev/null
NOTIFIER_POLICY=$(cat <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/lambda/notification-lambda:*"
}
]
}
JSON
)
$AWS iam put-role-policy \
--role-name notification-lambda-role \
--policy-name notification-lambda-inline \
--policy-document "$NOTIFIER_POLICY" >/dev/null
NOTIFIER_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/notification-lambda-role"
# ------------------------------------------------------------------
# 4. Package + create lambdas
# ------------------------------------------------------------------
echo ">>> Lambdas"
( cd ${BUILD}/rotator && zip -q -r /tmp/rotator.zip handler.py )
( cd ${BUILD}/notifier && zip -q -r /tmp/notifier.zip handler.py )
$AWS lambda create-function \
--function-name secrets-rotator \
--runtime python3.11 \
--role "$ROTATOR_ROLE_ARN" \
--handler handler.handler \
--zip-file fileb:///tmp/rotator.zip \
--timeout 30 \
--environment "Variables={AWS_ENDPOINT_URL=${ENDPOINT},SECRET_ID=app/db/password}" \
>/dev/null
ROTATOR_FN_ARN=$($AWS lambda get-function --function-name secrets-rotator --query 'Configuration.FunctionArn' --output text)
echo "rotator_arn=$ROTATOR_FN_ARN"
$AWS lambda create-function \
--function-name notification-lambda \
--runtime python3.11 \
--role "$NOTIFIER_ROLE_ARN" \
--handler handler.handler \
--zip-file fileb:///tmp/notifier.zip \
--timeout 15 \
>/dev/null
NOTIFIER_FN_ARN=$($AWS lambda get-function --function-name notification-lambda --query 'Configuration.FunctionArn' --output text)
echo "notifier_arn=$NOTIFIER_FN_ARN"
# Wait for both to be Active
for fn in secrets-rotator notification-lambda; do
for i in $(seq 1 30); do
state=$($AWS lambda get-function --function-name $fn --query 'Configuration.State' --output text)
[[ "$state" == "Active" ]] && break
sleep 1
done
done
# ------------------------------------------------------------------
# 5. EventBridge custom bus + rule + target + lambda resource policy
# ------------------------------------------------------------------
echo ">>> EventBridge bus + rule"
$AWS events create-event-bus --name rotation-events >/dev/null
BUS_ARN="arn:aws:events:${REGION}:${ACCOUNT}:event-bus/rotation-events"
$AWS events put-rule \
--name on-rotation-success \
--event-bus-name rotation-events \
--event-pattern '{"source":["rotation.pipeline"],"detail-type":["RotationComplete"]}' \
--state ENABLED >/dev/null
RULE_ARN="arn:aws:events:${REGION}:${ACCOUNT}:rule/rotation-events/on-rotation-success"
echo "rule_arn=$RULE_ARN"
# allow EventBridge to invoke the notifier
$AWS lambda add-permission \
--function-name notification-lambda \
--statement-id allow-eventbridge-on-rotation-success \
--action lambda:InvokeFunction \
--principal events.amazonaws.com \
--source-arn "$RULE_ARN" >/dev/null
$AWS events put-targets \
--rule on-rotation-success \
--event-bus-name rotation-events \
--targets "Id=1,Arn=${NOTIFIER_FN_ARN}" >/dev/null
# ------------------------------------------------------------------
# 6. Child state machine + role
# ------------------------------------------------------------------
echo ">>> Child SM"
$AWS iam create-role \
--role-name rotation-child-role \
--assume-role-policy-document "file://${POL}/trust-states.json" >/dev/null
CHILD_POLICY=$(cat <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "InvokeRotator",
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "${ROTATOR_FN_ARN}"
},
{
"Sid": "PutEventsOnBus",
"Effect": "Allow",
"Action": "events:PutEvents",
"Resource": "${BUS_ARN}"
}
]
}
JSON
)
$AWS iam put-role-policy \
--role-name rotation-child-role \
--policy-name rotation-child-inline \
--policy-document "$CHILD_POLICY" >/dev/null
CHILD_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/rotation-child-role"
CHILD_DEF=$(cat <<JSON
{
"Comment": "Rotate secret then publish RotationComplete event",
"StartAt": "Rotate",
"States": {
"Rotate": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "${ROTATOR_FN_ARN}",
"Payload": {}
},
"ResultPath": "\$.rotation",
"Next": "Notify"
},
"Notify": {
"Type": "Task",
"Resource": "arn:aws:states:::events:putEvents",
"Parameters": {
"Entries": [
{
"Source": "rotation.pipeline",
"DetailType": "RotationComplete",
"EventBusName": "rotation-events",
"Detail": "{\"secret\":\"app/db/password\"}"
}
]
},
"End": true
}
}
}
JSON
)
echo "$CHILD_DEF" > ${SMDIR}/child.json
CHILD_ARN=$($AWS stepfunctions create-state-machine \
--name rotation-child \
--type STANDARD \
--definition "$CHILD_DEF" \
--role-arn "$CHILD_ROLE_ARN" \
--query stateMachineArn --output text)
echo "child_arn=$CHILD_ARN"
# ------------------------------------------------------------------
# 7. Parent state machine + role (needs events:PutRule/PutTargets/DescribeRule)
# ------------------------------------------------------------------
echo ">>> Parent SM"
$AWS iam create-role \
--role-name rotation-orchestrator-role \
--assume-role-policy-document "file://${POL}/trust-states.json" >/dev/null
MANAGED_RULE_ARN="arn:aws:events:${REGION}:${ACCOUNT}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule"
PARENT_POLICY=$(cat <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "StartChild",
"Effect": "Allow",
"Action": "states:StartExecution",
"Resource": "${CHILD_ARN}"
},
{
"Sid": "MonitorChild",
"Effect": "Allow",
"Action": [
"states:DescribeExecution",
"states:StopExecution"
],
"Resource": "arn:aws:states:${REGION}:${ACCOUNT}:execution:rotation-child:*"
},
{
"Sid": "SyncManagedRule",
"Effect": "Allow",
"Action": [
"events:PutRule",
"events:PutTargets",
"events:DescribeRule"
],
"Resource": "${MANAGED_RULE_ARN}"
}
]
}
JSON
)
$AWS iam put-role-policy \
--role-name rotation-orchestrator-role \
--policy-name rotation-orchestrator-inline \
--policy-document "$PARENT_POLICY" >/dev/null
PARENT_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/rotation-orchestrator-role"
PARENT_DEF=$(cat <<JSON
{
"Comment": "Parent orchestrator; hands off to child synchronously",
"StartAt": "RunChild",
"States": {
"RunChild": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync:2",
"Parameters": {
"StateMachineArn": "${CHILD_ARN}",
"Input": {}
},
"End": true
}
}
}
JSON
)
echo "$PARENT_DEF" > ${SMDIR}/parent.json
PARENT_ARN=$($AWS stepfunctions create-state-machine \
--name rotation-orchestrator \
--type STANDARD \
--definition "$PARENT_DEF" \
--role-arn "$PARENT_ROLE_ARN" \
--query stateMachineArn --output text)
echo "parent_arn=$PARENT_ARN"
# Persist discovered ARNs for any follow-up tooling
cat > /app/state.env <<EOF
KEY_ID=${KEY_ID}
KEY_ARN=${KEY_ARN}
SECRET_ARN=${SECRET_ARN}
ROTATOR_FN_ARN=${ROTATOR_FN_ARN}
NOTIFIER_FN_ARN=${NOTIFIER_FN_ARN}
BUS_ARN=${BUS_ARN}
RULE_ARN=${RULE_ARN}
CHILD_ARN=${CHILD_ARN}
PARENT_ARN=${PARENT_ARN}
EOF
echo ">>> Done. State written to /app/state.env"
{
"query": "select:TodoWrite",
"max_results": 1
}
, output,
{"type": "tool_reference", "tool_name": "TodoWrite"}
Run setup script
chmod +x /app/setup.sh && /app/setup.sh 2>&1
account=000000000000 >>> KMS key key_id=7a6d2709-97f7-455d-b78c-3f6b1c362469 alias=alias/app-rotation-key -> 7a6d2709-97f7-455d-b78c-3f6b1c362469 >>> Secret secret_arn=arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-UcLqbc >>> IAM roles >>> Lambdas rotator_arn=arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator notifier_arn=arn:aws:lambda:us-east-1:000000000000:function:notification-lambda >>> EventBridge bus + rule rule_arn=arn:aws:events:us-east-1:000000000000:rule/rotation-events/on-rotation-success >>> Child SM child_arn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child >>> Parent SM parent_arn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator >>> Done. State written to /app/state.env [stdout] account=000000000000 >>> KMS key key_id=7a6d2709-97f7-455d-b78c-3f6b1c362469 alias=alias/app-rotation-key -> 7a6d2709-97f7-455d-b78c-3f6b1c362469 >>> Secret secret_arn=arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-UcLqbc >>> IAM roles >>> Lambdas rotator_arn=arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator notifier_arn=arn:aws:lambda:us-east-1:000000000000:function:notification-lambda >>> EventBridge bus + rule rule_arn=arn:aws:events:us-east-1:000000000000:rule/rotation-events/on-rotation-success >>> Child SM child_arn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child >>> Parent SM parent_arn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator >>> Done. State written to /app/state.env
Start parent SM and check rotation
set -e source /app/state.env export AWS_PAGER="" AWS="aws --endpoint-url http://localstack:4566 --region us-east-1" # Capture AWSCURRENT before rotation BEFORE=$($AWS secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSCURRENT --query SecretString --output text) BEFORE_VER=$($AWS secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSCURRENT --query VersionId --output text) echo "BEFORE_STRING=$BEFORE" echo "BEFORE_VERSION=$BEFORE_VER" # Start parent SM, no input EXEC_ARN=$($AWS stepfunctions start-execution --state-machine-arn "$PARENT_ARN" --query executionArn --output text) echo "exec=$EXEC_ARN" # Poll for completion up to 90s STATUS=RUNNING for i in $(seq 1 45); do STATUS=$($AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query status --output text) [[ "$STATUS" != "RUNNING" ]] && break sleep 2 done echo "parent_status=$STATUS (after $i polls)" if [[ "$STATUS" != "SUCCEEDED" ]]; then echo "--- execution detail ---" $AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" echo "--- history ---" $AWS stepfunctions get-execution-history --execution-arn "$EXEC_ARN" exit 1 fi # Check AWSCURRENT after AFTER=$($AWS secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSCURRENT --query SecretString --output text) AFTER_VER=$($AWS secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSCURRENT --query VersionId --output text) echo "AFTER_STRING=$AFTER" echo "AFTER_VERSION=$AFTER_VER" # Previous stage check PREV_VER=$($AWS secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSPREVIOUS --query VersionId --output text) echo "PREV_VERSION=$PREV_VER" if [[ "$BEFORE" == "$AFTER" ]]; then echo "FAIL: string unchanged"; exit 1; fi if [[ "$PREV_VER" != "$BEFORE_VER" ]]; then echo "FAIL: AWSPREVIOUS not set to old version"; exit 1; fi echo "secret rotation OK"
BEFORE_STRING={"password":"initial-placeholder"}
BEFORE_VERSION=4499150f-7a31-4931-b558-881ca7fb70dd
exec=arn:aws:states:us-east-1:000000000000:execution:rotation-orchestrator:30d134cf-d16d-4143-925c-e9f8aeaabff5
parent_status=SUCCEEDED (after 2 polls)
AFTER_STRING={"password": "Wzc0WUIqz7U8w8bjKgW4cEq94RuKe5KeS5Ox5bVC"}
AFTER_VERSION=bc2e926c-f0de-4f64-a457-a6927f1fb087
PREV_VERSION=4499150f-7a31-4931-b558-881ca7fb70dd
secret rotation OK
[stdout]
BEFORE_STRING={"password":"initial-placeholder"}
BEFORE_VERSION=4499150f-7a31-4931-b558-881ca7fb70dd
exec=arn:aws:states:us-east-1:000000000000:execution:rotation-orchestrator:30d134cf-d16d-4143-925c-e9f8aeaabff5
parent_status=SUCCEEDED (after 2 polls)
AFTER_STRING={"password": "Wzc0WUIqz7U8w8bjKgW4cEq94RuKe5KeS5Ox5bVC"}
AFTER_VERSION=bc2e926c-f0de-4f64-a457-a6927f1fb087
PREV_VERSION=4499150f-7a31-4931-b558-881ca7fb70dd
secret rotation OKCheck notifier lambda logs for notified: line
AWS="aws --endpoint-url http://localstack:4566 --region us-east-1"
export AWS_PAGER=""
# wait for log group to show up (EventBridge delivery is async)
for i in $(seq 1 30); do
LG=$($AWS logs describe-log-groups --log-group-name-prefix /aws/lambda/notification-lambda --query 'logGroups[0].logGroupName' --output text 2>/dev/null || echo "None")
[[ "$LG" != "None" && -n "$LG" ]] && break
sleep 1
done
echo "log_group=$LG"
for i in $(seq 1 30); do
STREAMS=$($AWS logs describe-log-streams --log-group-name "$LG" --query 'logStreams[].logStreamName' --output text 2>/dev/null || echo "")
[[ -n "$STREAMS" ]] && break
sleep 1
done
echo "streams=$STREAMS"
FOUND=""
for s in $STREAMS; do
MSGS=$($AWS logs get-log-events --log-group-name "$LG" --log-stream-name "$s" --query 'events[].message' --output text)
echo "--- stream $s ---"
echo "$MSGS"
if echo "$MSGS" | grep -q '^notified:'; then
FOUND=1
fi
done
[[ -n "$FOUND" ]] && echo "OK: notifier logged a 'notified:' line" || { echo "FAIL: no 'notified:' line"; exit 1; }
Exit code 1
log_group=/aws/lambda/notification-lambda
streams=2026/04/22/[$LATEST]07707ff62ed2d9f5996200134f2a427f
--- stream 2026/04/22/[$LATEST]07707ff62ed2d9f5996200134f2a427f ---
START RequestId: 8026397f-e84e-4359-bc8b-c35a8a10e580 Version: $LATEST
notified: {"secret": "app/db/password"}
END RequestId: 8026397f-e84e-4359-bc8b-c35a8a10e580
REPORT RequestId: 8026397f-e84e-4359-bc8b-c35a8a10e580 Duration: 9.01 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
FAIL: no 'notified:' line
[error] tool reported failureConfirm notified: log line per-message
AWS="aws --endpoint-url http://localstack:4566 --region us-east-1"
export AWS_PAGER=""
LG=/aws/lambda/notification-lambda
STREAM=$($AWS logs describe-log-streams --log-group-name "$LG" --query 'logStreams[0].logStreamName' --output text)
$AWS logs get-log-events --log-group-name "$LG" --log-stream-name "$STREAM" --output json \
| jq -r '.events[].message' | while IFS= read -r line; do
if [[ "$line" == notified:* ]]; then
echo "MATCH: $line"
fi
doneMATCH: notified: {"secret": "app/db/password"}
[stdout]
MATCH: notified: {"secret": "app/db/password"}Second rotation plus policy dump
set -e
source /app/state.env
AWS="aws --endpoint-url http://localstack:4566 --region us-east-1"
export AWS_PAGER=""
BEFORE=$($AWS secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSCURRENT --query SecretString --output text)
EXEC_ARN=$($AWS stepfunctions start-execution --state-machine-arn "$PARENT_ARN" --query executionArn --output text)
for i in $(seq 1 45); do
STATUS=$($AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query status --output text)
[[ "$STATUS" != "RUNNING" ]] && break
sleep 2
done
AFTER=$($AWS secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSCURRENT --query SecretString --output text)
echo "second_status=$STATUS"
echo "before=$BEFORE"
echo "after=$AFTER"
[[ "$BEFORE" != "$AFTER" ]] && echo "OK: second rotation produced a different string"
echo
echo "===== inline policies (as the verifier would read them) ====="
for r in secrets-rotator-role notification-lambda-role rotation-child-role rotation-orchestrator-role; do
PN=$($AWS iam list-role-policies --role-name $r --query 'PolicyNames[0]' --output text)
echo "--- $r / $PN ---"
$AWS iam get-role-policy --role-name $r --policy-name $PN --query PolicyDocument
done
echo
echo "===== wildcard scan in Resources (excluding key-policy root admin which is default) ====="
for r in secrets-rotator-role notification-lambda-role rotation-child-role rotation-orchestrator-role; do
PN=$($AWS iam list-role-policies --role-name $r --query 'PolicyNames[0]' --output text)
DOC=$($AWS iam get-role-policy --role-name $r --policy-name $PN --query PolicyDocument --output json)
WC=$(echo "$DOC" | jq -r '.Statement[] | select(.Resource=="*") | .Sid // "unnamed"')
if [[ -n "$WC" ]]; then
echo "WARN $r has wildcard Resource on: $WC"
else
echo "OK $r: no wildcard Resource"
fi
done
second_status=SUCCEEDED
before={"password": "Wzc0WUIqz7U8w8bjKgW4cEq94RuKe5KeS5Ox5bVC"}
after={"password": "RxOZVhhe2ITjAvMacvU0Zdh6I9l2uY-VU0gfcRrN"}
OK: second rotation produced a different string
===== inline policies (as the verifier would read them) =====
--- secrets-rotator-role / secrets-rotator-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/secrets-rotator:*"
},
{
"Sid": "SecretRotate",
"Effect": "Allow",
"Action": [
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:DescribeSecret",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-UcLqbc"
},
{
"Sid": "KmsForSecret",
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": "arn:aws:kms:us-east-1:000000000000:key/7a6d2709-97f7-455d-b78c-3f6b1c362469"
}
]
}
--- notification-lambda-role / notification-lambda-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/notification-lambda:*"
}
]
}
--- rotation-child-role / rotation-child-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "InvokeRotator",
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator"
},
{
"Sid": "PutEventsOnBus",
"Effect": "Allow",
"Action": "events:PutEvents",
"Resource": "arn:aws:events:us-east-1:000000000000:event-bus/rotation-events"
}
]
}
--- rotation-orchestrator-role / rotation-orchestrator-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "StartChild",
"Effect": "Allow",
"Action": "states:StartExecution",
"Resource": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child"
},
{
"Sid": "MonitorChild",
"Effect": "Allow",
"Action": [
"states:DescribeExecution",
"states:StopExecution"
],
"Resource": "arn:aws:states:us-east-1:000000000000:execution:rotation-child:*"
},
{
"Sid": "SyncManagedRule",
"Effect": "Allow",
"Action": [
"events:PutRule",
"events:PutTargets",
"events:DescribeRule"
],
"Resource": "arn:aws:events:us-east-1:000000000000:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule"
}
]
}
===== wildcard scan in Resources (excluding key-policy root admin which is default) =====
OK secrets-rotator-role: no wildcard Resource
OK notification-lambda-role: no wildcard Resource
OK rotation-child-role: no wildcard Resource
OK rotation-orchestrator-role: no wildcard Resource
[stdout]
second_status=SUCCEEDED
before={"password": "Wzc0WUIqz7U8w8bjKgW4cEq94RuKe5KeS5Ox5bVC"}
after={"password": "RxOZVhhe2ITjAvMacvU0Zdh6I9l2uY-VU0gfcRrN"}
OK: second rotation produced a different string
===== inline policies (as the verifier would read them) =====
--- secrets-rotator-role / secrets-rotator-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/secrets-rotator:*"
},
{
"Sid": "SecretRotate",
"Effect": "Allow",
"Action": [
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:DescribeSecret",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-UcLqbc"
},
{
"Sid": "KmsForSecret",
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": "arn:aws:kms:us-east-1:000000000000:key/7a6d2709-97f7-455d-b78c-3f6b1c362469"
}
]
}
--- notification-lambda-role / notification-lambda-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/notification-lambda:*"
}
]
}
--- rotation-child-role / rotation-child-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "InvokeRotator",
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator"
},
{
"Sid": "PutEventsOnBus",
"Effect": "Allow",
"Action": "events:PutEvents",
"Resource": "arn:aws:events:us-east-1:000000000000:event-bus/rotation-events"
}
]
}
--- rotation-orchestrator-role / rotation-orchestrator-inline ---
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "StartChild",
"Effect": "Allow",
"Action": "states:StartExecution",
"Resource": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child"
},
{
"Sid": "MonitorChild",
"Effect": "Allow",
"Action": [
"states:DescribeExecution",
"states:StopExecution"
],
"Resource": "arn:aws:states:us-east-1:000000000000:execution:rotation-child:*"
},
{
"Sid": "SyncManagedRule",
"Effect": "Allow",
"Action": [
"events:PutRule",
"events:PutTargets",
"events:DescribeRule"
],
"Resource": "arn:aws:events:us-east-1:000000000000:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule"
}
]
}
===== wildcard scan in Resources (excluding key-policy root admin which is default) =====
OK secrets-rotator-role: no wildcard Resource
OK notification-lambda-role: no wildcard Resource
OK rotation-child-role: no wildcard Resource
OK rotation-orchestrator-role: no wildcard ResourceNo file changes, the agent left the workspace untouched.
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0 -- /root/.cache/uv/archive-v0/yZNHavQahMp7XazqxSOlo/bin/python
cachedir: .pytest_cache
rootdir: /tests
plugins: json-ctrf-0.3.5
collecting ... collected 15 items
../tests/test_state.py::test_aws_cli_binary_is_real PASSED [ 6%]
../tests/test_state.py::test_localstack_reachable PASSED [ 13%]
../tests/test_state.py::test_secret_exists_and_uses_customer_cmk PASSED [ 20%]
../tests/test_state.py::test_rotation_lambda_exists PASSED [ 26%]
../tests/test_state.py::test_notification_lambda_exists PASSED [ 33%]
../tests/test_state.py::test_parent_and_child_state_machines_exist PASSED [ 40%]
../tests/test_state.py::test_event_bus_and_rule_exist PASSED [ 46%]
../tests/test_state.py::test_rule_target_points_at_notification_lambda PASSED [ 53%]
../tests/test_state.py::test_rotator_role_grants_kms_actions PASSED [ 60%]
../tests/test_state.py::test_kms_key_policy_grants_rotator_role FAILED [ 66%]
../tests/test_state.py::test_rotator_lambda_allows_secretsmanager_invoke FAILED [ 73%]
../tests/test_state.py::test_parent_role_grants_sync2_managed_rule_perms FAILED [ 80%]
../tests/test_state.py::test_child_role_can_rotate_and_publish FAILED [ 86%]
../tests/test_state.py::test_notification_lambda_allows_eventbridge_invoke PASSED [ 93%]
../tests/test_state.py::test_end_to_end_parent_execution_rotates_and_notifies PASSED [100%]
=================================== FAILURES ===================================
___________________ test_kms_key_policy_grants_rotator_role ____________________
iam = <botocore.client.IAM object at 0xffff97bfdca0>
kms = <botocore.client.KMS object at 0xffff981481d0>
def test_kms_key_policy_grants_rotator_role(iam, kms):
role_arn = iam.get_role(RoleName=ROTATOR_ROLE)["Role"]["Arn"]
policy_str = kms.get_key_policy(KeyId=KEY_ALIAS, PolicyName="default")[
"Policy"
]
policy = json.loads(policy_str)
match = False
for st in policy.get("Statement", []):
if st.get("Effect") != "Allow":
continue
principal = st.get("Principal") or {}
if not isinstance(principal, dict):
continue
arns = set(_normalise_list(principal.get("AWS")))
if role_arn not in arns:
continue
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, REQUIRED_KMS_ACTIONS):
match = True
break
> assert match, (
f"KMS key policy on {KEY_ALIAS} has no Allow statement whose "
f"Principal.AWS includes {role_arn} and whose Action covers "
f"{sorted(REQUIRED_KMS_ACTIONS)}. Key policy: {policy_str}"
)
E AssertionError: KMS key policy on alias/app-rotation-key has no Allow statement whose Principal.AWS includes arn:aws:iam::000000000000:role/secrets-rotator-role and whose Action covers ['kms:Decrypt', 'kms:GenerateDataKey']. Key policy: {
E "Version": "2012-10-17",
E "Id": "app-rotation-key",
E "Statement": [
E {
E "Sid": "RootAccountAdmin",
E "Effect": "Allow",
E "Principal": {"AWS": "arn:aws:iam::000000000000:root"},
E "Action": "kms:*",
E "Resource": "*"
E }
E ]
E }
E assert False
/tests/test_state.py:322: AssertionError
_______________ test_rotator_lambda_allows_secretsmanager_invoke _______________
lmb = <botocore.client.Lambda object at 0xffff98149b80>
def test_rotator_lambda_allows_secretsmanager_invoke(lmb):
try:
policy_doc = json.loads(
> lmb.get_policy(FunctionName=ROTATOR_FUNCTION)["Policy"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
/tests/test_state.py:332:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/root/.cache/uv/archive-v0/yZNHavQahMp7XazqxSOlo/lib/python3.12/site-packages/botocore/client.py:569: in _api_call
return self._make_api_call(operation_name, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <botocore.client.Lambda object at 0xffff98149b80>
operation_name = 'GetPolicy', api_params = {'FunctionName': 'secrets-rotator'}
def _make_api_call(self, operation_name, api_params):
operation_model = self._service_model.operation_model(operation_name)
service_name = self._service_model.service_name
history_recorder.record(
'API_CALL',
{
'service': service_name,
'operation': operation_name,
'params': api_params,
},
)
if operation_model.deprecated:
logger.debug(
'Warning: %s.%s() is deprecated', service_name, operation_name
)
request_context = {
'client_region': self.meta.region_name,
'client_config': self.meta.config,
'has_streaming_input': operation_model.has_streaming_input,
'auth_type': operation_model.resolved_auth_type,
'unsigned_payload': operation_model.unsigned_payload,
}
api_params = self._emit_api_params(
api_params=api_params,
operation_model=operation_model,
context=request_context,
)
(
endpoint_url,
additional_headers,
properties,
) = self._resolve_endpoint_ruleset(
operation_model, api_params, request_context
)
if properties:
# Pass arbitrary endpoint info with the Request
# for use during construction.
request_context['endpoint_properties'] = properties
request_dict = self._convert_to_request_dict(
api_params=api_params,
operation_model=operation_model,
endpoint_url=endpoint_url,
context=request_context,
headers=additional_headers,
)
resolve_checksum_context(request_dict, operation_model, api_params)
service_id = self._service_model.service_id.hyphenize()
handler, event_response = self.meta.events.emit_until_response(
f'before-call.{service_id}.{operation_name}',
model=operation_model,
params=request_dict,
request_signer=self._request_signer,
context=request_context,
)
if event_response is not None:
http, parsed_response = event_response
else:
maybe_compress_request(
self.meta.config, request_dict, operation_model
)
apply_request_checksum(request_dict)
http, parsed_response = self._make_request(
operation_model, request_dict, request_context
)
self.meta.events.emit(
f'after-call.{service_id}.{operation_name}',
http_response=http,
parsed=parsed_response,
model=operation_model,
context=request_context,
)
if http.status_code >= 300:
error_info = parsed_response.get("Error", {})
error_code = error_info.get("QueryErrorCode") or error_info.get(
"Code"
)
error_class = self.exceptions.from_code(error_code)
> raise error_class(parsed_response, operation_name)
E botocore.errorfactory.ResourceNotFoundException: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
/root/.cache/uv/archive-v0/yZNHavQahMp7XazqxSOlo/lib/python3.12/site-packages/botocore/client.py:1023: ResourceNotFoundException
During handling of the above exception, another exception occurred:
lmb = <botocore.client.Lambda object at 0xffff98149b80>
def test_rotator_lambda_allows_secretsmanager_invoke(lmb):
try:
policy_doc = json.loads(
lmb.get_policy(FunctionName=ROTATOR_FUNCTION)["Policy"]
)
except ClientError as e:
> pytest.fail(
f"Lambda {ROTATOR_FUNCTION} has no resource-based policy , "
f"secretsmanager.amazonaws.com cannot invoke it: {e}"
)
E Failed: Lambda secrets-rotator has no resource-based policy , secretsmanager.amazonaws.com cannot invoke it: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
/tests/test_state.py:335: Failed
_______________ test_parent_role_grants_sync2_managed_rule_perms _______________
iam = <botocore.client.IAM object at 0xffff97bfdca0>
def test_parent_role_grants_sync2_managed_rule_perms(iam):
"""The .sync:2 integration registers an EventBridge-managed callback
rule. Parent role needs events:PutRule/PutTargets/DescribeRule plus
full states:* on the child."""
found_events = False
found_states = False
for _pname, st in _inline_statements(iam, PARENT_ROLE):
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, REQUIRED_PARENT_EVENTS_ACTIONS):
found_events = True
if _actions_cover(actions, REQUIRED_PARENT_STATES_ACTIONS):
found_states = True
assert found_events, (
f"{PARENT_ROLE} missing events:PutRule/PutTargets/DescribeRule. "
f".sync:2 uses an EventBridge-managed callback rule; without "
f"these the parent cannot set up the callback."
)
> assert found_states, (
f"{PARENT_ROLE} missing states:StartExecution/DescribeExecution/"
f"StopExecution. .sync:2 requires all three on the child SM."
)
E AssertionError: rotation-orchestrator-role missing states:StartExecution/DescribeExecution/StopExecution. .sync:2 requires all three on the child SM.
E assert False
/tests/test_state.py:377: AssertionError
____________________ test_child_role_can_rotate_and_publish ____________________
iam = <botocore.client.IAM object at 0xffff97bfdca0>
def test_child_role_can_rotate_and_publish(iam):
found_rotate = False
found_putevents = False
for _pname, st in _inline_statements(iam, CHILD_ROLE):
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, {"secretsmanager:RotateSecret"}):
found_rotate = True
if _actions_cover(actions, {"events:PutEvents"}):
found_putevents = True
> assert found_rotate, (
f"{CHILD_ROLE} lacks secretsmanager:RotateSecret , the child "
f"state machine cannot trigger rotation."
)
E AssertionError: rotation-child-role lacks secretsmanager:RotateSecret , the child state machine cannot trigger rotation.
E assert False
/tests/test_state.py:392: AssertionError
=============================== warnings summary ===============================
test_state.py: 34 warnings
/root/.cache/uv/archive-v0/yZNHavQahMp7XazqxSOlo/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
==================================== PASSES ====================================
=========================== short test summary info ============================
PASSED ../tests/test_state.py::test_aws_cli_b
… (truncated at 12,000 chars, full verifier log is in the trial artifacts)Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_2033a492a9c94be1. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_2033a492a9c94be1 · verifier authoritative; classifier explanatory.