tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.Check localstack health
curl -s http://localstack:4566/_localstack/health | head -c 2000
{"services": {"acm": "disabled", "apigateway": "disabled", "cloudformation": "disabled", "cloudwatch": "disabled", "config": "disabled", "dynamodb": "disabled", "dynamodbstreams": "disabled", "ec2": "disabled", "es": "disabled", "events": "available", "firehose": "disabled", "iam": "available", "kinesis": "disabled", "kms": "available", "lambda": "available", "logs": "available", "opensearch": "disabled", "redshift": "disabled", "resource-groups": "disabled", "resourcegroupstaggingapi": "disabled", "route53": "disabled", "route53resolver": "disabled", "s3": "available", "s3control": "disabled", "scheduler": "disabled", "secretsmanager": "available", "ses": "disabled", "sns": "disabled", "sqs": "disabled", "ssm": "disabled", "stepfunctions": "available", "sts": "available", "support": "disabled", "swf": "disabled", "transcribe": "disabled"}, "edition": "community", "version": "4.13.1"}
[stdout]
{"services": {"acm": "disabled", "apigateway": "disabled", "cloudformation": "disabled", "cloudwatch": "disabled", "config": "disabled", "dynamodb": "disabled", "dynamodbstreams": "disabled", "ec2": "disabled", "es": "disabled", "events": "available", "firehose": "disabled", "iam": "available", "kinesis": "disabled", "kms": "available", "lambda": "available", "logs": "available", "opensearch": "disabled", "redshift": "disabled", "resource-groups": "disabled", "resourcegroupstaggingapi": "disabled", "route53": "disabled", "route53resolver": "disabled", "s3": "available", "s3control": "disabled", "scheduler": "disabled", "secretsmanager": "available", "ses": "disabled", "sns": "disabled", "sqs": "disabled", "ssm": "disabled", "stepfunctions": "available", "sts": "available", "support": "disabled", "swf": "disabled", "transcribe": "disabled"}, "edition": "community", "version": "4.13.1"}Check workdir and tools
ls -la /app && echo "---" && aws --version && python3 --version && which jq zip
total 8 drwxr-xr-x 2 root root 4096 Apr 21 20:05 . drwxr-xr-x 1 root root 4096 Apr 22 06:14 .. --- aws-cli/2.34.33 Python/3.14.4 Linux/6.10.14-linuxkit exe/aarch64.ubuntu.24 Python 3.12.3 /usr/bin/jq /usr/bin/zip [stdout] total 8 drwxr-xr-x 2 root root 4096 Apr 21 20:05 . drwxr-xr-x 1 root root 4096 Apr 22 06:14 .. --- aws-cli/2.34.33 Python/3.14.4 Linux/6.10.14-linuxkit exe/aarch64.ubuntu.24 Python 3.12.3 /usr/bin/jq /usr/bin/zip
Confirm the box is empty
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1" && $AWS secretsmanager list-secrets --query 'SecretList[].Name' && $AWS stepfunctions list-state-machines --query 'stateMachines[].name' && $AWS lambda list-functions --query 'Functions[].FunctionName' && $AWS events list-event-buses --query 'EventBuses[].Name' && $AWS kms list-aliases --query 'Aliases[?starts_with(AliasName, `alias/app`)].AliasName'
[]
[]
[]
[
"default"
]
[]
[stdout]
[]
[]
[]
[
"default"
]
[]Create directory structure
mkdir -p /app/lambdas/rotator /app/lambdas/notification /app/iam /app/sm
(Bash completed with no output)
import os
import secrets
import string
import boto3
ENDPOINT = os.environ.get("AWS_ENDPOINT_URL", "http://localstack:4566")
REGION = os.environ.get("AWS_REGION", "us-east-1")
SECRET_ID = "app/db/password"
_client = boto3.client("secretsmanager", endpoint_url=ENDPOINT, region_name=REGION)
def _generate_password(length: int = 32) -> str:
alphabet = string.ascii_letters + string.digits + "-_"
return "".join(secrets.choice(alphabet) for _ in range(length))
def handler(event, context):
new_password = _generate_password()
resp = _client.put_secret_value(SecretId=SECRET_ID, SecretString=new_password)
return {"VersionId": resp.get("VersionId", ""), "Name": resp.get("Name", SECRET_ID)}
import json
def handler(event, context):
print("notified: " + json.dumps(event, default=str))
return {"ok": True}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "Service": "lambda.amazonaws.com" },
"Action": "sts:AssumeRole"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "Service": "states.amazonaws.com" },
"Action": "sts:AssumeRole"
}
]
}
{
"query": "select:TodoWrite",
"max_results": 1
}
, output,
{"type": "tool_reference", "tool_name": "TodoWrite"}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "RotateSecret",
"Effect": "Allow",
"Action": [
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:UpdateSecretVersionStage",
"secretsmanager:DescribeSecret"
],
"Resource": "__SECRET_ARN__"
},
{
"Sid": "UseCustomerKmsKey",
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": "__KMS_KEY_ARN__"
},
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/secrets-rotator:*"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/notification-lambda:*"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "InvokeRotatorLambda",
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "__ROTATOR_ARN__"
},
{
"Sid": "PutRotationEvents",
"Effect": "Allow",
"Action": "events:PutEvents",
"Resource": "__BUS_ARN__"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "StartChildExecution",
"Effect": "Allow",
"Action": "states:StartExecution",
"Resource": "__CHILD_SM_ARN__"
},
{
"Sid": "ObserveChildExecutions",
"Effect": "Allow",
"Action": [
"states:DescribeExecution",
"states:StopExecution"
],
"Resource": "arn:aws:states:us-east-1:000000000000:execution:rotation-child:*"
},
{
"Sid": "ManageSyncCallbackRule",
"Effect": "Allow",
"Action": [
"events:PutRule",
"events:PutTargets",
"events:DescribeRule"
],
"Resource": "arn:aws:events:us-east-1:000000000000:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule"
}
]
}
{
"Comment": "Rotates the app secret, then emits RotationComplete to the custom bus",
"StartAt": "Rotate",
"States": {
"Rotate": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "__ROTATOR_ARN__",
"Payload": {}
},
"ResultPath": "$.rotate",
"Next": "Notify"
},
"Notify": {
"Type": "Task",
"Resource": "arn:aws:states:::events:putEvents",
"Parameters": {
"Entries": [
{
"Source": "rotation.pipeline",
"DetailType": "RotationComplete",
"EventBusName": "rotation-events",
"Detail": {
"status": "ok",
"secretId": "app/db/password"
}
}
]
},
"End": true
}
}
}
{
"Comment": "Entry-point orchestrator. Hands off to rotation-child via sync:2.",
"StartAt": "InvokeChild",
"States": {
"InvokeChild": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync:2",
"Parameters": {
"StateMachineArn": "__CHILD_SM_ARN__"
},
"End": true
}
}
}
{
"source": ["rotation.pipeline"],
"detail-type": ["RotationComplete"]
}
#!/usr/bin/env bash
set -euo pipefail
ENDPOINT="${ENDPOINT:-http://localstack:4566}"
REGION="${AWS_REGION:-us-east-1}"
ACCOUNT="000000000000"
AWSC="aws --endpoint-url=$ENDPOINT --region $REGION"
ROOT="$(cd "$(dirname "$0")" && pwd)"
say() { printf '\n\033[1;36m== %s ==\033[0m\n' "$*"; }
say "KMS customer-managed key + alias"
KEY_ID=$($AWSC kms create-key \
--description "CMK for app secret rotation pipeline" \
--key-usage ENCRYPT_DECRYPT \
--key-spec SYMMETRIC_DEFAULT \
--query KeyMetadata.KeyId --output text)
KEY_ARN="arn:aws:kms:${REGION}:${ACCOUNT}:key/${KEY_ID}"
$AWSC kms create-alias --alias-name alias/app-rotation-key --target-key-id "$KEY_ID"
echo "KeyId=$KEY_ID"
echo "KeyArn=$KEY_ARN"
say "Secret encrypted with CMK"
$AWSC secretsmanager create-secret \
--name app/db/password \
--kms-key-id alias/app-rotation-key \
--secret-string "initial-placeholder-$(date +%s%N)" \
>/dev/null
SECRET_ARN=$($AWSC secretsmanager describe-secret --secret-id app/db/password --query ARN --output text)
echo "SecretArn=$SECRET_ARN"
say "Rotator lambda role + least-privilege policy"
$AWSC iam create-role \
--role-name secrets-rotator-role \
--assume-role-policy-document "file://$ROOT/iam/lambda-trust.json" >/dev/null
sed -e "s|__SECRET_ARN__|$SECRET_ARN|" -e "s|__KMS_KEY_ARN__|$KEY_ARN|" \
"$ROOT/iam/secrets-rotator-policy.tmpl.json" > "$ROOT/iam/secrets-rotator-policy.json"
$AWSC iam put-role-policy \
--role-name secrets-rotator-role \
--policy-name secrets-rotator-inline \
--policy-document "file://$ROOT/iam/secrets-rotator-policy.json"
ROTATOR_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/secrets-rotator-role"
say "Notification lambda role"
$AWSC iam create-role \
--role-name notification-lambda-role \
--assume-role-policy-document "file://$ROOT/iam/lambda-trust.json" >/dev/null
$AWSC iam put-role-policy \
--role-name notification-lambda-role \
--policy-name notification-lambda-inline \
--policy-document "file://$ROOT/iam/notification-lambda-policy.json"
NOTIF_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/notification-lambda-role"
say "Package lambdas"
( cd "$ROOT/lambdas/rotator" && rm -f lambda.zip && zip -q lambda.zip lambda_function.py )
( cd "$ROOT/lambdas/notification" && rm -f lambda.zip && zip -q lambda.zip lambda_function.py )
say "Deploy rotator lambda"
$AWSC lambda create-function \
--function-name secrets-rotator \
--runtime python3.11 \
--role "$ROTATOR_ROLE_ARN" \
--handler lambda_function.handler \
--zip-file "fileb://$ROOT/lambdas/rotator/lambda.zip" \
--environment "Variables={AWS_ENDPOINT_URL=$ENDPOINT}" \
--timeout 30 >/dev/null
$AWSC lambda wait function-active --function-name secrets-rotator
ROTATOR_ARN=$($AWSC lambda get-function --function-name secrets-rotator --query Configuration.FunctionArn --output text)
echo "RotatorArn=$ROTATOR_ARN"
say "Deploy notification lambda"
$AWSC lambda create-function \
--function-name notification-lambda \
--runtime python3.11 \
--role "$NOTIF_ROLE_ARN" \
--handler lambda_function.handler \
--zip-file "fileb://$ROOT/lambdas/notification/lambda.zip" \
--timeout 30 >/dev/null
$AWSC lambda wait function-active --function-name notification-lambda
NOTIF_ARN=$($AWSC lambda get-function --function-name notification-lambda --query Configuration.FunctionArn --output text)
echo "NotificationArn=$NOTIF_ARN"
say "EventBridge bus + rule + target"
$AWSC events create-event-bus --name rotation-events >/dev/null
BUS_ARN="arn:aws:events:${REGION}:${ACCOUNT}:event-bus/rotation-events"
echo "BusArn=$BUS_ARN"
$AWSC events put-rule \
--name on-rotation-success \
--event-bus-name rotation-events \
--event-pattern "file://$ROOT/iam/rule-bus-pattern.json" \
--state ENABLED >/dev/null
RULE_ARN=$($AWSC events describe-rule --name on-rotation-success --event-bus-name rotation-events --query Arn --output text)
echo "RuleArn=$RULE_ARN"
$AWSC events put-targets \
--rule on-rotation-success \
--event-bus-name rotation-events \
--targets "Id=notif,Arn=$NOTIF_ARN" >/dev/null
$AWSC lambda add-permission \
--function-name notification-lambda \
--statement-id allow-eventbridge-rotation \
--action lambda:InvokeFunction \
--principal events.amazonaws.com \
--source-arn "$RULE_ARN" >/dev/null
say "Child SM role + policy"
$AWSC iam create-role \
--role-name rotation-child-role \
--assume-role-policy-document "file://$ROOT/iam/states-trust.json" >/dev/null
sed -e "s|__ROTATOR_ARN__|$ROTATOR_ARN|" -e "s|__BUS_ARN__|$BUS_ARN|" \
"$ROOT/iam/rotation-child-policy.tmpl.json" > "$ROOT/iam/rotation-child-policy.json"
$AWSC iam put-role-policy \
--role-name rotation-child-role \
--policy-name rotation-child-inline \
--policy-document "file://$ROOT/iam/rotation-child-policy.json"
CHILD_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/rotation-child-role"
say "Create child SM (rotation-child)"
sed -e "s|__ROTATOR_ARN__|$ROTATOR_ARN|" "$ROOT/sm/child.tmpl.json" > "$ROOT/sm/child.json"
CHILD_SM_ARN=$($AWSC stepfunctions create-state-machine \
--name rotation-child \
--definition "file://$ROOT/sm/child.json" \
--role-arn "$CHILD_ROLE_ARN" \
--type STANDARD \
--query stateMachineArn --output text)
echo "ChildSmArn=$CHILD_SM_ARN"
say "Parent SM role + policy"
$AWSC iam create-role \
--role-name rotation-orchestrator-role \
--assume-role-policy-document "file://$ROOT/iam/states-trust.json" >/dev/null
sed -e "s|__CHILD_SM_ARN__|$CHILD_SM_ARN|" \
"$ROOT/iam/rotation-orchestrator-policy.tmpl.json" > "$ROOT/iam/rotation-orchestrator-policy.json"
$AWSC iam put-role-policy \
--role-name rotation-orchestrator-role \
--policy-name rotation-orchestrator-inline \
--policy-document "file://$ROOT/iam/rotation-orchestrator-policy.json"
PARENT_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/rotation-orchestrator-role"
say "Create parent SM (rotation-orchestrator)"
sed -e "s|__CHILD_SM_ARN__|$CHILD_SM_ARN|" "$ROOT/sm/parent.tmpl.json" > "$ROOT/sm/parent.json"
PARENT_SM_ARN=$($AWSC stepfunctions create-state-machine \
--name rotation-orchestrator \
--definition "file://$ROOT/sm/parent.json" \
--role-arn "$PARENT_ROLE_ARN" \
--type STANDARD \
--query stateMachineArn --output text)
echo "ParentSmArn=$PARENT_SM_ARN"
say "Summary"
cat <<EOF
kms alias: alias/app-rotation-key (key: $KEY_ARN)
secret: $SECRET_ARN
rotator lambda: $ROTATOR_ARN
notification lambda: $NOTIF_ARN
event bus: $BUS_ARN
rule: $RULE_ARN
child SM: $CHILD_SM_ARN
parent SM: $PARENT_SM_ARN
EOF
Run the setup script
chmod +x /app/setup.sh && /app/setup.sh
[1;36m== KMS customer-managed key + alias ==[0m KeyId=f5165b9e-8ae8-48dc-90eb-da742d06c0e6 KeyArn=arn:aws:kms:us-east-1:000000000000:key/f5165b9e-8ae8-48dc-90eb-da742d06c0e6 [1;36m== Secret encrypted with CMK ==[0m SecretArn=arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-kStXlH [1;36m== Rotator lambda role + least-privilege policy ==[0m [1;36m== Notification lambda role ==[0m [1;36m== Package lambdas ==[0m [1;36m== Deploy rotator lambda ==[0m RotatorArn=arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator [1;36m== Deploy notification lambda ==[0m NotificationArn=arn:aws:lambda:us-east-1:000000000000:function:notification-lambda [1;36m== EventBridge bus + rule + target ==[0m BusArn=arn:aws:events:us-east-1:000000000000:event-bus/rotation-events RuleArn=arn:aws:events:us-east-1:000000000000:rule/rotation-events/on-rotation-success [1;36m== Child SM role + policy ==[0m [1;36m== Create child SM (rotation-child) ==[0m ChildSmArn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child [1;36m== Parent SM role + policy ==[0m [1;36m== Create parent SM (rotation-orchestrator) ==[0m ParentSmArn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator [1;36m== Summary ==[0m kms alias: alias/app-rotation-key (key: arn:aws:kms:us-east-1:000000000000:key/f5165b9e-8ae8-48dc-90eb-da742d06c0e6) secret: arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-kStXlH rotator lambda: arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator notification lambda: arn:aws:lambda:us-east-1:000000000000:function:notification-lambda event bus: arn:aws:events:us-east-1:000000000000:event-bus/rotation-events rule: arn:aws:events:us-east-1:000000000000:rule/rotation-events/on-rotation-success child SM: arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child parent SM: arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator [stdout] [1;36m== KMS customer-managed key + alias ==[0m KeyId=f5165b9e-8ae8-48dc-90eb-da742d06c0e6 KeyArn=arn:aws:kms:us-east-1:000000000000:key/f5165b9e-8ae8-48dc-90eb-da742d06c0e6 [1;36m== Secret encrypted with CMK ==[0m SecretArn=arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-kStXlH [1;36m== Rotator lambda role + least-privilege policy ==[0m [1;36m== Notification lambda role ==[0m [1;36m== Package lambdas ==[0m [1;36m== Deploy rotator lambda ==[0m RotatorArn=arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator [1;36m== Deploy notification lambda ==[0m NotificationArn=arn:aws:lambda:us-east-1:000000000000:function:notification-lambda [1;36m== EventBridge bus + rule + target ==[0m BusArn=arn:aws:events:us-east-1:000000000000:event-bus/rotation-events RuleArn=arn:aws:events:us-east-1:000000000000:rule/rotation-events/on-rotation-success [1;36m== Child SM role + policy ==[0m [1;36m== Create child SM (rotation-child) ==[0m ChildSmArn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child [1;36m== Parent SM role + policy ==[0m [1;36m== Create parent SM (rotation-orchestrator) ==[0m ParentSmArn=arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator [1;36m== Summary ==[0m kms alias: alias/app-rotation-key (key: arn:aws:kms:us-east-1:000000000000:key/f5165b9e-8ae8-48dc-90eb-da742d06c0e6) secret: arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-kStXlH rotator lambda: arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator notification lambda: arn:aws:lambda:us-east-1:000000000000:function:notification-lambda event bus: arn:aws:events:us-east-1:000000000000:event-bus/rotation-events rule: arn:aws:events:us-east-1:000000000000:rule/rotation-events/on-rotation-success child SM: arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child parent SM: arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator
End-to-end acceptance run
set -euo pipefail
AWSC="aws --endpoint-url=http://localstack:4566 --region us-east-1"
echo "== BEFORE =="
BEFORE_CURRENT=$($AWSC secretsmanager get-secret-value --secret-id app/db/password --query SecretString --output text)
BEFORE_VERSION=$($AWSC secretsmanager get-secret-value --secret-id app/db/password --query VersionId --output text)
echo "AWSCURRENT value: $BEFORE_CURRENT"
echo "AWSCURRENT version: $BEFORE_VERSION"
echo "== START PARENT =="
START_TS=$(date +%s)
EXEC_ARN=$($AWSC stepfunctions start-execution \
--state-machine-arn arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator \
--query executionArn --output text)
echo "ExecArn=$EXEC_ARN"
echo "== POLL UNTIL TERMINAL (<=90s) =="
for i in $(seq 1 90); do
STATUS=$($AWSC stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query status --output text)
if [ "$STATUS" != "RUNNING" ]; then
break
fi
sleep 1
done
END_TS=$(date +%s)
ELAPSED=$((END_TS - START_TS))
echo "Final status: $STATUS after ${ELAPSED}s"
if [ "$STATUS" != "SUCCEEDED" ]; then
echo "!! parent did not succeed"
$AWSC stepfunctions describe-execution --execution-arn "$EXEC_ARN"
exit 1
fi
echo "== AFTER =="
AFTER_CURRENT=$($AWSC secretsmanager get-secret-value --secret-id app/db/password --query SecretString --output text)
AFTER_VERSION=$($AWSC secretsmanager get-secret-value --secret-id app/db/password --query VersionId --output text)
echo "AWSCURRENT value: $AFTER_CURRENT"
echo "AWSCURRENT version: $AFTER_VERSION"
PREVIOUS=$($AWSC secretsmanager get-secret-value --secret-id app/db/password --version-stage AWSPREVIOUS --query SecretString --output text)
echo "AWSPREVIOUS value: $PREVIOUS"
if [ "$BEFORE_CURRENT" = "$AFTER_CURRENT" ]; then echo "!! secret string did not change"; exit 1; fi
if [ "$BEFORE_CURRENT" != "$PREVIOUS" ]; then echo "!! old value is not AWSPREVIOUS"; exit 1; fi
echo "OK: rotation occurred and staging labels moved correctly"
echo "== NOTIFICATION LAMBDA LOGS =="
sleep 2
LOG_GROUP=/aws/lambda/notification-lambda
STREAMS=$($AWSC logs describe-log-streams --log-group-name $LOG_GROUP --order-by LastEventTime --descending --max-items 3 --query 'logStreams[].logStreamName' --output text || echo "")
echo "Streams: $STREAMS"
for s in $STREAMS; do
$AWSC logs get-log-events --log-group-name $LOG_GROUP --log-stream-name "$s" --query 'events[].message' --output text
done | tee /tmp/notif-logs.txt
if grep -q '^notified:' /tmp/notif-logs.txt; then
echo "OK: notification lambda logged 'notified:' line"
else
echo "!! no 'notified:' line found"; exit 1
fiExit code 1
== BEFORE ==
AWSCURRENT value: initial-placeholder-1776838762433231295
AWSCURRENT version: 4b2b492e-63c1-4b47-ba04-d30240c20f9b
== START PARENT ==
ExecArn=arn:aws:states:us-east-1:000000000000:execution:rotation-orchestrator:bb78a15b-d811-4e67-a554-1ed6e7fcb021
== POLL UNTIL TERMINAL (<=90s) ==
Final status: SUCCEEDED after 4s
== AFTER ==
AWSCURRENT value: DcvCpn0bkKv55FLLiNdYq7hKXRiVhWuK
AWSCURRENT version: ee068dbe-6b4b-4adf-9a49-f6d1d6487802
AWSPREVIOUS value: initial-placeholder-1776838762433231295
OK: rotation occurred and staging labels moved correctly
== NOTIFICATION LAMBDA LOGS ==
Streams: 2026/04/22/[$LATEST]f050faabb7fb04459b0dfd8866d59ed2
START RequestId: 03049de6-2f97-4730-8ec8-05fc55dab52b Version: $LATEST
notified: {"version": "0", "id": "19693df1-8591-468c-8755-5efefa4e1bf6", "detail-type": "RotationComplete", "source": "rotation.pipeline", "account": "000000000000", "time": "2026-04-22T06:19:55Z", "region": "us-east-1", "resources": ["arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child", "arn:aws:states:us-east-1:000000000000:execution:rotation-child:5529fd07-4782-42c0-9100-2fde41c4f2be"], "detail": {"status": "ok", "secretId": "app/db/password"}}
END RequestId: 03049de6-2f97-4730-8ec8-05fc55dab52b
REPORT RequestId: 03049de6-2f97-4730-8ec8-05fc55dab52b Duration: 11.85 ms Billed Duration: 12 ms Memory Size: 128 MB Max Memory Used: 128 MB
!! no 'notified:' line found
[error] tool reported failureFix notified grep and test two-rotation determinism
set -euo pipefail
AWSC="aws --endpoint-url=http://localstack:4566 --region us-east-1"
# leading whitespace allowed
if grep -qE '^\s*notified:' /tmp/notif-logs.txt; then
echo "OK: notification lambda logged 'notified:' line (tab-prefixed per Lambda log format)"
else
echo "!! no notified line"; exit 1
fi
echo "== SECOND ROTATION (determinism check: two rotations produce different strings) =="
V1=$($AWSC secretsmanager get-secret-value --secret-id app/db/password --query SecretString --output text)
EXEC=$($AWSC stepfunctions start-execution --state-machine-arn arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator --query executionArn --output text)
for i in $(seq 1 90); do
S=$($AWSC stepfunctions describe-execution --execution-arn "$EXEC" --query status --output text)
[ "$S" != "RUNNING" ] && break
sleep 1
done
echo "status=$S"
V2=$($AWSC secretsmanager get-secret-value --secret-id app/db/password --query SecretString --output text)
echo "v1=$V1"
echo "v2=$V2"
if [ "$V1" = "$V2" ]; then echo "!! consecutive rotations produced same value"; exit 1; fi
echo "OK: consecutive rotations differ"OK: notification lambda logged 'notified:' line (tab-prefixed per Lambda log format) == SECOND ROTATION (determinism check: two rotations produce different strings) == status=SUCCEEDED v1=DcvCpn0bkKv55FLLiNdYq7hKXRiVhWuK v2=8_kOTOCAf32DyiV5ndq4BZdtxxLM8VEY OK: consecutive rotations differ [stdout] OK: notification lambda logged 'notified:' line (tab-prefixed per Lambda log format) == SECOND ROTATION (determinism check: two rotations produce different strings) == status=SUCCEEDED v1=DcvCpn0bkKv55FLLiNdYq7hKXRiVhWuK v2=8_kOTOCAf32DyiV5ndq4BZdtxxLM8VEY OK: consecutive rotations differ
No file changes, the agent left the workspace untouched.
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0 -- /root/.cache/uv/archive-v0/Eklpkw38n76eq7Qn9jL4r/bin/python
cachedir: .pytest_cache
rootdir: /tests
plugins: json-ctrf-0.3.5
collecting ... collected 15 items
../tests/test_state.py::test_aws_cli_binary_is_real PASSED [ 6%]
../tests/test_state.py::test_localstack_reachable PASSED [ 13%]
../tests/test_state.py::test_secret_exists_and_uses_customer_cmk PASSED [ 20%]
../tests/test_state.py::test_rotation_lambda_exists PASSED [ 26%]
../tests/test_state.py::test_notification_lambda_exists PASSED [ 33%]
../tests/test_state.py::test_parent_and_child_state_machines_exist PASSED [ 40%]
../tests/test_state.py::test_event_bus_and_rule_exist PASSED [ 46%]
../tests/test_state.py::test_rule_target_points_at_notification_lambda PASSED [ 53%]
../tests/test_state.py::test_rotator_role_grants_kms_actions PASSED [ 60%]
../tests/test_state.py::test_kms_key_policy_grants_rotator_role FAILED [ 66%]
../tests/test_state.py::test_rotator_lambda_allows_secretsmanager_invoke FAILED [ 73%]
../tests/test_state.py::test_parent_role_grants_sync2_managed_rule_perms FAILED [ 80%]
../tests/test_state.py::test_child_role_can_rotate_and_publish FAILED [ 86%]
../tests/test_state.py::test_notification_lambda_allows_eventbridge_invoke PASSED [ 93%]
../tests/test_state.py::test_end_to_end_parent_execution_rotates_and_notifies PASSED [100%]
=================================== FAILURES ===================================
___________________ test_kms_key_policy_grants_rotator_role ____________________
iam = <botocore.client.IAM object at 0xffff7d4cdf40>
kms = <botocore.client.KMS object at 0xffff7da18320>
def test_kms_key_policy_grants_rotator_role(iam, kms):
role_arn = iam.get_role(RoleName=ROTATOR_ROLE)["Role"]["Arn"]
policy_str = kms.get_key_policy(KeyId=KEY_ALIAS, PolicyName="default")[
"Policy"
]
policy = json.loads(policy_str)
match = False
for st in policy.get("Statement", []):
if st.get("Effect") != "Allow":
continue
principal = st.get("Principal") or {}
if not isinstance(principal, dict):
continue
arns = set(_normalise_list(principal.get("AWS")))
if role_arn not in arns:
continue
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, REQUIRED_KMS_ACTIONS):
match = True
break
> assert match, (
f"KMS key policy on {KEY_ALIAS} has no Allow statement whose "
f"Principal.AWS includes {role_arn} and whose Action covers "
f"{sorted(REQUIRED_KMS_ACTIONS)}. Key policy: {policy_str}"
)
E AssertionError: KMS key policy on alias/app-rotation-key has no Allow statement whose Principal.AWS includes arn:aws:iam::000000000000:role/secrets-rotator-role and whose Action covers ['kms:Decrypt', 'kms:GenerateDataKey']. Key policy: {"Version": "2012-10-17", "Id": "key-default-1", "Statement": [{"Sid": "Enable IAM User Permissions", "Effect": "Allow", "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, "Action": "kms:*", "Resource": "*"}]}
E assert False
/tests/test_state.py:322: AssertionError
_______________ test_rotator_lambda_allows_secretsmanager_invoke _______________
lmb = <botocore.client.Lambda object at 0xffff7da1a090>
def test_rotator_lambda_allows_secretsmanager_invoke(lmb):
try:
policy_doc = json.loads(
> lmb.get_policy(FunctionName=ROTATOR_FUNCTION)["Policy"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
/tests/test_state.py:332:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/root/.cache/uv/archive-v0/Eklpkw38n76eq7Qn9jL4r/lib/python3.12/site-packages/botocore/client.py:569: in _api_call
return self._make_api_call(operation_name, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <botocore.client.Lambda object at 0xffff7da1a090>
operation_name = 'GetPolicy', api_params = {'FunctionName': 'secrets-rotator'}
def _make_api_call(self, operation_name, api_params):
operation_model = self._service_model.operation_model(operation_name)
service_name = self._service_model.service_name
history_recorder.record(
'API_CALL',
{
'service': service_name,
'operation': operation_name,
'params': api_params,
},
)
if operation_model.deprecated:
logger.debug(
'Warning: %s.%s() is deprecated', service_name, operation_name
)
request_context = {
'client_region': self.meta.region_name,
'client_config': self.meta.config,
'has_streaming_input': operation_model.has_streaming_input,
'auth_type': operation_model.resolved_auth_type,
'unsigned_payload': operation_model.unsigned_payload,
}
api_params = self._emit_api_params(
api_params=api_params,
operation_model=operation_model,
context=request_context,
)
(
endpoint_url,
additional_headers,
properties,
) = self._resolve_endpoint_ruleset(
operation_model, api_params, request_context
)
if properties:
# Pass arbitrary endpoint info with the Request
# for use during construction.
request_context['endpoint_properties'] = properties
request_dict = self._convert_to_request_dict(
api_params=api_params,
operation_model=operation_model,
endpoint_url=endpoint_url,
context=request_context,
headers=additional_headers,
)
resolve_checksum_context(request_dict, operation_model, api_params)
service_id = self._service_model.service_id.hyphenize()
handler, event_response = self.meta.events.emit_until_response(
f'before-call.{service_id}.{operation_name}',
model=operation_model,
params=request_dict,
request_signer=self._request_signer,
context=request_context,
)
if event_response is not None:
http, parsed_response = event_response
else:
maybe_compress_request(
self.meta.config, request_dict, operation_model
)
apply_request_checksum(request_dict)
http, parsed_response = self._make_request(
operation_model, request_dict, request_context
)
self.meta.events.emit(
f'after-call.{service_id}.{operation_name}',
http_response=http,
parsed=parsed_response,
model=operation_model,
context=request_context,
)
if http.status_code >= 300:
error_info = parsed_response.get("Error", {})
error_code = error_info.get("QueryErrorCode") or error_info.get(
"Code"
)
error_class = self.exceptions.from_code(error_code)
> raise error_class(parsed_response, operation_name)
E botocore.errorfactory.ResourceNotFoundException: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
/root/.cache/uv/archive-v0/Eklpkw38n76eq7Qn9jL4r/lib/python3.12/site-packages/botocore/client.py:1023: ResourceNotFoundException
During handling of the above exception, another exception occurred:
lmb = <botocore.client.Lambda object at 0xffff7da1a090>
def test_rotator_lambda_allows_secretsmanager_invoke(lmb):
try:
policy_doc = json.loads(
lmb.get_policy(FunctionName=ROTATOR_FUNCTION)["Policy"]
)
except ClientError as e:
> pytest.fail(
f"Lambda {ROTATOR_FUNCTION} has no resource-based policy , "
f"secretsmanager.amazonaws.com cannot invoke it: {e}"
)
E Failed: Lambda secrets-rotator has no resource-based policy , secretsmanager.amazonaws.com cannot invoke it: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
/tests/test_state.py:335: Failed
_______________ test_parent_role_grants_sync2_managed_rule_perms _______________
iam = <botocore.client.IAM object at 0xffff7d4cdf40>
def test_parent_role_grants_sync2_managed_rule_perms(iam):
"""The .sync:2 integration registers an EventBridge-managed callback
rule. Parent role needs events:PutRule/PutTargets/DescribeRule plus
full states:* on the child."""
found_events = False
found_states = False
for _pname, st in _inline_statements(iam, PARENT_ROLE):
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, REQUIRED_PARENT_EVENTS_ACTIONS):
found_events = True
if _actions_cover(actions, REQUIRED_PARENT_STATES_ACTIONS):
found_states = True
assert found_events, (
f"{PARENT_ROLE} missing events:PutRule/PutTargets/DescribeRule. "
f".sync:2 uses an EventBridge-managed callback rule; without "
f"these the parent cannot set up the callback."
)
> assert found_states, (
f"{PARENT_ROLE} missing states:StartExecution/DescribeExecution/"
f"StopExecution. .sync:2 requires all three on the child SM."
)
E AssertionError: rotation-orchestrator-role missing states:StartExecution/DescribeExecution/StopExecution. .sync:2 requires all three on the child SM.
E assert False
/tests/test_state.py:377: AssertionError
____________________ test_child_role_can_rotate_and_publish ____________________
iam = <botocore.client.IAM object at 0xffff7d4cdf40>
def test_child_role_can_rotate_and_publish(iam):
found_rotate = False
found_putevents = False
for _pname, st in _inline_statements(iam, CHILD_ROLE):
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, {"secretsmanager:RotateSecret"}):
found_rotate = True
if _actions_cover(actions, {"events:PutEvents"}):
found_putevents = True
> assert found_rotate, (
f"{CHILD_ROLE} lacks secretsmanager:RotateSecret , the child "
f"state machine cannot trigger rotation."
)
E AssertionError: rotation-child-role lacks secretsmanager:RotateSecret , the child state machine cannot trigger rotation.
E assert False
/tests/test_state.py:392: AssertionError
=============================== warnings summary ===============================
test_state.py: 33 warnings
/root/.cache/uv/archive-v0/Eklpkw38n76eq7Qn9jL4r/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
==================================== PASSES ====================================
=========================== short test summary info ============================
PASSED ../tests/test_state.py::test_aws_cli_binary_is_real
PASSED ../tests/test_state.py::test_localstack_reachable
PASSED ../tests/test_state.py::test_secret_exists_and_uses_customer_cmk
PASSED ../tests/test_
… (truncated at 12,000 chars, full verifier log is in the trial artifacts)Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_5190973914b24efd. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_5190973914b24efd · verifier authoritative; classifier explanatory.