tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.Check working directory and tools
ls /app/ && aws --version && python3 --version
aws-cli/2.34.33 Python/3.14.4 Linux/6.10.14-linuxkit exe/aarch64.ubuntu.24 Python 3.12.3 [stdout] aws-cli/2.34.33 Python/3.14.4 Linux/6.10.14-linuxkit exe/aarch64.ubuntu.24 Python 3.12.3
Verify localstack health
curl -s http://localstack:4566/_localstack/health | jq .
{
"services": {
"acm": "disabled",
"apigateway": "disabled",
"cloudformation": "disabled",
"cloudwatch": "disabled",
"config": "disabled",
"dynamodb": "disabled",
"dynamodbstreams": "disabled",
"ec2": "disabled",
"es": "disabled",
"events": "available",
"firehose": "disabled",
"iam": "available",
"kinesis": "disabled",
"kms": "available",
"lambda": "available",
"logs": "available",
"opensearch": "disabled",
"redshift": "disabled",
"resource-groups": "disabled",
"resourcegroupstaggingapi": "disabled",
"route53": "disabled",
"route53resolver": "disabled",
"s3": "available",
"s3control": "disabled",
"scheduler": "disabled",
"secretsmanager": "available",
"ses": "disabled",
"sns": "disabled",
"sqs": "disabled",
"ssm": "disabled",
"stepfunctions": "available",
"sts": "available",
"support": "disabled",
"swf": "disabled",
"transcribe": "disabled"
},
"edition": "community",
"version": "4.13.1"
}
[stdout]
{
"services": {
"acm": "disabled",
"apigateway": "disabled",
"cloudformation": "disabled",
"cloudwatch": "disabled",
"config": "disabled",
"dynamodb": "disabled",
"dynamodbstreams": "disabled",
"ec2": "disabled",
"es": "disabled",
"events": "available",
"firehose": "disabled",
"iam": "available",
"kinesis": "disabled",
"kms": "available",
"lambda": "available",
"logs": "available",
"opensearch": "disabled",
"redshift": "disabled",
"resource-groups": "disabled",
"resourcegroupstaggingapi": "disabled",
"route53": "disabled",
"route53resolver": "disabled",
"s3": "available",
"s3control": "disabled",
"scheduler": "disabled",
"secretsmanager": "available",
"ses": "disabled",
"sns": "disabled",
"sqs": "disabled",
"ssm": "disabled",
"stepfunctions": "available",
"sts": "available",
"support": "disabled",
"swf": "disabled",
"transcribe": "disabled"
},
"edition": "community",
"version": "4.13.1"
}Check account identity and create build dir
aws --endpoint-url=http://localstack:4566 sts get-caller-identity && mkdir -p /app/build
{
"UserId": "AKIAIOSFODNN7EXAMPLE",
"Account": "000000000000",
"Arn": "arn:aws:iam::000000000000:root"
}
[stdout]
{
"UserId": "AKIAIOSFODNN7EXAMPLE",
"Account": "000000000000",
"Arn": "arn:aws:iam::000000000000:root"
}import json
import os
import secrets as pysecrets
import string
import boto3
ENDPOINT = os.environ.get("AWS_ENDPOINT_URL", "http://localstack:4566")
SECRET_ID = os.environ.get("SECRET_ID", "app/db/password")
REGION = os.environ.get("AWS_REGION", "us-east-1")
ALPHABET = string.ascii_letters + string.digits + "-_"
def generate_password(length: int = 32) -> str:
return "".join(pysecrets.choice(ALPHABET) for _ in range(length))
def lambda_handler(event, context):
client = boto3.client("secretsmanager", endpoint_url=ENDPOINT, region_name=REGION)
try:
current = client.get_secret_value(SecretId=SECRET_ID)
current_value = current.get("SecretString")
except client.exceptions.ResourceNotFoundException:
current_value = None
new_password = generate_password()
while new_password == current_value:
new_password = generate_password()
resp = client.put_secret_value(
SecretId=SECRET_ID,
SecretString=new_password,
)
return {
"status": "rotated",
"versionId": resp.get("VersionId"),
"arn": resp.get("ARN"),
"secretId": SECRET_ID,
}
import json
def lambda_handler(event, context):
detail = event.get("detail", {}) if isinstance(event, dict) else {}
secret_id = detail.get("secretId")
version_id = detail.get("versionId")
print(f"notified: rotation complete secretId={secret_id} versionId={version_id}")
return {"ok": True}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"Service": "lambda.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"Service": "states.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
Create KMS key, alias, and secret
set -e AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1" # KMS key with alias KMS_KEY_ID=$($AWS kms create-key --description "app rotation CMK" --key-usage ENCRYPT_DECRYPT --query 'KeyMetadata.KeyId' --output text) echo "KMS_KEY_ID=$KMS_KEY_ID" KMS_KEY_ARN=$($AWS kms describe-key --key-id "$KMS_KEY_ID" --query 'KeyMetadata.Arn' --output text) echo "KMS_KEY_ARN=$KMS_KEY_ARN" $AWS kms create-alias --alias-name alias/app-rotation-key --target-key-id "$KMS_KEY_ID" # Save for later echo "$KMS_KEY_ID" > /app/build/kms_key_id echo "$KMS_KEY_ARN" > /app/build/kms_key_arn # Secret with initial value, encrypted by our CMK $AWS secretsmanager create-secret \ --name app/db/password \ --description "application db password" \ --kms-key-id "alias/app-rotation-key" \ --secret-string "initial-bootstrap-password" \ --query 'ARN' --output text > /app/build/secret_arn cat /app/build/secret_arn
KMS_KEY_ID=527ee366-bbfc-4956-87fa-5a5f61b4ec5f KMS_KEY_ARN=arn:aws:kms:us-east-1:000000000000:key/527ee366-bbfc-4956-87fa-5a5f61b4ec5f arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM [stdout] KMS_KEY_ID=527ee366-bbfc-4956-87fa-5a5f61b4ec5f KMS_KEY_ARN=arn:aws:kms:us-east-1:000000000000:key/527ee366-bbfc-4956-87fa-5a5f61b4ec5f arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM
{
"query": "select:TodoWrite",
"max_results": 1
}
, output,
{"type": "tool_reference", "tool_name": "TodoWrite"}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Logs",
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/secrets-rotator:*"
},
{
"Sid": "SecretReadWrite",
"Effect": "Allow",
"Action": [
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:DescribeSecret"
],
"Resource": "__SECRET_ARN__"
},
{
"Sid": "KmsForSecret",
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": "__KMS_KEY_ARN__"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/lambda/notification-lambda:*"
}
]
}
Create lambda roles, package, deploy lambdas
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
cd /app/build
SECRET_ARN=$(cat secret_arn)
KMS_KEY_ARN=$(cat kms_key_arn)
# Fill in ARN placeholders in rotator policy
sed -e "s|__SECRET_ARN__|$SECRET_ARN|g" \
-e "s|__KMS_KEY_ARN__|$KMS_KEY_ARN|g" \
policy-rotator.json > policy-rotator.final.json
# Create roles
$AWS iam create-role --role-name secrets-rotator-role \
--assume-role-policy-document file://trust-lambda.json >/dev/null
$AWS iam put-role-policy --role-name secrets-rotator-role \
--policy-name rotator-inline \
--policy-document file://policy-rotator.final.json
$AWS iam create-role --role-name notification-lambda-role \
--assume-role-policy-document file://trust-lambda.json >/dev/null
$AWS iam put-role-policy --role-name notification-lambda-role \
--policy-name notifier-inline \
--policy-document file://policy-notifier.json
# Package lambdas
zip -q rotator.zip rotator.py
zip -q notifier.zip notifier.py
ROTATOR_ROLE_ARN=$($AWS iam get-role --role-name secrets-rotator-role --query 'Role.Arn' --output text)
NOTIFIER_ROLE_ARN=$($AWS iam get-role --role-name notification-lambda-role --query 'Role.Arn' --output text)
$AWS lambda create-function \
--function-name secrets-rotator \
--runtime python3.11 \
--role "$ROTATOR_ROLE_ARN" \
--handler rotator.lambda_handler \
--zip-file fileb://rotator.zip \
--timeout 30 \
--environment "Variables={AWS_ENDPOINT_URL=http://localstack:4566,SECRET_ID=app/db/password}" \
--query 'FunctionArn' --output text > rotator_lambda_arn
$AWS lambda create-function \
--function-name notification-lambda \
--runtime python3.11 \
--role "$NOTIFIER_ROLE_ARN" \
--handler notifier.lambda_handler \
--zip-file fileb://notifier.zip \
--timeout 30 \
--query 'FunctionArn' --output text > notifier_lambda_arn
echo "rotator: $(cat rotator_lambda_arn)"
echo "notifier: $(cat notifier_lambda_arn)"
# Wait briefly for Active state
for f in secrets-rotator notification-lambda; do
for i in 1 2 3 4 5 6 7 8 9 10; do
STATE=$($AWS lambda get-function --function-name $f --query 'Configuration.State' --output text 2>/dev/null || echo "Pending")
if [ "$STATE" = "Active" ]; then echo "$f Active"; break; fi
sleep 1
done
donerotator: arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator notifier: arn:aws:lambda:us-east-1:000000000000:function:notification-lambda secrets-rotator Active notification-lambda Active [stdout] rotator: arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator notifier: arn:aws:lambda:us-east-1:000000000000:function:notification-lambda secrets-rotator Active notification-lambda Active
Smoke-test rotator lambda
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
# Quick sanity: invoke rotator once to confirm lambda can talk to localstack
cd /app/build
$AWS lambda invoke --function-name secrets-rotator --payload '{}' --cli-binary-format raw-in-base64-out out.json >/tmp/invoke_meta.json
cat /tmp/invoke_meta.json
echo "----"
cat out.json{
"StatusCode": 200,
"ExecutedVersion": "$LATEST"
}
----
{"status": "rotated", "versionId": "117aa962-6223-46d9-a668-1b213c807f21", "arn": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM", "secretId": "app/db/password"}
[stdout]
{
"StatusCode": 200,
"ExecutedVersion": "$LATEST"
}
----
{"status": "rotated", "versionId": "117aa962-6223-46d9-a668-1b213c807f21", "arn": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM", "secretId": "app/db/password"}Check secret versions
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1" $AWS secretsmanager list-secret-version-ids --secret-id app/db/password
{
"Versions": [
{
"VersionId": "117aa962-6223-46d9-a668-1b213c807f21",
"VersionStages": [
"AWSCURRENT"
],
"CreatedDate": "2026-04-23T06:13:07+00:00"
},
{
"VersionId": "4420a6f3-5db1-4748-80ef-a1339fae7f24",
"VersionStages": [
"AWSPREVIOUS"
],
"LastAccessedDate": "2026-04-23T00:00:00+00:00",
"CreatedDate": "2026-04-23T06:12:25+00:00"
}
],
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM",
"Name": "app/db/password"
}
[stdout]
{
"Versions": [
{
"VersionId": "117aa962-6223-46d9-a668-1b213c807f21",
"VersionStages": [
"AWSCURRENT"
],
"CreatedDate": "2026-04-23T06:13:07+00:00"
},
{
"VersionId": "4420a6f3-5db1-4748-80ef-a1339fae7f24",
"VersionStages": [
"AWSPREVIOUS"
],
"LastAccessedDate": "2026-04-23T00:00:00+00:00",
"CreatedDate": "2026-04-23T06:12:25+00:00"
}
],
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM",
"Name": "app/db/password"
}Create EventBridge bus, rule, lambda target
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
cd /app/build
# Custom bus
$AWS events create-event-bus --name rotation-events >/dev/null
BUS_ARN="arn:aws:events:us-east-1:000000000000:event-bus/rotation-events"
# Rule on that bus
EVENT_PATTERN='{"source":["rotation.pipeline"],"detail-type":["RotationComplete"]}'
$AWS events put-rule \
--name on-rotation-success \
--event-bus-name rotation-events \
--event-pattern "$EVENT_PATTERN" \
--state ENABLED >/dev/null
# Target: notification lambda
NOTIFIER_ARN=$(cat notifier_lambda_arn)
$AWS events put-targets \
--rule on-rotation-success \
--event-bus-name rotation-events \
--targets "Id=notifier,Arn=$NOTIFIER_ARN" >/dev/null
# Permission: EventBridge rule -> notification lambda
RULE_ARN="arn:aws:events:us-east-1:000000000000:rule/rotation-events/on-rotation-success"
$AWS lambda add-permission \
--function-name notification-lambda \
--statement-id allow-eventbridge-on-rotation-success \
--action lambda:InvokeFunction \
--principal events.amazonaws.com \
--source-arn "$RULE_ARN" >/dev/null
echo "bus + rule + target wired"bus + rule + target wired [stdout] bus + rule + target wired
{
"Comment": "Rotates the secret and emits a RotationComplete event.",
"StartAt": "Rotate",
"States": {
"Rotate": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator",
"Payload": {}
},
"ResultSelector": {
"secretId.$": "$.Payload.secretId",
"versionId.$": "$.Payload.versionId",
"arn.$": "$.Payload.arn"
},
"ResultPath": "$.rotation",
"Next": "Notify"
},
"Notify": {
"Type": "Task",
"Resource": "arn:aws:states:::events:putEvents",
"Parameters": {
"Entries": [
{
"EventBusName": "rotation-events",
"Source": "rotation.pipeline",
"DetailType": "RotationComplete",
"Detail": {
"secretId.$": "$.rotation.secretId",
"versionId.$": "$.rotation.versionId",
"arn.$": "$.rotation.arn"
}
}
]
},
"End": true
}
}
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "InvokeRotator",
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator"
},
{
"Sid": "PutRotationEvents",
"Effect": "Allow",
"Action": "events:PutEvents",
"Resource": "arn:aws:events:us-east-1:000000000000:event-bus/rotation-events"
}
]
}
Deploy child state machine with scoped role
set -e AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1" cd /app/build $AWS iam create-role --role-name rotation-child-role \ --assume-role-policy-document file://trust-states.json >/dev/null $AWS iam put-role-policy --role-name rotation-child-role \ --policy-name child-inline \ --policy-document file://policy-child.json CHILD_ROLE_ARN=$($AWS iam get-role --role-name rotation-child-role --query 'Role.Arn' --output text) $AWS stepfunctions create-state-machine \ --name rotation-child \ --definition file://child-sm.json \ --role-arn "$CHILD_ROLE_ARN" \ --query 'stateMachineArn' --output text > child_sm_arn cat child_sm_arn
arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child [stdout] arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child
Smoke test child state machine
set -e AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1" cd /app/build # Quick end-to-end test of child SM alone EXEC_ARN=$($AWS stepfunctions start-execution --state-machine-arn "$(cat child_sm_arn)" --query 'executionArn' --output text) echo "child exec: $EXEC_ARN" for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do S=$($AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query 'status' --output text) echo "$i: $S" if [ "$S" != "RUNNING" ]; then break; fi sleep 1 done $AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN"
child exec: arn:aws:states:us-east-1:000000000000:execution:rotation-child:d9875fb1-ef15-4bd5-bc25-c437198f9d7d
1: RUNNING
2: SUCCEEDED
{
"executionArn": "arn:aws:states:us-east-1:000000000000:execution:rotation-child:d9875fb1-ef15-4bd5-bc25-c437198f9d7d",
"stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child",
"name": "d9875fb1-ef15-4bd5-bc25-c437198f9d7d",
"status": "SUCCEEDED",
"startDate": "2026-04-23T06:13:53.597028+00:00",
"stopDate": "2026-04-23T06:13:54.958015+00:00",
"input": "{}",
"inputDetails": {
"included": true
},
"output": "{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"2b53c3b4-45df-4aff-b6fb-7e16e6b53262\"}]}",
"outputDetails": {
"included": true
}
}
[stdout]
child exec: arn:aws:states:us-east-1:000000000000:execution:rotation-child:d9875fb1-ef15-4bd5-bc25-c437198f9d7d
1: RUNNING
2: SUCCEEDED
{
"executionArn": "arn:aws:states:us-east-1:000000000000:execution:rotation-child:d9875fb1-ef15-4bd5-bc25-c437198f9d7d",
"stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child",
"name": "d9875fb1-ef15-4bd5-bc25-c437198f9d7d",
"status": "SUCCEEDED",
"startDate": "2026-04-23T06:13:53.597028+00:00",
"stopDate": "2026-04-23T06:13:54.958015+00:00",
"input": "{}",
"inputDetails": {
"included": true
},
"output": "{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"2b53c3b4-45df-4aff-b6fb-7e16e6b53262\"}]}",
"outputDetails": {
"included": true
}
}Check notifier logs
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1" sleep 2 $AWS logs describe-log-streams --log-group-name /aws/lambda/notification-lambda --query 'logStreams[].logStreamName' --output text 2>&1 | head -5 echo "----" $AWS logs filter-log-events --log-group-name /aws/lambda/notification-lambda --query 'events[].message' --output text 2>&1 | head -20
2026/04/23/[$LATEST]ef3c33a9f465a843c84a417987ad7c5a ---- START RequestId: af3db930-19d4-461f-ae1f-ae5d4bfb811b Version: $LATEST notified: rotation complete secretId=app/db/password versionId=d29ef77e-00aa-416b-a5b2-1548ef06f423 END RequestId: af3db930-19d4-461f-ae1f-ae5d4bfb811b REPORT RequestId: af3db930-19d4-461f-ae1f-ae5d4bfb811b Duration: 9.01 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB [stdout] 2026/04/23/[$LATEST]ef3c33a9f465a843c84a417987ad7c5a ---- START RequestId: af3db930-19d4-461f-ae1f-ae5d4bfb811b Version: $LATEST notified: rotation complete secretId=app/db/password versionId=d29ef77e-00aa-416b-a5b2-1548ef06f423 END RequestId: af3db930-19d4-461f-ae1f-ae5d4bfb811b REPORT RequestId: af3db930-19d4-461f-ae1f-ae5d4bfb811b Duration: 9.01 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
{
"Comment": "Entry point. Delegates rotation to rotation-child and waits.",
"StartAt": "RunChild",
"States": {
"RunChild": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync:2",
"Parameters": {
"StateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child",
"Input": {}
},
"End": true
}
}
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "RunChildExecution",
"Effect": "Allow",
"Action": [
"states:StartExecution"
],
"Resource": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child"
},
{
"Sid": "InspectChildExecution",
"Effect": "Allow",
"Action": [
"states:DescribeExecution",
"states:StopExecution"
],
"Resource": "arn:aws:states:us-east-1:000000000000:execution:rotation-child:*"
},
{
"Sid": "ManagedRuleForSyncExecution",
"Effect": "Allow",
"Action": [
"events:PutRule",
"events:PutTargets",
"events:DescribeRule"
],
"Resource": "arn:aws:events:us-east-1:000000000000:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule"
}
]
}
Deploy parent state machine with scoped role
set -e AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1" cd /app/build $AWS iam create-role --role-name rotation-orchestrator-role \ --assume-role-policy-document file://trust-states.json >/dev/null $AWS iam put-role-policy --role-name rotation-orchestrator-role \ --policy-name orchestrator-inline \ --policy-document file://policy-parent.json PARENT_ROLE_ARN=$($AWS iam get-role --role-name rotation-orchestrator-role --query 'Role.Arn' --output text) $AWS stepfunctions create-state-machine \ --name rotation-orchestrator \ --definition file://parent-sm.json \ --role-arn "$PARENT_ROLE_ARN" \ --query 'stateMachineArn' --output text > parent_sm_arn cat parent_sm_arn
arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator [stdout] arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator
End-to-end verification: start parent, assert SUCCEEDED + version move + notifier log
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
cd /app/build
BEFORE_CURRENT=$($AWS secretsmanager describe-secret --secret-id app/db/password --query 'VersionIdsToStages' --output json)
BEFORE_VALUE=$($AWS secretsmanager get-secret-value --secret-id app/db/password --query 'SecretString' --output text)
echo "BEFORE stages: $BEFORE_CURRENT"
echo "BEFORE value: $BEFORE_VALUE"
EXEC_ARN=$($AWS stepfunctions start-execution --state-machine-arn "$(cat parent_sm_arn)" --query 'executionArn' --output text)
echo "parent exec: $EXEC_ARN"
START_TS=$(date +%s)
for i in $(seq 1 90); do
S=$($AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query 'status' --output text)
if [ "$S" != "RUNNING" ]; then
END_TS=$(date +%s); echo "parent status=$S in $((END_TS-START_TS))s"; break
fi
sleep 1
done
$AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN"
echo "--- post versions ---"
$AWS secretsmanager list-secret-version-ids --secret-id app/db/password
AFTER_VALUE=$($AWS secretsmanager get-secret-value --secret-id app/db/password --query 'SecretString' --output text)
echo "AFTER value: $AFTER_VALUE"
[ "$BEFORE_VALUE" != "$AFTER_VALUE" ] && echo "value changed: yes" || echo "value changed: NO"
echo "--- notifier log (last 5 min) ---"
sleep 2
$AWS logs filter-log-events --log-group-name /aws/lambda/notification-lambda --start-time $(( (START_TS - 10) * 1000 )) --query 'events[].message' --output textBEFORE stages: {
"117aa962-6223-46d9-a668-1b213c807f21": [
"AWSPREVIOUS"
],
"d29ef77e-00aa-416b-a5b2-1548ef06f423": [
"AWSCURRENT"
]
}
BEFORE value: C_fvHXOTpAAMCRJbOZAMZr2aaRs-SH-I
parent exec: arn:aws:states:us-east-1:000000000000:execution:rotation-orchestrator:502820c1-2683-4dd1-aefd-59c27bb692c9
parent status=SUCCEEDED in 2s
{
"executionArn": "arn:aws:states:us-east-1:000000000000:execution:rotation-orchestrator:502820c1-2683-4dd1-aefd-59c27bb692c9",
"stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator",
"name": "502820c1-2683-4dd1-aefd-59c27bb692c9",
"status": "SUCCEEDED",
"startDate": "2026-04-23T06:14:37.726253+00:00",
"stopDate": "2026-04-23T06:14:39.288309+00:00",
"input": "{}",
"inputDetails": {
"included": true
},
"output": "{\"ExecutionArn\":\"arn:aws:states:us-east-1:000000000000:execution:rotation-child:c9fc6c58-bedd-472c-8a6f-97b440e61a95\",\"StateMachineArn\":\"arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child\",\"Name\":\"c9fc6c58-bedd-472c-8a6f-97b440e61a95\",\"Status\":\"SUCCEEDED\",\"StartDate\":\"2026-04-23T06:14:37.754538+00:00\",\"StopDate\":\"2026-04-23T06:14:39.090573+00:00\",\"Input\":{},\"InputDetails\":{\"Included\":true},\"Output\":{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"5ebb1494-6a85-4481-9f27-eb5a6e829873\"}]},\"OutputDetails\":{\"Included\":true}}",
"outputDetails": {
"included": true
}
}
--- post versions ---
{
"Versions": [
{
"VersionId": "d21024e2-dae9-44cf-a6fa-d97255c18caf",
"VersionStages": [
"AWSCURRENT"
],
"CreatedDate": "2026-04-23T06:14:39+00:00"
},
{
"VersionId": "d29ef77e-00aa-416b-a5b2-1548ef06f423",
"VersionStages": [
"AWSPREVIOUS"
],
"LastAccessedDate": "2026-04-23T00:00:00+00:00",
"CreatedDate": "2026-04-23T06:13:54+00:00"
}
],
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM",
"Name": "app/db/password"
}
AFTER value: aTticHGM9p6gX8XjcGdNo23FcNO6Yvh4
value changed: yes
--- notifier log (last 5 min) ---
START RequestId: 8fbfd785-15fc-4619-8ab4-aa931c8440a4 Version: $LATEST
notified: rotation complete secretId=app/db/password versionId=d21024e2-dae9-44cf-a6fa-d97255c18caf
END RequestId: 8fbfd785-15fc-4619-8ab4-aa931c8440a4
REPORT RequestId: 8fbfd785-15fc-4619-8ab4-aa931c8440a4 Duration: 8.35 ms Billed Duration: 9 ms Memory Size: 128 MB Max Memory Used: 128 MB
[stdout]
BEFORE stages: {
"117aa962-6223-46d9-a668-1b213c807f21": [
"AWSPREVIOUS"
],
"d29ef77e-00aa-416b-a5b2-1548ef06f423": [
"AWSCURRENT"
]
}
BEFORE value: C_fvHXOTpAAMCRJbOZAMZr2aaRs-SH-I
parent exec: arn:aws:states:us-east-1:000000000000:execution:rotation-orchestrator:502820c1-2683-4dd1-aefd-59c27bb692c9
parent status=SUCCEEDED in 2s
{
"executionArn": "arn:aws:states:us-east-1:000000000000:execution:rotation-orchestrator:502820c1-2683-4dd1-aefd-59c27bb692c9",
"stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:rotation-orchestrator",
"name": "502820c1-2683-4dd1-aefd-59c27bb692c9",
"status": "SUCCEEDED",
"startDate": "2026-04-23T06:14:37.726253+00:00",
"stopDate": "2026-04-23T06:14:39.288309+00:00",
"input": "{}",
"inputDetails": {
"included": true
},
"output": "{\"ExecutionArn\":\"arn:aws:states:us-east-1:000000000000:execution:rotation-child:c9fc6c58-bedd-472c-8a6f-97b440e61a95\",\"StateMachineArn\":\"arn:aws:states:us-east-1:000000000000:stateMachine:rotation-child\",\"Name\":\"c9fc6c58-bedd-472c-8a6f-97b440e61a95\",\"Status\":\"SUCCEEDED\",\"StartDate\":\"2026-04-23T06:14:37.754538+00:00\",\"StopDate\":\"2026-04-23T06:14:39.090573+00:00\",\"Input\":{},\"InputDetails\":{\"Included\":true},\"Output\":{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"5ebb1494-6a85-4481-9f27-eb5a6e829873\"}]},\"OutputDetails\":{\"Included\":true}}",
"outputDetails": {
"included": true
}
}
--- post versions ---
{
"Versions": [
{
"VersionId": "d21024e2-dae9-44cf-a6fa-d97255c18caf",
"VersionStages": [
"AWSCURRENT"
],
"CreatedDate": "2026-04-23T06:14:39+00:00"
},
{
"VersionId": "d29ef77e-00aa-416b-a5b2-1548ef06f423",
"VersionStages": [
"AWSPREVIOUS"
],
"LastAccessedDate": "2026-04-23T00:00:00+00:00",
"CreatedDate": "2026-04-23T06:13:54+00:00"
}
],
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-IdDYJM",
"Name": "app/db/password"
}
AFTER value: aTticHGM9p6gX8XjcGdNo23FcNO6Yvh4
value changed: yes
--- notifier log (last 5 min) ---
START RequestId: 8fbfd785-15fc-4619-8ab4-aa931c8440a4 Version: $LATEST
notified: rotation complete secretId=app/db/password versionId=d21024e2-dae9-44cf-a6fa-d97255c18caf
END RequestId: 8fbfd785-15fc-4619-8ab4-aa931c8440a4
REPORT RequestId: 8fbfd785-15fc-4619-8ab4-aa931c8440a4 Duration: 8.35 ms Billed Duration: 9 ms Memory Size: 128 MB Max Memory Used: 128 MBNo file changes, the agent left the workspace untouched.
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0 -- /root/.cache/uv/archive-v0/PxhhFO8behxJHsw2FGRWv/bin/python
cachedir: .pytest_cache
rootdir: /tests
plugins: json-ctrf-0.3.5
collecting ... collected 15 items
../tests/test_state.py::test_aws_cli_binary_is_real PASSED [ 6%]
../tests/test_state.py::test_localstack_reachable PASSED [ 13%]
../tests/test_state.py::test_secret_exists_and_uses_customer_cmk PASSED [ 20%]
../tests/test_state.py::test_rotation_lambda_exists PASSED [ 26%]
../tests/test_state.py::test_notification_lambda_exists PASSED [ 33%]
../tests/test_state.py::test_parent_and_child_state_machines_exist PASSED [ 40%]
../tests/test_state.py::test_event_bus_and_rule_exist PASSED [ 46%]
../tests/test_state.py::test_rule_target_points_at_notification_lambda PASSED [ 53%]
../tests/test_state.py::test_rotator_role_grants_kms_actions PASSED [ 60%]
../tests/test_state.py::test_kms_key_policy_grants_rotator_role FAILED [ 66%]
../tests/test_state.py::test_rotator_lambda_allows_secretsmanager_invoke FAILED [ 73%]
../tests/test_state.py::test_parent_role_grants_sync2_managed_rule_perms FAILED [ 80%]
../tests/test_state.py::test_child_role_can_rotate_and_publish FAILED [ 86%]
../tests/test_state.py::test_notification_lambda_allows_eventbridge_invoke PASSED [ 93%]
../tests/test_state.py::test_end_to_end_parent_execution_rotates_and_notifies PASSED [100%]
=================================== FAILURES ===================================
___________________ test_kms_key_policy_grants_rotator_role ____________________
iam = <botocore.client.IAM object at 0xffff8597de80>
kms = <botocore.client.KMS object at 0xffff85ec83b0>
def test_kms_key_policy_grants_rotator_role(iam, kms):
role_arn = iam.get_role(RoleName=ROTATOR_ROLE)["Role"]["Arn"]
policy_str = kms.get_key_policy(KeyId=KEY_ALIAS, PolicyName="default")[
"Policy"
]
policy = json.loads(policy_str)
match = False
for st in policy.get("Statement", []):
if st.get("Effect") != "Allow":
continue
principal = st.get("Principal") or {}
if not isinstance(principal, dict):
continue
arns = set(_normalise_list(principal.get("AWS")))
if role_arn not in arns:
continue
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, REQUIRED_KMS_ACTIONS):
match = True
break
> assert match, (
f"KMS key policy on {KEY_ALIAS} has no Allow statement whose "
f"Principal.AWS includes {role_arn} and whose Action covers "
f"{sorted(REQUIRED_KMS_ACTIONS)}. Key policy: {policy_str}"
)
E AssertionError: KMS key policy on alias/app-rotation-key has no Allow statement whose Principal.AWS includes arn:aws:iam::000000000000:role/secrets-rotator-role and whose Action covers ['kms:Decrypt', 'kms:GenerateDataKey']. Key policy: {"Version": "2012-10-17", "Id": "key-default-1", "Statement": [{"Sid": "Enable IAM User Permissions", "Effect": "Allow", "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, "Action": "kms:*", "Resource": "*"}]}
E assert False
/tests/test_state.py:322: AssertionError
_______________ test_rotator_lambda_allows_secretsmanager_invoke _______________
lmb = <botocore.client.Lambda object at 0xffff85e3f3e0>
def test_rotator_lambda_allows_secretsmanager_invoke(lmb):
try:
policy_doc = json.loads(
> lmb.get_policy(FunctionName=ROTATOR_FUNCTION)["Policy"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
/tests/test_state.py:332:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/root/.cache/uv/archive-v0/PxhhFO8behxJHsw2FGRWv/lib/python3.12/site-packages/botocore/client.py:569: in _api_call
return self._make_api_call(operation_name, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <botocore.client.Lambda object at 0xffff85e3f3e0>
operation_name = 'GetPolicy', api_params = {'FunctionName': 'secrets-rotator'}
def _make_api_call(self, operation_name, api_params):
operation_model = self._service_model.operation_model(operation_name)
service_name = self._service_model.service_name
history_recorder.record(
'API_CALL',
{
'service': service_name,
'operation': operation_name,
'params': api_params,
},
)
if operation_model.deprecated:
logger.debug(
'Warning: %s.%s() is deprecated', service_name, operation_name
)
request_context = {
'client_region': self.meta.region_name,
'client_config': self.meta.config,
'has_streaming_input': operation_model.has_streaming_input,
'auth_type': operation_model.resolved_auth_type,
'unsigned_payload': operation_model.unsigned_payload,
}
api_params = self._emit_api_params(
api_params=api_params,
operation_model=operation_model,
context=request_context,
)
(
endpoint_url,
additional_headers,
properties,
) = self._resolve_endpoint_ruleset(
operation_model, api_params, request_context
)
if properties:
# Pass arbitrary endpoint info with the Request
# for use during construction.
request_context['endpoint_properties'] = properties
request_dict = self._convert_to_request_dict(
api_params=api_params,
operation_model=operation_model,
endpoint_url=endpoint_url,
context=request_context,
headers=additional_headers,
)
resolve_checksum_context(request_dict, operation_model, api_params)
service_id = self._service_model.service_id.hyphenize()
handler, event_response = self.meta.events.emit_until_response(
f'before-call.{service_id}.{operation_name}',
model=operation_model,
params=request_dict,
request_signer=self._request_signer,
context=request_context,
)
if event_response is not None:
http, parsed_response = event_response
else:
maybe_compress_request(
self.meta.config, request_dict, operation_model
)
apply_request_checksum(request_dict)
http, parsed_response = self._make_request(
operation_model, request_dict, request_context
)
self.meta.events.emit(
f'after-call.{service_id}.{operation_name}',
http_response=http,
parsed=parsed_response,
model=operation_model,
context=request_context,
)
if http.status_code >= 300:
error_info = parsed_response.get("Error", {})
error_code = error_info.get("QueryErrorCode") or error_info.get(
"Code"
)
error_class = self.exceptions.from_code(error_code)
> raise error_class(parsed_response, operation_name)
E botocore.errorfactory.ResourceNotFoundException: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
/root/.cache/uv/archive-v0/PxhhFO8behxJHsw2FGRWv/lib/python3.12/site-packages/botocore/client.py:1023: ResourceNotFoundException
During handling of the above exception, another exception occurred:
lmb = <botocore.client.Lambda object at 0xffff85e3f3e0>
def test_rotator_lambda_allows_secretsmanager_invoke(lmb):
try:
policy_doc = json.loads(
lmb.get_policy(FunctionName=ROTATOR_FUNCTION)["Policy"]
)
except ClientError as e:
> pytest.fail(
f"Lambda {ROTATOR_FUNCTION} has no resource-based policy , "
f"secretsmanager.amazonaws.com cannot invoke it: {e}"
)
E Failed: Lambda secrets-rotator has no resource-based policy , secretsmanager.amazonaws.com cannot invoke it: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
/tests/test_state.py:335: Failed
_______________ test_parent_role_grants_sync2_managed_rule_perms _______________
iam = <botocore.client.IAM object at 0xffff8597de80>
def test_parent_role_grants_sync2_managed_rule_perms(iam):
"""The .sync:2 integration registers an EventBridge-managed callback
rule. Parent role needs events:PutRule/PutTargets/DescribeRule plus
full states:* on the child."""
found_events = False
found_states = False
for _pname, st in _inline_statements(iam, PARENT_ROLE):
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, REQUIRED_PARENT_EVENTS_ACTIONS):
found_events = True
if _actions_cover(actions, REQUIRED_PARENT_STATES_ACTIONS):
found_states = True
assert found_events, (
f"{PARENT_ROLE} missing events:PutRule/PutTargets/DescribeRule. "
f".sync:2 uses an EventBridge-managed callback rule; without "
f"these the parent cannot set up the callback."
)
> assert found_states, (
f"{PARENT_ROLE} missing states:StartExecution/DescribeExecution/"
f"StopExecution. .sync:2 requires all three on the child SM."
)
E AssertionError: rotation-orchestrator-role missing states:StartExecution/DescribeExecution/StopExecution. .sync:2 requires all three on the child SM.
E assert False
/tests/test_state.py:377: AssertionError
____________________ test_child_role_can_rotate_and_publish ____________________
iam = <botocore.client.IAM object at 0xffff8597de80>
def test_child_role_can_rotate_and_publish(iam):
found_rotate = False
found_putevents = False
for _pname, st in _inline_statements(iam, CHILD_ROLE):
actions = set(_normalise_list(st.get("Action")))
if _actions_cover(actions, {"secretsmanager:RotateSecret"}):
found_rotate = True
if _actions_cover(actions, {"events:PutEvents"}):
found_putevents = True
> assert found_rotate, (
f"{CHILD_ROLE} lacks secretsmanager:RotateSecret , the child "
f"state machine cannot trigger rotation."
)
E AssertionError: rotation-child-role lacks secretsmanager:RotateSecret , the child state machine cannot trigger rotation.
E assert False
/tests/test_state.py:392: AssertionError
=============================== warnings summary ===============================
test_state.py: 33 warnings
/root/.cache/uv/archive-v0/PxhhFO8behxJHsw2FGRWv/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
==================================== PASSES ====================================
=========================== short test summary info ============================
PASSED ../tests/test_state.py::test_aws_cli_binary_is_real
PASSED ../tests/test_state.py::test_localstack_reachable
PASSED ../tests/test_state.py::test_secret_exists_and_uses_customer_cmk
PASSED ../tests/test_
… (truncated at 12,000 chars, full verifier log is in the trial artifacts)Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_a629b8c916de4523. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_a629b8c916de4523 · verifier authoritative; classifier explanatory.