tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.List app directory and view setup script
ls /app/ && cat /app/setup.sh 2>/dev/null | head -200
handler.py
setup.sh
starter
#!/bin/bash
# Seeds LocalStack with the broken-by-design state the agent inherits:
# - customer-managed KMS CMK (alias alias/app-rotation-key)
# - Secrets Manager secret encrypted with that CMK
# - IAM role for the rotation Lambda (with deliberately incomplete policy)
# - Rotation Lambda (with a subtle bug in finishSecret)
# - Rotation configured on the secret, pointing at the Lambda
#
# The task expects these resources to already exist when the agent starts
# working. The agent must find the bugs and make `rotate-secret` actually
# advance AWSCURRENT end-to-end.
set -euo pipefail
REGION="${AWS_DEFAULT_REGION:-us-east-1}"
ACCOUNT_ID="000000000000"
SECRET_NAME="app/db/password"
ROLE_NAME="secrets-rotator-role"
FUNCTION="secrets-rotator"
KEY_ALIAS="alias/app-rotation-key"
log() { echo "[setup] $*" >&2; }
log "waiting for localstack health..."
for _ in $(seq 1 60); do
if curl -sf http://localstack:4566/_localstack/health | grep -q '"secretsmanager": "available"'; then
break
fi
sleep 2
done
# 1. CMK with a minimal key policy (root admin only , no grant for the
# rotation Lambda role yet).
log "creating KMS CMK"
KEY_POLICY=$(cat <<JSON
{
"Version": "2012-10-17",
"Id": "app-rotation-key-policy",
"Statement": [
{
"Sid": "EnableRootAdmin",
"Effect": "Allow",
"Principal": { "AWS": "arn:aws:iam::${ACCOUNT_ID}:root" },
"Action": "kms:*",
"Resource": "*"
}
]
}
JSON
)
KEY_ID=$(aws kms create-key \
--description "Customer CMK for app/db/password" \
--key-usage ENCRYPT_DECRYPT \
--policy "$KEY_POLICY" \
--query 'KeyMetadata.KeyId' --output text)
aws kms create-alias --alias-name "$KEY_ALIAS" --target-key-id "$KEY_ID" >/dev/null
KEY_ARN="arn:aws:kms:${REGION}:${ACCOUNT_ID}:key/${KEY_ID}"
log "created CMK $KEY_ID"
# 2. Secret, encrypted with the CMK.
log "creating secret"
aws secretsmanager create-secret \
--name "$SECRET_NAME" \
--kms-key-id "$KEY_ARN" \
--secret-string '{"password": "initial-placeholder-value"}' \
--description "App DB password, rotated by Lambda" >/dev/null
SECRET_ARN=$(aws secretsmanager describe-secret --secret-id "$SECRET_NAME" \
--query 'ARN' --output text)
log "created secret $SECRET_ARN"
# 3. IAM role for the rotation Lambda. Deliberately incomplete , has
# secretsmanager:* but no KMS actions. The broken state the agent
# inherits.
log "creating rotation Lambda role"
TRUST=$(cat <<'JSON'
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Principal": {"Service": "lambda.amazonaws.com"}, "Action": "sts:AssumeRole"}
]
}
JSON
)
aws iam create-role \
--role-name "$ROLE_NAME" \
--assume-role-policy-document "$TRUST" >/dev/null
aws iam attach-role-policy \
--role-name "$ROLE_NAME" \
--policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
INLINE=$(cat <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "${SECRET_ARN}"
},
{
"Effect": "Allow",
"Action": "secretsmanager:GetRandomPassword",
"Resource": "*"
}
]
}
JSON
)
aws iam put-role-policy \
--role-name "$ROLE_NAME" \
--policy-name SecretsRotatorPolicy \
--policy-document "$INLINE"
ROLE_ARN=$(aws iam get-role --role-name "$ROLE_NAME" --query 'Role.Arn' --output text)
log "created role $ROLE_ARN"
# 4. Package + deploy the rotation Lambda (with its buggy handler).
log "packaging rotation Lambda"
WORKDIR="$(mktemp -d)"
cp /app/handler.py "${WORKDIR}/handler.py"
(cd "$WORKDIR" && zip -q handler.zip handler.py)
aws lambda create-function \
--function-name "$FUNCTION" \
--runtime python3.11 \
--role "$ROLE_ARN" \
--handler handler.lambda_handler \
--timeout 30 \
--memory-size 256 \
--environment "Variables={SECRETS_MANAGER_ENDPOINT=http://localstack:4566}" \
--zip-file "fileb://${WORKDIR}/handler.zip" >/dev/null
for _ in $(seq 1 30); do
STATE=$(aws lambda get-function --function-name "$FUNCTION" \
--query 'Configuration.State' --output text 2>/dev/null || echo "Pending")
[ "$STATE" = "Active" ] && break
sleep 1
done
log "Lambda $FUNCTION active"
# NOTE: deliberately NOT calling `aws lambda add-permission` with
# principal secretsmanager.amazonaws.com. The agent must add that.
# 5. Attach rotation config on the secret. Secrets Manager refuses to
# attach unless the rotation Lambda already grants it InvokeFunction,
# so we temporarily add that permission, attach rotation, then remove
# the permission so the agent still has to re-add it as part of their
# fix. Net result: RotationEnabled=true but rotation fails at runtime
# because of the other broken layers (KMS grants, handler bug, Lambda
# resource policy).
log "attaching rotation config (temp Lambda permission)"
FUNCTION_ARN="arn:aws:lambda:${REGION}:${ACCOUNT_ID}:function:${FUNCTION}"
aws lambda add-permission \
--function-name "$FUNCTION" \
--statement-id TempRotationSetup \
--action lambda:InvokeFunction \
--principal secretsmanager.amazonaws.com \
--source-arn "$SECRET_ARN" >/dev/null 2>&1 || true
if aws secretsmanager rotate-secret \
--secret-id "$SECRET_NAME" \
--rotation-lambda-arn "$FUNCTION_ARN" \
--rotation-rules AutomaticallyAfterDays=30 \
--no-rotate-immediately >/dev/null 2>&1; then
log "rotation attached (no immediate trigger)"
elif aws secretsmanager rotate-secret \
--secret-id "$SECRET_NAME" \
--rotation-lambda-arn "$FUNCTION_ARN" \
--rotation-rules AutomaticallyAfterDays=30 >/dev/null 2>&1; then
log "rotation attached (first run may have fired)"
else
log "rotate-secret failed even with temp permission , check LocalStack compat"
fi
# Strip the temp permission so the agent still has to add it.
aws lambda remove-permission \
--function-name "$FUNCTION" \
--statement-id TempRotationSetup >/dev/null 2>&1 || true
log "done"
[stdout]
handler.py
setup.sh
starter
#!/bin/bash
# Seeds LocalStack with the broken-by-design state the agent inherits:
# - customer-managed KMS CMK (alias alias/app-rotation-key)
# - Secrets Manager secret encrypted with that CMK
# - IAM role for the rotation Lambda (with deliberately incomplete policy)
# - Rotation Lambda (with a subtle bug in finishSecret)
# - Rotation configured on the secret, pointing at the Lambda
#
# The task expects these resources to already exist when the agent starts
# working. The agent must find the bugs and make `rotate-secret` actually
# advance AWSCURRENT end-to-end.
set -euo pipefail
REGION="${AWS_DEFAULT_REGION:-us-east-1}"
ACCOUNT_ID="000000000000"
SECRET_NAME="app/db/password"
ROLE_NAME="secrets-rotator-role"
FUNCTION="secrets-rotator"
KEY_ALIAS="alias/app-rotation-key"
log() { echo "[setup] $*" >&2; }
log "waiting for localstack health..."
for _ in $(seq 1 60); do
if curl -sf http://localstack:4566/_localstack/health | grep -q '"secretsmanager": "available"'; then
break
fi
sleep 2
done
# 1. CMK with a minimal key policy (root admin only , no grant for the
# rotation Lambda role yet).
log "creating KMS CMK"
KEY_POLICY=$(cat <<JSON
{
"Version": "2012-10-17",
"Id": "app-rotation-key-policy",
"Statement": [
{
"Sid": "EnableRootAdmin",
"Effect": "Allow",
"Principal": { "AWS": "arn:aws:iam::${ACCOUNT_ID}:root" },
"Action": "kms:*",
"Resource": "*"
}
]
}
JSON
)
KEY_ID=$(aws kms create-key \
--description "Customer CMK for app/db/password" \
--key-usage ENCRYPT_DECRYPT \
--policy "$KEY_POLICY" \
--query 'KeyMetadata.KeyId' --output text)
aws kms create-alias --alias-name "$KEY_ALIAS" --target-key-id "$KEY_ID" >/dev/null
KEY_ARN="arn:aws:kms:${REGION}:${ACCOUNT_ID}:key/${KEY_ID}"
log "created CMK $KEY_ID"
# 2. Secret, encrypted with the CMK.
log "creating secret"
aws secretsmanager create-secret \
--name "$SECRET_NAME" \
--kms-key-id "$KEY_ARN" \
--secret-string '{"password": "initial-placeholder-value"}' \
--description "App DB password, rotated by Lambda" >/dev/null
SECRET_ARN=$(aws secretsmanager describe-secret --secret-id "$SECRET_NAME" \
--query 'ARN' --output text)
log "created secret $SECRET_ARN"
# 3. IAM role for the rotation Lambda. Deliberately incomplete , has
# secretsmanager:* but no KMS actions. The broken state the agent
# inherits.
log "creating rotation Lambda role"
TRUST=$(cat <<'JSON'
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Principal": {"Service": "lambda.amazonaws.com"}, "Action": "sts:AssumeRole"}
]
}
JSON
)
aws iam create-role \
--role-name "$ROLE_NAME" \
--assume-role-policy-document "$TRUST" >/dev/null
aws iam attach-role-policy \
--role-name "$ROLE_NAME" \
--policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
INLINE=$(cat <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "${SECRET_ARN}"
},
{
"Effect": "Allow",
"Action": "secretsmanager:GetRandomPassword",
"Resource": "*"
}
]
}
JSON
)
aws iam put-role-policy \
--role-name "$ROLE_NAME" \
--policy-name SecretsRotatorPolicy \
--policy-document "$INLINE"
ROLE_ARN=$(aws iam get-role --role-name "$ROLE_NAME" --query 'Role.Arn' --output text)
log "created role $ROLE_ARN"
# 4. Package + deploy the rotation Lambda (with its buggy handler).
log "packaging rotation Lambda"
WORKDIR="$(mktemp -d)"
cp /app/handler.py "${WORKDIR}/handler.py"
(cd "$WORKDIR" && zip -q handler.zip handler.py)
aws lambda create-function \
--function-name "$FUNCTION" \
--runtime python3.11 \
--role "$ROLE_ARN" \
--handler handler.lambda_handler \
--timeout 30 \
--memory-size 256 \
--environment "Variables={SECRETS_MANAGER_ENDPOINT=http://localstack:4566}" \
--zip-file "fileb://${WORKDIR}/handler.zip" >/dev/null
for _ in $(seq 1 30); do
STATE=$(aws lambda get-function --function-name "$FUNCTION" \
--query 'Configuration.State' --output text 2>/dev/null || echo "Pending")
[ "$STATE" = "Active" ] && break
sleep 1
done
log "Lambda $FUNCTION active"
# NOTE: deliberately NOT calling `aws lambda add-permission` with
# principal secretsmanager.amazonaws.com. The agent must add that.
# 5. Attach rotation config on the secret. Secrets Manager refuses to
# attach unless the rotation Lambda already grants it InvokeFunction,
# so we temporarily add that permission, attach rotation, then remove
# the permission so the agent still has to re-add it as part of their
# fix. Net result: RotationEnabled=true but rotation fails at runtime
# because of the other broken layers (KMS grants, handler bug, Lambda
# resource policy).
log "attaching rotation config (temp Lambda permission)"
FUNCTION_ARN="arn:aws:lambda:${REGION}:${ACCOUNT_ID}:function:${FUNCTION}"
aws lambda add-permission \
--function-name "$FUNCTION" \
--statement-id TempRotationSetup \
--action lambda:InvokeFunction \
--principal secretsmanager.amazonaws.com \
--source-arn "$SECRET_ARN" >/dev/null 2>&1 || true
if aws secretsmanager rotate-secret \
--secret-id "$SECRET_NAME" \
--rotation-lambda-arn "$FUNCTION_ARN" \
--rotation-rules AutomaticallyAfterDays=30 \
--no-rotate-immediately >/dev/null 2>&1; then
log "rotation attached (no immediate trigger)"
elif aws secretsmanager rotate-secret \
--secret-id "$SECRET_NAME" \
--rotation-lambda-arn "$FUNCTION_ARN" \
--rotation-rules AutomaticallyAfterDays=30 >/dev/null 2>&1; then
log "rotation attached (first run may have fired)"
else
log "rotate-secret failed even with temp permission , check LocalStack compat"
fi
# Strip the temp permission so the agent still has to add it.
aws lambda remove-permission \
--function-name "$FUNCTION" \
--statement-id TempRotationSetup >/dev/null 2>&1 || true
log "done"/app/handler.py
1 # Rotation Lambda for AWS Secrets Manager, "single-user" pattern
2 # (no external database, value is self-contained).
3 #
4 # Secrets Manager calls this function 4 times in sequence per rotation,
5 # passing a Step field: createSecret, setSecret, testSecret, finishSecret.
6 # Each step must return cleanly for the rotation to advance.
7 #
8 # Reference:
9 # https://docs.aws.amazon.com/secretsmanager/latest/userguide/rotate-secrets_lambda-functions.html
10
11 import json
12 import logging
13 import os
14
15 import boto3
16
17 logger = logging.getLogger()
18 logger.setLevel(logging.INFO)
19
20 ENDPOINT = os.environ.get("SECRETS_MANAGER_ENDPOINT") or os.environ.get(
21 "AWS_ENDPOINT_URL"
22 )
23
24
25 def _client():
26 return boto3.client("secretsmanager", endpoint_url=ENDPOINT) if ENDPOINT else boto3.client("secretsmanager")
27
28
29 def lambda_handler(event, context):
30 arn = event["SecretId"]
31 token = event["ClientRequestToken"]
32 step = event["Step"]
33
34 client = _client()
35
36 desc = client.describe_secret(SecretId=arn)
37 if not desc.get("RotationEnabled"):
38 logger.error("Secret %s is not enabled for rotation", arn)
39 raise ValueError(f"Secret {arn} is not enabled for rotation")
40
41 versions = desc.get("VersionIdsToStages", {})
42 if token not in versions:
43 logger.error("Secret version %s has no stage for rotation of %s", token, arn)
44 raise ValueError(f"Secret version {token} has no stage for rotation of secret {arn}")
45 if "AWSCURRENT" in versions[token]:
46 logger.info("Secret version %s already AWSCURRENT for %s", token, arn)
47 return
48 if "AWSPENDING" not in versions[token]:
49 logger.error("Secret version %s not staged as AWSPENDING for %s", token, arn)
50 raise ValueError(f"Secret version {token} not set as AWSPENDING for rotation of secret {arn}")
51
52 if step == "createSecret":
53 create_secret(client, arn, token)
54 elif step == "setSecret":
55 set_secret(client, arn, token)
56 elif step == "testSecret":
57 test_secret(client, arn, token)
58 elif step == "finishSecret":
59 finish_secret(client, arn, token)
60 else:
61 raise ValueError(f"Invalid step parameter: {step}")
62
63
64 def create_secret(client, arn, token):
65 # Generate a new candidate value and stash it as AWSPENDING.
66 client.get_secret_value(SecretId=arn, VersionStage="AWSCURRENT")
67
68 try:
69 client.get_secret_value(SecretId=arn, VersionId=token, VersionStage="AWSPENDING")
70 logger.info("createSecret: pending version %s already exists", token)
71 except client.exceptions.ResourceNotFoundException:
72 password = client.get_random_password(PasswordLength=32, ExcludePunctuation=True)
73 client.put_secret_value(
74 SecretId=arn,
75 ClientRequestToken=token,
76 SecretString=json.dumps({"password": password["RandomPassword"]}),
77 VersionStages=["AWSPENDING"],
78 )
79 logger.info("createSecret: stored new AWSPENDING version %s", token)
80
81
82 def set_secret(client, arn, token):
83 # No external system to update for a self-contained secret; validate
84 # that both AWSCURRENT and AWSPENDING are readable and move on.
85 client.get_secret_value(SecretId=arn, VersionStage="AWSCURRENT")
86 client.get_secret_value(SecretId=arn, VersionId=token, VersionStage="AWSPENDING")
87 logger.info("setSecret: both stages readable")
88
89
90 def test_secret(client, arn, token):
91 # Sanity-check the pending value parses as JSON with a non-empty
92 # password field. Real handlers would attempt a login here.
93 pending = client.get_secret_value(SecretId=arn, VersionId=token, VersionStage="AWSPENDING")
94 parsed = json.loads(pending["SecretString"])
95 if not parsed.get("password"):
96 raise ValueError("testSecret: pending value has no password")
97 logger.info("testSecret: pending value validated")
98
99
100 def finish_secret(client, arn, token):
101 # TODO: promote the AWSPENDING version to AWSCURRENT here.
102 # Without this, rotation will appear to succeed from Secrets
103 # Manager's orchestrator but AWSCURRENT never advances and
104 # consumers keep reading the old password.
105 logger.info("finishSecret: (no-op)")
106
Describe current secret state
aws secretsmanager describe-secret --secret-id app/db/password 2>&1
{
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ",
"Name": "app/db/password",
"Description": "App DB password, rotated by Lambda",
"KmsKeyId": "arn:aws:kms:us-east-1:000000000000:key/d4743a2c-843d-4b4f-a23d-e2125c185960",
"RotationEnabled": true,
"RotationLambdaARN": "arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator",
"RotationRules": {
"AutomaticallyAfterDays": 30
},
"LastRotatedDate": "2026-04-22T19:36:59+00:00",
"LastChangedDate": "2026-04-22T19:36:55.563000+00:00",
"LastAccessedDate": "2026-04-22T00:00:00+00:00",
"NextRotationDate": "2026-05-22T19:36:54+00:00",
"VersionIdsToStages": {
"47f72e74-4dcc-48a7-8fd7-77bec90c4386": [
"AWSCURRENT"
],
"13140d44-8143-40f8-8e8a-602809ea919d": [
"AWSPENDING"
]
},
"CreatedDate": "2026-04-22T19:36:45.341312+00:00"
}
[stdout]
{
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ",
"Name": "app/db/password",
"Description": "App DB password, rotated by Lambda",
"KmsKeyId": "arn:aws:kms:us-east-1:000000000000:key/d4743a2c-843d-4b4f-a23d-e2125c185960",
"RotationEnabled": true,
"RotationLambdaARN": "arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator",
"RotationRules": {
"AutomaticallyAfterDays": 30
},
"LastRotatedDate": "2026-04-22T19:36:59+00:00",
"LastChangedDate": "2026-04-22T19:36:55.563000+00:00",
"LastAccessedDate": "2026-04-22T00:00:00+00:00",
"NextRotationDate": "2026-05-22T19:36:54+00:00",
"VersionIdsToStages": {
"47f72e74-4dcc-48a7-8fd7-77bec90c4386": [
"AWSCURRENT"
],
"13140d44-8143-40f8-8e8a-602809ea919d": [
"AWSPENDING"
]
},
"CreatedDate": "2026-04-22T19:36:45.341312+00:00"
}Check Lambda policy and role policy
aws lambda get-policy --function-name secrets-rotator 2>&1; echo "---"; aws iam get-role-policy --role-name secrets-rotator-role --policy-name SecretsRotatorPolicy 2>&1
aws: [ERROR]: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
Additional error details:
Type: User
---
{
"RoleName": "secrets-rotator-role",
"PolicyName": "SecretsRotatorPolicy",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ"
},
{
"Effect": "Allow",
"Action": "secretsmanager:GetRandomPassword",
"Resource": "*"
}
]
}
}
[stdout]
aws: [ERROR]: An error occurred (ResourceNotFoundException) when calling the GetPolicy operation: The resource you requested does not exist.
Additional error details:
Type: User
---
{
"RoleName": "secrets-rotator-role",
"PolicyName": "SecretsRotatorPolicy",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ"
},
{
"Effect": "Allow",
"Action": "secretsmanager:GetRandomPassword",
"Resource": "*"
}
]
}
}Check KMS key policy
aws kms describe-key --key-id alias/app-rotation-key 2>&1; echo "---"; aws kms get-key-policy --key-id alias/app-rotation-key --policy-name default 2>&1
{
"KeyMetadata": {
"AWSAccountId": "000000000000",
"KeyId": "d4743a2c-843d-4b4f-a23d-e2125c185960",
"Arn": "arn:aws:kms:us-east-1:000000000000:key/d4743a2c-843d-4b4f-a23d-e2125c185960",
"CreationDate": "2026-04-22T19:36:44.599004+00:00",
"Enabled": true,
"Description": "Customer CMK for app/db/password",
"KeyUsage": "ENCRYPT_DECRYPT",
"KeyState": "Enabled",
"Origin": "AWS_KMS",
"KeyManager": "CUSTOMER",
"CustomerMasterKeySpec": "SYMMETRIC_DEFAULT",
"KeySpec": "SYMMETRIC_DEFAULT",
"EncryptionAlgorithms": [
"SYMMETRIC_DEFAULT"
],
"MultiRegion": false,
"CurrentKeyMaterialId": "36b96cf345b851568746f057987d41c836b96cf345b851568746f057987d41c8"
}
}
---
{
"Policy": "{\n \"Version\": \"2012-10-17\",\n \"Id\": \"app-rotation-key-policy\",\n \"Statement\": [\n {\n \"Sid\": \"EnableRootAdmin\",\n \"Effect\": \"Allow\",\n \"Principal\": { \"AWS\": \"arn:aws:iam::000000000000:root\" },\n \"Action\": \"kms:*\",\n \"Resource\": \"*\"\n }\n ]\n}"
}
[stdout]
{
"KeyMetadata": {
"AWSAccountId": "000000000000",
"KeyId": "d4743a2c-843d-4b4f-a23d-e2125c185960",
"Arn": "arn:aws:kms:us-east-1:000000000000:key/d4743a2c-843d-4b4f-a23d-e2125c185960",
"CreationDate": "2026-04-22T19:36:44.599004+00:00",
"Enabled": true,
"Description": "Customer CMK for app/db/password",
"KeyUsage": "ENCRYPT_DECRYPT",
"KeyState": "Enabled",
"Origin": "AWS_KMS",
"KeyManager": "CUSTOMER",
"CustomerMasterKeySpec": "SYMMETRIC_DEFAULT",
"KeySpec": "SYMMETRIC_DEFAULT",
"EncryptionAlgorithms": [
"SYMMETRIC_DEFAULT"
],
"MultiRegion": false,
"CurrentKeyMaterialId": "36b96cf345b851568746f057987d41c836b96cf345b851568746f057987d41c8"
}
}
---
{
"Policy": "{\n \"Version\": \"2012-10-17\",\n \"Id\": \"app-rotation-key-policy\",\n \"Statement\": [\n {\n \"Sid\": \"EnableRootAdmin\",\n \"Effect\": \"Allow\",\n \"Principal\": { \"AWS\": \"arn:aws:iam::000000000000:root\" },\n \"Action\": \"kms:*\",\n \"Resource\": \"*\"\n }\n ]\n}"
}/app/handler.py
def finish_secret(client, arn, token):
# TODO: promote the AWSPENDING version to AWSCURRENT here.
# Without this, rotation will appear to succeed from Secrets
# Manager's orchestrator but AWSCURRENT never advances and
# consumers keep reading the old password.
logger.info("finishSecret: (no-op)")def finish_secret(client, arn, token):
desc = client.describe_secret(SecretId=arn)
current_version = None
for version_id, stages in desc.get("VersionIdsToStages", {}).items():
if "AWSCURRENT" in stages:
if version_id == token:
logger.info("finishSecret: %s already AWSCURRENT", token)
return
current_version = version_id
break
client.update_secret_version_stage(
SecretId=arn,
VersionStage="AWSCURRENT",
MoveToVersionId=token,
RemoveFromVersionId=current_version,
)
logger.info("finishSecret: moved AWSCURRENT from %s to %s", current_version, token)Repackage and redeploy Lambda
WORKDIR=$(mktemp -d) && cp /app/handler.py "$WORKDIR/" && (cd "$WORKDIR" && zip -q handler.zip handler.py) && aws lambda update-function-code --function-name secrets-rotator --zip-file "fileb://${WORKDIR}/handler.zip" --query 'LastUpdateStatus' --output text && sleep 2 && aws lambda get-function --function-name secrets-rotator --query 'Configuration.[State,LastUpdateStatus]' --output textInProgress Active Successful [stdout] InProgress Active Successful
Capture ARNs for configuration
SECRET_ARN=$(aws secretsmanager describe-secret --secret-id app/db/password --query 'ARN' --output text) KEY_ARN=$(aws kms describe-key --key-id alias/app-rotation-key --query 'KeyMetadata.Arn' --output text) ROLE_ARN=$(aws iam get-role --role-name secrets-rotator-role --query 'Role.Arn' --output text) echo "SECRET_ARN=$SECRET_ARN" echo "KEY_ARN=$KEY_ARN" echo "ROLE_ARN=$ROLE_ARN"
SECRET_ARN=arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ KEY_ARN=arn:aws:kms:us-east-1:000000000000:key/d4743a2c-843d-4b4f-a23d-e2125c185960 ROLE_ARN=arn:aws:iam::000000000000:role/secrets-rotator-role [stdout] SECRET_ARN=arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ KEY_ARN=arn:aws:kms:us-east-1:000000000000:key/d4743a2c-843d-4b4f-a23d-e2125c185960 ROLE_ARN=arn:aws:iam::000000000000:role/secrets-rotator-role
Update IAM role policy with KMS permissions
SECRET_ARN="arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ"
KEY_ARN="arn:aws:kms:us-east-1:000000000000:key/d4743a2c-843d-4b4f-a23d-e2125c185960"
cat > /tmp/role-policy.json <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue",
"secretsmanager:PutSecretValue",
"secretsmanager:UpdateSecretVersionStage"
],
"Resource": "${SECRET_ARN}"
},
{
"Effect": "Allow",
"Action": "secretsmanager:GetRandomPassword",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": "${KEY_ARN}",
"Condition": {
"StringEquals": {
"kms:ViaService": "secretsmanager.us-east-1.amazonaws.com"
}
}
}
]
}
JSON
aws iam put-role-policy \
--role-name secrets-rotator-role \
--policy-name SecretsRotatorPolicy \
--policy-document file:///tmp/role-policy.json
echo "role policy updated"role policy updated [stdout] role policy updated
Update KMS key policy to grant role access
cat > /tmp/key-policy.json <<'JSON'
{
"Version": "2012-10-17",
"Id": "app-rotation-key-policy",
"Statement": [
{
"Sid": "EnableRootAdmin",
"Effect": "Allow",
"Principal": { "AWS": "arn:aws:iam::000000000000:root" },
"Action": "kms:*",
"Resource": "*"
},
{
"Sid": "AllowRotationLambdaUseViaSecretsManager",
"Effect": "Allow",
"Principal": { "AWS": "arn:aws:iam::000000000000:role/secrets-rotator-role" },
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": "*",
"Condition": {
"StringEquals": {
"kms:ViaService": "secretsmanager.us-east-1.amazonaws.com"
}
}
}
]
}
JSON
aws kms put-key-policy \
--key-id alias/app-rotation-key \
--policy-name default \
--policy file:///tmp/key-policy.json
echo "key policy updated"key policy updated [stdout] key policy updated
Add Secrets Manager invoke permission to Lambda
SECRET_ARN="arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ" aws lambda add-permission \ --function-name secrets-rotator \ --statement-id SecretsManagerRotate \ --action lambda:InvokeFunction \ --principal secretsmanager.amazonaws.com \ --source-arn "$SECRET_ARN"
{
"Statement": "{\"Sid\": \"SecretsManagerRotate\", \"Effect\": \"Allow\", \"Action\": \"lambda:InvokeFunction\", \"Resource\": \"arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator\", \"Principal\": {\"Service\": \"secretsmanager.amazonaws.com\"}, \"Condition\": {\"ArnLike\": {\"AWS:SourceArn\": \"arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ\"}}}"
}
[stdout]
{
"Statement": "{\"Sid\": \"SecretsManagerRotate\", \"Effect\": \"Allow\", \"Action\": \"lambda:InvokeFunction\", \"Resource\": \"arn:aws:lambda:us-east-1:000000000000:function:secrets-rotator\", \"Principal\": {\"Service\": \"secretsmanager.amazonaws.com\"}, \"Condition\": {\"ArnLike\": {\"AWS:SourceArn\": \"arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ\"}}}"
}Clear stuck AWSPENDING version
CURRENT=$(aws secretsmanager get-secret-value --secret-id app/db/password --query 'VersionId' --output text) echo "AWSCURRENT version before: $CURRENT" aws secretsmanager get-secret-value --secret-id app/db/password --query 'SecretString' --output text echo "---" # Clear stuck AWSPENDING PENDING=$(aws secretsmanager describe-secret --secret-id app/db/password --query 'VersionIdsToStages' --output json | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((v for v,s in d.items() if 'AWSPENDING' in s and 'AWSCURRENT' not in s), ''))") if [ -n "$PENDING" ]; then echo "clearing stuck AWSPENDING version: $PENDING" aws secretsmanager update-secret-version-stage --secret-id app/db/password --version-stage AWSPENDING --remove-from-version-id "$PENDING" fi aws secretsmanager describe-secret --secret-id app/db/password --query 'VersionIdsToStages'
AWSCURRENT version before: 47f72e74-4dcc-48a7-8fd7-77bec90c4386
{"password": "initial-placeholder-value"}
---
clearing stuck AWSPENDING version: 13140d44-8143-40f8-8e8a-602809ea919d
{
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ",
"Name": "app/db/password"
}
{
"47f72e74-4dcc-48a7-8fd7-77bec90c4386": [
"AWSCURRENT"
]
}
[stdout]
AWSCURRENT version before: 47f72e74-4dcc-48a7-8fd7-77bec90c4386
{"password": "initial-placeholder-value"}
---
clearing stuck AWSPENDING version: 13140d44-8143-40f8-8e8a-602809ea919d
{
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ",
"Name": "app/db/password"
}
{
"47f72e74-4dcc-48a7-8fd7-77bec90c4386": [
"AWSCURRENT"
]
}Trigger rotation
aws secretsmanager rotate-secret --secret-id app/db/password 2>&1
{
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ",
"Name": "app/db/password",
"VersionId": "c9a790c7-2311-4f4a-a716-733fc8ecff9d"
}
[stdout]
{
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:app/db/password-goqRSZ",
"Name": "app/db/password",
"VersionId": "c9a790c7-2311-4f4a-a716-733fc8ecff9d"
}Check rotation result
sleep 5 echo "=== describe-secret ===" aws secretsmanager describe-secret --secret-id app/db/password --query 'VersionIdsToStages' echo "" echo "=== get-secret-value AWSCURRENT ===" aws secretsmanager get-secret-value --secret-id app/db/password --query '[VersionId,SecretString]' --output text
=== describe-secret ===
{
"47f72e74-4dcc-48a7-8fd7-77bec90c4386": [
"AWSPREVIOUS"
],
"c9a790c7-2311-4f4a-a716-733fc8ecff9d": [
"AWSPENDING",
"AWSCURRENT"
]
}
=== get-secret-value AWSCURRENT ===
c9a790c7-2311-4f4a-a716-733fc8ecff9d {"password": "5quBPPWCOTsYPrCDrjEWzwFvqiNtXwG9"}
[stdout]
=== describe-secret ===
{
"47f72e74-4dcc-48a7-8fd7-77bec90c4386": [
"AWSPREVIOUS"
],
"c9a790c7-2311-4f4a-a716-733fc8ecff9d": [
"AWSPENDING",
"AWSCURRENT"
]
}
=== get-secret-value AWSCURRENT ===
c9a790c7-2311-4f4a-a716-733fc8ecff9d {"password": "5quBPPWCOTsYPrCDrjEWzwFvqiNtXwG9"}No file changes, the agent left the workspace untouched.
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0 -- /root/.cache/uv/archive-v0/opfF8Xes-D2pUs5uwd8hT/bin/python
cachedir: .pytest_cache
rootdir: /tests
plugins: json-ctrf-0.3.5
collecting ... collected 8 items
../tests/test_outputs.py::test_aws_cli_binary_is_real PASSED [ 12%]
../tests/test_outputs.py::test_localstack_reachable PASSED [ 25%]
../tests/test_outputs.py::test_secret_exists_and_uses_customer_cmk PASSED [ 37%]
../tests/test_outputs.py::test_rotation_lambda_exists PASSED [ 50%]
../tests/test_outputs.py::test_role_policy_grants_kms_actions PASSED [ 62%]
../tests/test_outputs.py::test_kms_key_policy_grants_lambda_role PASSED [ 75%]
../tests/test_outputs.py::test_lambda_permission_allows_secretsmanager_invoke PASSED [ 87%]
../tests/test_outputs.py::test_rotate_secret_advances_awscurrent PASSED [100%]
=============================== warnings summary ===============================
test_outputs.py: 14 warnings
/root/.cache/uv/archive-v0/opfF8Xes-D2pUs5uwd8hT/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
==================================== PASSES ====================================
=========================== short test summary info ============================
PASSED ../tests/test_outputs.py::test_aws_cli_binary_is_real
PASSED ../tests/test_outputs.py::test_localstack_reachable
PASSED ../tests/test_outputs.py::test_secret_exists_and_uses_customer_cmk
PASSED ../tests/test_outputs.py::test_rotation_lambda_exists
PASSED ../tests/test_outputs.py::test_role_policy_grants_kms_actions
PASSED ../tests/test_outputs.py::test_kms_key_policy_grants_lambda_role
PASSED ../tests/test_outputs.py::test_lambda_permission_allows_secretsmanager_invoke
PASSED ../tests/test_outputs.py::test_rotate_secret_advances_awscurrent
======================== 8 passed, 14 warnings in 6.59s ========================Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_ca22c4bf94cc436e. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_ca22c4bf94cc436e · verifier authoritative; classifier explanatory.