tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.Check environment and tools
ls /app && echo "---" && which aws python3 zip jq && echo "---" && python3 -c "import boto3; print(boto3.__version__)"
--- /usr/local/bin/aws /opt/venv/bin/python3 /usr/bin/zip /usr/bin/jq --- 1.35.99 [stdout] --- /usr/local/bin/aws /opt/venv/bin/python3 /usr/bin/zip /usr/bin/jq --- 1.35.99
Check localstack health
curl -s http://localstack:4566/_localstack/health | head -200
{"services": {"acm": "disabled", "apigateway": "disabled", "cloudformation": "disabled", "cloudwatch": "disabled", "config": "disabled", "dynamodb": "available", "dynamodbstreams": "available", "ec2": "disabled", "es": "disabled", "events": "available", "firehose": "disabled", "iam": "available", "kinesis": "available", "kms": "available", "lambda": "available", "logs": "available", "opensearch": "disabled", "redshift": "disabled", "resource-groups": "disabled", "resourcegroupstaggingapi": "disabled", "route53": "disabled", "route53resolver": "disabled", "s3": "available", "s3control": "disabled", "scheduler": "disabled", "secretsmanager": "available", "ses": "disabled", "sns": "disabled", "sqs": "available", "ssm": "disabled", "stepfunctions": "available", "sts": "available", "support": "disabled", "swf": "disabled", "transcribe": "disabled"}, "edition": "community", "version": "4.13.1"}
[stdout]
{"services": {"acm": "disabled", "apigateway": "disabled", "cloudformation": "disabled", "cloudwatch": "disabled", "config": "disabled", "dynamodb": "available", "dynamodbstreams": "available", "ec2": "disabled", "es": "disabled", "events": "available", "firehose": "disabled", "iam": "available", "kinesis": "available", "kms": "available", "lambda": "available", "logs": "available", "opensearch": "disabled", "redshift": "disabled", "resource-groups": "disabled", "resourcegroupstaggingapi": "disabled", "route53": "disabled", "route53resolver": "disabled", "s3": "available", "s3control": "disabled", "scheduler": "disabled", "secretsmanager": "available", "ses": "disabled", "sns": "disabled", "sqs": "available", "ssm": "disabled", "stepfunctions": "available", "sts": "available", "support": "disabled", "swf": "disabled", "transcribe": "disabled"}, "edition": "community", "version": "4.13.1"}Set up workspace and check AWS env
mkdir -p /app/saga/lambdas && cd /app/saga && ls && echo "AWS env:" && env | grep -E "^AWS_"
lambdas AWS env: AWS_DEFAULT_REGION=us-east-1 AWS_PAGER= AWS_SECRET_ACCESS_KEY=test AWS_ACCESS_KEY_ID=test AWS_ENDPOINT_URL=http://localstack:4566 [stdout] lambdas AWS env: AWS_DEFAULT_REGION=us-east-1 AWS_PAGER= AWS_SECRET_ACCESS_KEY=test AWS_ACCESS_KEY_ID=test AWS_ENDPOINT_URL=http://localstack:4566
import os
import time
import boto3
from botocore.exceptions import ClientError
TABLE_INVENTORY = os.environ["TABLE_INVENTORY"]
TABLE_IDEM = os.environ["TABLE_IDEM"]
ddb = boto3.resource("dynamodb")
inv = ddb.Table(TABLE_INVENTORY)
idem = ddb.Table(TABLE_IDEM)
STEP = "reserve"
def handler(event, context):
sku = event["sku"]
qty = int(event["qty"])
key = f"{STEP}:{event['idempotency_key']}"
existing = idem.get_item(Key={"idempotency_key": key}).get("Item")
if existing:
return {"status": "already_reserved", "sku": sku, "qty": qty}
inv.update_item(
Key={"sku": sku},
UpdateExpression="ADD reserved_qty :q",
ExpressionAttributeValues={":q": qty},
)
try:
idem.put_item(
Item={
"idempotency_key": key,
"step": STEP,
"sku": sku,
"qty": qty,
"ttl": int(time.time()) + 86400,
},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise
return {"status": "reserved", "sku": sku, "qty": qty}
import os
import time
import boto3
from botocore.exceptions import ClientError
TABLE_PAYMENTS = os.environ["TABLE_PAYMENTS"]
TABLE_IDEM = os.environ["TABLE_IDEM"]
SECRET_ID = os.environ["SECRET_ID"]
ddb = boto3.resource("dynamodb")
payments = ddb.Table(TABLE_PAYMENTS)
idem = ddb.Table(TABLE_IDEM)
sm = boto3.client("secretsmanager")
STEP = "charge"
def handler(event, context):
if event.get("force_failure"):
raise RuntimeError("forced payment failure for compensation test")
order_id = event["order_id"]
amount = event["amount"]
key = f"{STEP}:{event['idempotency_key']}"
payment_id = f"pay-{order_id}"
existing = idem.get_item(Key={"idempotency_key": key}).get("Item")
if existing:
return {"status": "already_captured", "payment_id": payment_id}
secret = sm.get_secret_value(SecretId=SECRET_ID)
api_key_present = bool(secret.get("SecretString"))
print(f"charge-payment: loaded gateway api key ok={api_key_present} order={order_id}")
payments.put_item(
Item={
"payment_id": payment_id,
"order_id": order_id,
"amount": amount,
"status": "CAPTURED",
"captured_at": int(time.time()),
}
)
try:
idem.put_item(
Item={
"idempotency_key": key,
"step": STEP,
"payment_id": payment_id,
"ttl": int(time.time()) + 86400,
},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise
return {"status": "CAPTURED", "payment_id": payment_id}
import os
import time
import boto3
from botocore.exceptions import ClientError
TABLE_SHIPMENTS = os.environ["TABLE_SHIPMENTS"]
TABLE_IDEM = os.environ["TABLE_IDEM"]
ddb = boto3.resource("dynamodb")
shipments = ddb.Table(TABLE_SHIPMENTS)
idem = ddb.Table(TABLE_IDEM)
STEP = "ship"
def handler(event, context):
if event.get("force_ship_failure"):
raise RuntimeError("forced shipment failure")
order_id = event["order_id"]
key = f"{STEP}:{event['idempotency_key']}"
shipment_id = f"ship-{order_id}"
existing = idem.get_item(Key={"idempotency_key": key}).get("Item")
if existing:
return {"status": "already_shipped", "shipment_id": shipment_id}
shipments.put_item(
Item={
"shipment_id": shipment_id,
"order_id": order_id,
"status": "CREATED",
"created_at": int(time.time()),
}
)
try:
idem.put_item(
Item={
"idempotency_key": key,
"step": STEP,
"shipment_id": shipment_id,
"ttl": int(time.time()) + 86400,
},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise
return {"status": "CREATED", "shipment_id": shipment_id}
import os
import time
import boto3
from botocore.exceptions import ClientError
TABLE_INVENTORY = os.environ["TABLE_INVENTORY"]
TABLE_IDEM = os.environ["TABLE_IDEM"]
ddb = boto3.resource("dynamodb")
inv = ddb.Table(TABLE_INVENTORY)
idem = ddb.Table(TABLE_IDEM)
STEP = "release"
def handler(event, context):
sku = event["sku"]
qty = int(event["qty"])
key = f"{STEP}:{event['idempotency_key']}"
existing = idem.get_item(Key={"idempotency_key": key}).get("Item")
if existing:
return {"status": "already_released", "sku": sku}
try:
inv.update_item(
Key={"sku": sku},
UpdateExpression="ADD reserved_qty :nq",
ConditionExpression="attribute_exists(sku) AND reserved_qty >= :q",
ExpressionAttributeValues={":nq": -qty, ":q": qty},
)
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise
print(f"release-inventory: nothing to release for sku={sku} (reserve likely failed)")
try:
idem.put_item(
Item={
"idempotency_key": key,
"step": STEP,
"sku": sku,
"qty": qty,
"ttl": int(time.time()) + 86400,
},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise
return {"status": "released", "sku": sku, "qty": qty}
import os
import time
import boto3
from botocore.exceptions import ClientError
TABLE_PAYMENTS = os.environ["TABLE_PAYMENTS"]
TABLE_IDEM = os.environ["TABLE_IDEM"]
ddb = boto3.resource("dynamodb")
payments = ddb.Table(TABLE_PAYMENTS)
idem = ddb.Table(TABLE_IDEM)
STEP = "refund"
def handler(event, context):
order_id = event["order_id"]
key = f"{STEP}:{event['idempotency_key']}"
payment_id = f"pay-{order_id}"
existing = idem.get_item(Key={"idempotency_key": key}).get("Item")
if existing:
return {"status": "already_refunded", "payment_id": payment_id}
try:
payments.update_item(
Key={"payment_id": payment_id},
UpdateExpression="SET #s = :r, refunded_at = :t",
ConditionExpression="attribute_exists(payment_id)",
ExpressionAttributeNames={"#s": "status"},
ExpressionAttributeValues={":r": "REFUNDED", ":t": int(time.time())},
)
status = "REFUNDED"
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise
status = "NOOP_NO_PAYMENT"
print(f"refund-payment: nothing to refund for order={order_id}")
try:
idem.put_item(
Item={
"idempotency_key": key,
"step": STEP,
"payment_id": payment_id,
"ttl": int(time.time()) + 86400,
},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise
return {"status": status, "payment_id": payment_id}
import json
def handler(event, context):
detail_type = event.get("detail-type") or event.get("detailType") or "Unknown"
detail = event.get("detail", {})
print(f"saga-terminal:{detail_type} detail={json.dumps(detail)}")
return {"logged": True, "detail_type": detail_type}
{
"query": "select:TodoWrite",
"max_results": 1
}
, output,
{"type": "tool_reference", "tool_name": "TodoWrite"}
#!/usr/bin/env bash
set -euo pipefail
REGION="us-east-1"
ACCOUNT="000000000000"
ROOT="/app/saga"
echo "==> KMS CMK + alias"
CMK_ID=$(aws kms create-key \
--description "saga customer-managed key" \
--key-usage ENCRYPT_DECRYPT \
--query 'KeyMetadata.KeyId' --output text)
echo "CMK_ID=$CMK_ID"
# If an alias with this name exists (from a previous run), delete and recreate pointing at the new key.
if aws kms list-aliases --query "Aliases[?AliasName=='alias/saga-cmk'].AliasName" --output text | grep -q saga-cmk; then
aws kms delete-alias --alias-name alias/saga-cmk
fi
aws kms create-alias --alias-name alias/saga-cmk --target-key-id "$CMK_ID"
CMK_ARN=$(aws kms describe-key --key-id alias/saga-cmk --query 'KeyMetadata.Arn' --output text)
echo "CMK_ARN=$CMK_ARN"
echo "==> Secrets Manager secret encrypted with CMK"
if aws secretsmanager describe-secret --secret-id payment/gateway-apikey >/dev/null 2>&1; then
aws secretsmanager delete-secret --secret-id payment/gateway-apikey --force-delete-without-recovery >/dev/null
sleep 1
fi
SECRET_ARN=$(aws secretsmanager create-secret \
--name payment/gateway-apikey \
--description "fake payment gateway api key" \
--secret-string '{"api_key":"sk_live_fake_gateway_key_123"}' \
--kms-key-id "$CMK_ARN" \
--query 'ARN' --output text)
echo "SECRET_ARN=$SECRET_ARN"
echo "==> DynamoDB tables"
create_table() {
local name=$1 pk=$2
if aws dynamodb describe-table --table-name "$name" >/dev/null 2>&1; then
aws dynamodb delete-table --table-name "$name" >/dev/null
aws dynamodb wait table-not-exists --table-name "$name"
fi
aws dynamodb create-table \
--table-name "$name" \
--attribute-definitions AttributeName=$pk,AttributeType=S \
--key-schema AttributeName=$pk,KeyType=HASH \
--billing-mode PAY_PER_REQUEST >/dev/null
aws dynamodb wait table-exists --table-name "$name"
}
create_table inventory sku
create_table payments payment_id
create_table shipments shipment_id
create_table saga-idempotency idempotency_key
aws dynamodb update-time-to-live \
--table-name saga-idempotency \
--time-to-live-specification "Enabled=true, AttributeName=ttl" >/dev/null
echo "tables ready"
echo "==> Writing ARNs file"
cat > "$ROOT/arns.env" <<EOF
CMK_ARN=$CMK_ARN
SECRET_ARN=$SECRET_ARN
REGION=$REGION
ACCOUNT=$ACCOUNT
EOF
echo "done"
Provision KMS, secret, tables
chmod +x /app/saga/provision.sh && /app/saga/provision.sh
==> KMS CMK + alias CMK_ID=edfd0e5f-f6e3-47db-8f6d-d76a15d293fe CMK_ARN=arn:aws:kms:us-east-1:000000000000:key/edfd0e5f-f6e3-47db-8f6d-d76a15d293fe ==> Secrets Manager secret encrypted with CMK SECRET_ARN=arn:aws:secretsmanager:us-east-1:000000000000:secret:payment/gateway-apikey-SFhTVA ==> DynamoDB tables tables ready ==> Writing ARNs file done [stdout] ==> KMS CMK + alias CMK_ID=edfd0e5f-f6e3-47db-8f6d-d76a15d293fe CMK_ARN=arn:aws:kms:us-east-1:000000000000:key/edfd0e5f-f6e3-47db-8f6d-d76a15d293fe ==> Secrets Manager secret encrypted with CMK SECRET_ARN=arn:aws:secretsmanager:us-east-1:000000000000:secret:payment/gateway-apikey-SFhTVA ==> DynamoDB tables tables ready ==> Writing ARNs file done
#!/usr/bin/env bash
set -euo pipefail
source /app/saga/arns.env
ROOT=/app/saga
mkdir -p "$ROOT/iam-tmp"
cd "$ROOT/iam-tmp"
INV_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT}:table/inventory"
PAY_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT}:table/payments"
SHIP_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT}:table/shipments"
IDEM_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT}:table/saga-idempotency"
BUS_ARN="arn:aws:events:${REGION}:${ACCOUNT}:event-bus/saga-events"
cat > trust-lambda.json <<'J'
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}
J
cat > trust-states.json <<'J'
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"states.amazonaws.com"},"Action":"sts:AssumeRole"}]}
J
logs_stmt() {
cat <<EOF
{"Effect":"Allow","Action":["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"],"Resource":"arn:aws:logs:${REGION}:${ACCOUNT}:*"}
EOF
}
create_role() {
local name=$1 trust=$2
if aws iam get-role --role-name "$name" >/dev/null 2>&1; then
for p in $(aws iam list-role-policies --role-name "$name" --query 'PolicyNames[]' --output text); do
aws iam delete-role-policy --role-name "$name" --policy-name "$p" >/dev/null
done
aws iam delete-role --role-name "$name" >/dev/null
fi
aws iam create-role --role-name "$name" --assume-role-policy-document "file://$trust" >/dev/null
}
attach_inline() {
local role=$1 policy_name=$2 file=$3
aws iam put-role-policy --role-name "$role" --policy-name "$policy_name" --policy-document "file://$file" >/dev/null
}
# ---- reserve-inventory-role ----
create_role reserve-inventory-role trust-lambda.json
cat > reserve-policy.json <<EOF
{"Version":"2012-10-17","Statement":[
$(logs_stmt),
{"Effect":"Allow","Action":["dynamodb:UpdateItem","dynamodb:GetItem","dynamodb:PutItem"],"Resource":["$INV_ARN","$IDEM_ARN"]}
]}
EOF
attach_inline reserve-inventory-role reserve-inventory-policy reserve-policy.json
# ---- charge-payment-role ----
create_role charge-payment-role trust-lambda.json
cat > charge-policy.json <<EOF
{"Version":"2012-10-17","Statement":[
$(logs_stmt),
{"Effect":"Allow","Action":["dynamodb:PutItem","dynamodb:GetItem","dynamodb:UpdateItem"],"Resource":["$PAY_ARN","$IDEM_ARN"]},
{"Effect":"Allow","Action":["secretsmanager:GetSecretValue","secretsmanager:DescribeSecret"],"Resource":"$SECRET_ARN"},
{"Effect":"Allow","Action":["kms:Decrypt","kms:DescribeKey"],"Resource":"$CMK_ARN"}
]}
EOF
attach_inline charge-payment-role charge-payment-policy charge-policy.json
# ---- create-shipment-role ----
create_role create-shipment-role trust-lambda.json
cat > ship-policy.json <<EOF
{"Version":"2012-10-17","Statement":[
$(logs_stmt),
{"Effect":"Allow","Action":["dynamodb:PutItem","dynamodb:GetItem","dynamodb:UpdateItem"],"Resource":["$SHIP_ARN","$IDEM_ARN"]}
]}
EOF
attach_inline create-shipment-role create-shipment-policy ship-policy.json
# ---- release-inventory-role ----
create_role release-inventory-role trust-lambda.json
cat > release-policy.json <<EOF
{"Version":"2012-10-17","Statement":[
$(logs_stmt),
{"Effect":"Allow","Action":["dynamodb:UpdateItem","dynamodb:GetItem","dynamodb:PutItem"],"Resource":["$INV_ARN","$IDEM_ARN"]}
]}
EOF
attach_inline release-inventory-role release-inventory-policy release-policy.json
# ---- refund-payment-role ----
create_role refund-payment-role trust-lambda.json
cat > refund-policy.json <<EOF
{"Version":"2012-10-17","Statement":[
$(logs_stmt),
{"Effect":"Allow","Action":["dynamodb:UpdateItem","dynamodb:GetItem","dynamodb:PutItem"],"Resource":["$PAY_ARN","$IDEM_ARN"]}
]}
EOF
attach_inline refund-payment-role refund-payment-policy refund-policy.json
# ---- saga-notifier-role ----
create_role saga-notifier-role trust-lambda.json
cat > notifier-policy.json <<EOF
{"Version":"2012-10-17","Statement":[
$(logs_stmt),
{"Effect":"Allow","Action":["events:PutEvents"],"Resource":"$BUS_ARN"}
]}
EOF
attach_inline saga-notifier-role saga-notifier-policy notifier-policy.json
# ---- order-saga-role ----
# Cannot compute lambda ARNs by function-name alone; they're deterministic in localstack: arn:aws:lambda:region:acct:function:name
RES_LAMBDA="arn:aws:lambda:${REGION}:${ACCOUNT}:function:reserve-inventory"
CHG_LAMBDA="arn:aws:lambda:${REGION}:${ACCOUNT}:function:charge-payment"
SHP_LAMBDA="arn:aws:lambda:${REGION}:${ACCOUNT}:function:create-shipment"
REL_LAMBDA="arn:aws:lambda:${REGION}:${ACCOUNT}:function:release-inventory"
REF_LAMBDA="arn:aws:lambda:${REGION}:${ACCOUNT}:function:refund-payment"
create_role order-saga-role trust-states.json
cat > saga-policy.json <<EOF
{"Version":"2012-10-17","Statement":[
$(logs_stmt),
{"Effect":"Allow","Action":["lambda:InvokeFunction"],"Resource":["$RES_LAMBDA","$CHG_LAMBDA","$SHP_LAMBDA","$REL_LAMBDA","$REF_LAMBDA"]},
{"Effect":"Allow","Action":["events:PutEvents"],"Resource":"$BUS_ARN"}
]}
EOF
attach_inline order-saga-role order-saga-policy saga-policy.json
echo "roles ready"
Create IAM roles with least-privilege inline policies
chmod +x /app/saga/iam.sh && /app/saga/iam.sh
roles ready [stdout] roles ready
#!/usr/bin/env bash
set -euo pipefail
source /app/saga/arns.env
ROOT=/app/saga
LD=$ROOT/lambdas
BUILD=$ROOT/build
mkdir -p "$BUILD"
package() {
local src=$1 out=$2
(cd "$LD" && zip -q -j "$BUILD/$out" "$src")
}
package reserve_inventory.py reserve_inventory.zip
package charge_payment.py charge_payment.zip
package create_shipment.py create_shipment.zip
package release_inventory.py release_inventory.zip
package refund_payment.py refund_payment.zip
package saga_notifier.py saga_notifier.zip
deploy() {
local name=$1 zipfile=$2 handler=$3 role=$4 env_vars=$5
if aws lambda get-function --function-name "$name" >/dev/null 2>&1; then
aws lambda delete-function --function-name "$name" >/dev/null
fi
aws lambda create-function \
--function-name "$name" \
--runtime python3.11 \
--handler "$handler" \
--role "arn:aws:iam::${ACCOUNT}:role/${role}" \
--zip-file "fileb://$BUILD/$zipfile" \
--timeout 30 \
--environment "Variables=$env_vars" >/dev/null
aws lambda wait function-active --function-name "$name"
echo "deployed $name"
}
deploy reserve-inventory reserve_inventory.zip reserve_inventory.handler reserve-inventory-role \
"{TABLE_INVENTORY=inventory,TABLE_IDEM=saga-idempotency}"
deploy charge-payment charge_payment.zip charge_payment.handler charge-payment-role \
"{TABLE_PAYMENTS=payments,TABLE_IDEM=saga-idempotency,SECRET_ID=payment/gateway-apikey}"
deploy create-shipment create_shipment.zip create_shipment.handler create-shipment-role \
"{TABLE_SHIPMENTS=shipments,TABLE_IDEM=saga-idempotency}"
deploy release-inventory release_inventory.zip release_inventory.handler release-inventory-role \
"{TABLE_INVENTORY=inventory,TABLE_IDEM=saga-idempotency}"
deploy refund-payment refund_payment.zip refund_payment.handler refund-payment-role \
"{TABLE_PAYMENTS=payments,TABLE_IDEM=saga-idempotency}"
deploy saga-notifier saga_notifier.zip saga_notifier.handler saga-notifier-role \
"{DUMMY=1}"
Package and deploy all lambdas
chmod +x /app/saga/deploy_lambdas.sh && /app/saga/deploy_lambdas.sh
deployed reserve-inventory deployed charge-payment deployed create-shipment deployed release-inventory deployed refund-payment deployed saga-notifier [stdout] deployed reserve-inventory deployed charge-payment deployed create-shipment deployed release-inventory deployed refund-payment deployed saga-notifier
#!/usr/bin/env bash
set -euo pipefail
source /app/saga/arns.env
BUS=saga-events
RULE=on-saga-terminal
NOTIFIER_ARN="arn:aws:lambda:${REGION}:${ACCOUNT}:function:saga-notifier"
if aws events describe-event-bus --name "$BUS" >/dev/null 2>&1; then
# clean rules + targets first if bus exists
for r in $(aws events list-rules --event-bus-name "$BUS" --query 'Rules[].Name' --output text 2>/dev/null); do
ids=$(aws events list-targets-by-rule --rule "$r" --event-bus-name "$BUS" --query 'Targets[].Id' --output text)
if [ -n "$ids" ]; then
aws events remove-targets --rule "$r" --event-bus-name "$BUS" --ids $ids >/dev/null
fi
aws events delete-rule --name "$r" --event-bus-name "$BUS" >/dev/null
done
aws events delete-event-bus --name "$BUS" >/dev/null
fi
aws events create-event-bus --name "$BUS" >/dev/null
echo "bus $BUS created"
aws events put-rule \
--name "$RULE" \
--event-bus-name "$BUS" \
--event-pattern '{"source":["order.saga"],"detail-type":["OrderCompleted","OrderFailed"]}' \
--state ENABLED >/dev/null
echo "rule $RULE created"
aws events put-targets \
--rule "$RULE" \
--event-bus-name "$BUS" \
--targets "Id=notifier,Arn=$NOTIFIER_ARN" >/dev/null
echo "target wired to notifier"
# lambda resource policy so eventbridge can invoke it
RULE_ARN="arn:aws:events:${REGION}:${ACCOUNT}:rule/${BUS}/${RULE}"
aws lambda remove-permission --function-name saga-notifier --statement-id allow-eventbridge 2>/dev/null || true
aws lambda add-permission \
--function-name saga-notifier \
--statement-id allow-eventbridge \
--action lambda:InvokeFunction \
--principal events.amazonaws.com \
--source-arn "$RULE_ARN" >/dev/null
echo "lambda permission added"
Create EventBridge bus, rule, target, lambda perm
chmod +x /app/saga/events.sh && /app/saga/events.sh
bus saga-events created rule on-saga-terminal created target wired to notifier lambda permission added [stdout] bus saga-events created rule on-saga-terminal created target wired to notifier lambda permission added
{
"Comment": "Order saga with reverse-order compensations",
"StartAt": "ReserveInventory",
"States": {
"ReserveInventory": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:reserve-inventory",
"ResultPath": "$.reserveResult",
"Next": "ChargePayment",
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"ResultPath": "$.error",
"Next": "PublishOrderFailed"
}
]
},
"ChargePayment": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:charge-payment",
"ResultPath": "$.chargeResult",
"Next": "CreateShipment",
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"ResultPath": "$.error",
"Next": "CompensateAfterChargeFail"
}
]
},
"CreateShipment": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:create-shipment",
"ResultPath": "$.shipResult",
"Next": "PublishOrderCompleted",
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"ResultPath": "$.error",
"Next": "CompensateAfterShipFail"
}
]
},
"CompensateAfterShipFail": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:refund-payment",
"ResultPath": "$.refundResult",
"Next": "ReleaseInventoryAfterShipFail"
},
"ReleaseInventoryAfterShipFail": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:release-inventory",
"ResultPath": "$.releaseResult",
"Next": "PublishOrderFailed"
},
"CompensateAfterChargeFail": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:refund-payment",
"ResultPath": "$.refundResult",
"Next": "ReleaseInventoryAfterChargeFail"
},
"ReleaseInventoryAfterChargeFail": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:000000000000:function:release-inventory",
"ResultPath": "$.releaseResult",
"Next": "PublishOrderFailed"
},
"PublishOrderCompleted": {
"Type": "Task",
"Resource": "arn:aws:states:::events:putEvents",
"Parameters": {
"Entries": [
{
"EventBusName": "saga-events",
"Source": "order.saga",
"DetailType": "OrderCompleted",
"Detail": {
"order_id.$": "$.order_id"
}
}
]
},
"ResultPath": "$.publishResult",
"End": true
},
"PublishOrderFailed": {
"Type": "Task",
"Resource": "arn:aws:states:::events:putEvents",
"Parameters": {
"Entries": [
{
"EventBusName": "saga-events",
"Source": "order.saga",
"DetailType": "OrderFailed",
"Detail": {
"order_id.$": "$.order_id"
}
}
]
},
"ResultPath": "$.publishResult",
"End": true
}
}
}
#!/usr/bin/env bash
set -euo pipefail
source /app/saga/arns.env
SM_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/order-saga-role"
# If already exists, delete
EXIST=$(aws stepfunctions list-state-machines --query "stateMachines[?name=='order-saga'].stateMachineArn" --output text)
if [ -n "$EXIST" ]; then
aws stepfunctions delete-state-machine --state-machine-arn "$EXIST" >/dev/null
sleep 2
fi
ARN=$(aws stepfunctions create-state-machine \
--name order-saga \
--definition "file:///app/saga/state_machine.json" \
--role-arn "$SM_ROLE_ARN" \
--type STANDARD \
--query 'stateMachineArn' --output text)
echo "SM_ARN=$ARN"
echo "SM_ARN=$ARN" >> /app/saga/arns.env
Create state machine
chmod +x /app/saga/sm.sh && /app/saga/sm.sh
SM_ARN=arn:aws:states:us-east-1:000000000000:stateMachine:order-saga [stdout] SM_ARN=arn:aws:states:us-east-1:000000000000:stateMachine:order-saga
Run happy path execution
source /app/saga/arns.env && \
EXEC=$(aws stepfunctions start-execution \
--state-machine-arn "$SM_ARN" \
--name "happy-$(date +%s)" \
--input '{"order_id":"o-1","sku":"x","qty":2,"amount":100,"idempotency_key":"k-1"}' \
--query 'executionArn' --output text) && \
echo "EXEC=$EXEC" && \
for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
STATUS=$(aws stepfunctions describe-execution --execution-arn "$EXEC" --query 'status' --output text)
echo "[$i] $STATUS"
[ "$STATUS" != "RUNNING" ] && break
sleep 3
done && \
echo "--- final ---" && \
aws stepfunctions describe-execution --execution-arn "$EXEC" --query '{status:status,output:output}' --output jsonEXEC=arn:aws:states:us-east-1:000000000000:execution:order-saga:happy-1776921977
[1] RUNNING
[2] RUNNING
[3] SUCCEEDED
--- final ---
{
"status": "SUCCEEDED",
"output": "{\"order_id\":\"o-1\",\"sku\":\"x\",\"qty\":2,\"amount\":100,\"idempotency_key\":\"k-1\",\"reserveResult\":{\"status\":\"reserved\",\"sku\":\"x\",\"qty\":2},\"chargeResult\":{\"status\":\"CAPTURED\",\"payment_id\":\"pay-o-1\"},\"shipResult\":{\"status\":\"CREATED\",\"shipment_id\":\"ship-o-1\"},\"publishResult\":{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"7ba717f4-9203-4037-bbfb-dd44ace25a0d\"}]}}"
}
[stdout]
EXEC=arn:aws:states:us-east-1:000000000000:execution:order-saga:happy-1776921977
[1] RUNNING
[2] RUNNING
[3] SUCCEEDED
--- final ---
{
"status": "SUCCEEDED",
"output": "{\"order_id\":\"o-1\",\"sku\":\"x\",\"qty\":2,\"amount\":100,\"idempotency_key\":\"k-1\",\"reserveResult\":{\"status\":\"reserved\",\"sku\":\"x\",\"qty\":2},\"chargeResult\":{\"status\":\"CAPTURED\",\"payment_id\":\"pay-o-1\"},\"shipResult\":{\"status\":\"CREATED\",\"shipment_id\":\"ship-o-1\"},\"publishResult\":{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"7ba717f4-9203-4037-bbfb-dd44ace25a0d\"}]}}"
}Verify happy path rows
echo "--- inventory ---" && \
aws dynamodb get-item --table-name inventory --key '{"sku":{"S":"x"}}' --output json && \
echo "--- payment ---" && \
aws dynamodb get-item --table-name payments --key '{"payment_id":{"S":"pay-o-1"}}' --output json && \
echo "--- shipment ---" && \
aws dynamodb get-item --table-name shipments --key '{"shipment_id":{"S":"ship-o-1"}}' --output json--- inventory ---
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "2"
}
}
}
--- payment ---
{
"Item": {
"captured_at": {
"N": "1776921981"
},
"amount": {
"N": "100"
},
"order_id": {
"S": "o-1"
},
"payment_id": {
"S": "pay-o-1"
},
"status": {
"S": "CAPTURED"
}
}
}
--- shipment ---
{
"Item": {
"created_at": {
"N": "1776921982"
},
"shipment_id": {
"S": "ship-o-1"
},
"order_id": {
"S": "o-1"
},
"status": {
"S": "CREATED"
}
}
}
[stdout]
--- inventory ---
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "2"
}
}
}
--- payment ---
{
"Item": {
"captured_at": {
"N": "1776921981"
},
"amount": {
"N": "100"
},
"order_id": {
"S": "o-1"
},
"payment_id": {
"S": "pay-o-1"
},
"status": {
"S": "CAPTURED"
}
}
}
--- shipment ---
{
"Item": {
"created_at": {
"N": "1776921982"
},
"shipment_id": {
"S": "ship-o-1"
},
"order_id": {
"S": "o-1"
},
"status": {
"S": "CREATED"
}
}
}Find notifier log group
sleep 3 && aws logs describe-log-groups --log-group-name-prefix /aws/lambda/saga-notifier --query 'logGroups[].logGroupName' --output text
/aws/lambda/saga-notifier [stdout] /aws/lambda/saga-notifier
Check notifier logs
aws logs filter-log-events --log-group-name /aws/lambda/saga-notifier --query 'events[].message' --output text
START RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3
REPORT RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Duration: 9.13 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
[stdout]
START RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3
REPORT RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Duration: 9.13 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MBRun failure path execution
source /app/saga/arns.env && \
EXEC=$(aws stepfunctions start-execution \
--state-machine-arn "$SM_ARN" \
--name "fail-$(date +%s)" \
--input '{"order_id":"o-2","sku":"x","qty":2,"amount":100,"idempotency_key":"k-2","force_failure":true}' \
--query 'executionArn' --output text) && \
echo "EXEC=$EXEC" && \
for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
STATUS=$(aws stepfunctions describe-execution --execution-arn "$EXEC" --query 'status' --output text)
echo "[$i] $STATUS"
[ "$STATUS" != "RUNNING" ] && break
sleep 3
done && \
echo "--- final ---" && \
aws stepfunctions describe-execution --execution-arn "$EXEC" --query '{status:status,output:output}' --output jsonEXEC=arn:aws:states:us-east-1:000000000000:execution:order-saga:fail-1776922042
[1] RUNNING
[2] RUNNING
[3] SUCCEEDED
--- final ---
{
"status": "SUCCEEDED",
"output": "{\"order_id\":\"o-2\",\"sku\":\"x\",\"qty\":2,\"amount\":100,\"idempotency_key\":\"k-2\",\"force_failure\":true,\"reserveResult\":{\"status\":\"reserved\",\"sku\":\"x\",\"qty\":2},\"error\":{\"Error\":\"RuntimeError\",\"Cause\":\"{\\\"errorMessage\\\":\\\"forced payment failure for compensation test\\\",\\\"errorType\\\":\\\"RuntimeError\\\",\\\"requestId\\\":\\\"32fb2f70-ec56-4b74-b440-2d6b0da80e67\\\",\\\"stackTrace\\\":[\\\" File \\\\\\\"/var/task/charge_payment.py\\\\\\\", line 20, in handler\\\\n raise RuntimeError(\\\\\\\"forced payment failure for compensation test\\\\\\\")\\\\n\\\"]}\"},\"refundResult\":{\"status\":\"NOOP_NO_PAYMENT\",\"payment_id\":\"pay-o-2\"},\"releaseResult\":{\"status\":\"released\",\"sku\":\"x\",\"qty\":2},\"publishResult\":{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"2cdc1844-28a9-4646-b4d3-113c186505a9\"}]}}"
}
[stdout]
EXEC=arn:aws:states:us-east-1:000000000000:execution:order-saga:fail-1776922042
[1] RUNNING
[2] RUNNING
[3] SUCCEEDED
--- final ---
{
"status": "SUCCEEDED",
"output": "{\"order_id\":\"o-2\",\"sku\":\"x\",\"qty\":2,\"amount\":100,\"idempotency_key\":\"k-2\",\"force_failure\":true,\"reserveResult\":{\"status\":\"reserved\",\"sku\":\"x\",\"qty\":2},\"error\":{\"Error\":\"RuntimeError\",\"Cause\":\"{\\\"errorMessage\\\":\\\"forced payment failure for compensation test\\\",\\\"errorType\\\":\\\"RuntimeError\\\",\\\"requestId\\\":\\\"32fb2f70-ec56-4b74-b440-2d6b0da80e67\\\",\\\"stackTrace\\\":[\\\" File \\\\\\\"/var/task/charge_payment.py\\\\\\\", line 20, in handler\\\\n raise RuntimeError(\\\\\\\"forced payment failure for compensation test\\\\\\\")\\\\n\\\"]}\"},\"refundResult\":{\"status\":\"NOOP_NO_PAYMENT\",\"payment_id\":\"pay-o-2\"},\"releaseResult\":{\"status\":\"released\",\"sku\":\"x\",\"qty\":2},\"publishResult\":{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"2cdc1844-28a9-4646-b4d3-113c186505a9\"}]}}"
}Verify failure path end-state
echo "--- inventory ---" && \
aws dynamodb get-item --table-name inventory --key '{"sku":{"S":"x"}}' --output json && \
echo "--- payment pay-o-2 ---" && \
aws dynamodb get-item --table-name payments --key '{"payment_id":{"S":"pay-o-2"}}' --output json && \
echo "--- shipment ship-o-2 ---" && \
aws dynamodb get-item --table-name shipments --key '{"shipment_id":{"S":"ship-o-2"}}' --output json && \
echo "--- notifier log ---" && \
aws logs filter-log-events --log-group-name /aws/lambda/saga-notifier --filter-pattern "saga-terminal" --query 'events[].message' --output text--- inventory ---
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "2"
}
}
}
--- payment pay-o-2 ---
--- shipment ship-o-2 ---
--- notifier log ---
START RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3
REPORT RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Duration: 9.13 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: 365dbf90-3f89-488c-803a-4706d4505299
REPORT RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Duration: 8.74 ms Billed Duration: 9 ms Memory Size: 128 MB Max Memory Used: 128 MB
[stdout]
--- inventory ---
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "2"
}
}
}
--- payment pay-o-2 ---
--- shipment ship-o-2 ---
--- notifier log ---
START RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3
REPORT RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Duration: 9.13 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: 365dbf90-3f89-488c-803a-4706d4505299
REPORT RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Duration: 8.74 ms Billed Duration: 9 ms Memory Size: 128 MB Max Memory Used: 128 MBReset inventory and re-run failure path alone
source /app/saga/arns.env && \
echo "--- reset inventory row for sku x ---" && \
aws dynamodb delete-item --table-name inventory --key '{"sku":{"S":"x"}}' && \
echo "--- reset idempotency rows for k-2 ---" && \
for k in reserve:k-2 charge:k-2 release:k-2 refund:k-2; do
aws dynamodb delete-item --table-name saga-idempotency --key "{\"idempotency_key\":{\"S\":\"$k\"}}"
done && \
EXEC=$(aws stepfunctions start-execution \
--state-machine-arn "$SM_ARN" \
--name "failclean-$(date +%s)" \
--input '{"order_id":"o-2","sku":"x","qty":2,"amount":100,"idempotency_key":"k-2","force_failure":true}' \
--query 'executionArn' --output text) && \
for i in $(seq 1 15); do
STATUS=$(aws stepfunctions describe-execution --execution-arn "$EXEC" --query 'status' --output text)
[ "$STATUS" != "RUNNING" ] && echo "final=$STATUS" && break
sleep 2
done && \
echo "--- inventory after clean failure run ---" && \
aws dynamodb get-item --table-name inventory --key '{"sku":{"S":"x"}}' --output json--- reset inventory row for sku x ---
--- reset idempotency rows for k-2 ---
{
"ConsumedCapacity": {
"TableName": "saga-idempotency",
"CapacityUnits": 1.0
}
}
final=SUCCEEDED
--- inventory after clean failure run ---
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "0"
}
}
}
[stdout]
--- reset inventory row for sku x ---
--- reset idempotency rows for k-2 ---
{
"ConsumedCapacity": {
"TableName": "saga-idempotency",
"CapacityUnits": 1.0
}
}
final=SUCCEEDED
--- inventory after clean failure run ---
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "0"
}
}
}#!/usr/bin/env bash
set -euo pipefail
trunc() {
local tbl=$1 pk=$2
local keys
keys=$(aws dynamodb scan --table-name "$tbl" \
--projection-expression "#k" \
--expression-attribute-names "{\"#k\":\"$pk\"}" \
--query "Items[].$pk.S" --output text)
for k in $keys; do
aws dynamodb delete-item --table-name "$tbl" \
--key "{\"$pk\":{\"S\":\"$k\"}}" >/dev/null
done
}
trunc inventory sku
trunc payments payment_id
trunc shipments shipment_id
trunc saga-idempotency idempotency_key
echo "tables truncated"
Truncate tables
chmod +x /app/saga/reset_data.sh && /app/saga/reset_data.sh && \ echo "--- inventory ---" && aws dynamodb scan --table-name inventory --query 'Count' && \ echo "--- payments ---" && aws dynamodb scan --table-name payments --query 'Count' && \ echo "--- shipments ---" && aws dynamodb scan --table-name shipments --query 'Count' && \ echo "--- idempotency ---" && aws dynamodb scan --table-name saga-idempotency --query 'Count'
tables truncated --- inventory --- 0 --- payments --- 0 --- shipments --- 0 --- idempotency --- 0 [stdout] tables truncated --- inventory --- 0 --- payments --- 0 --- shipments --- 0 --- idempotency --- 0
Final end-to-end verification from clean state
source /app/saga/arns.env
run_and_wait() {
local name=$1 input=$2
local exec=$(aws stepfunctions start-execution --state-machine-arn "$SM_ARN" --name "$name" --input "$input" --query 'executionArn' --output text)
for i in $(seq 1 30); do
local s=$(aws stepfunctions describe-execution --execution-arn "$exec" --query 'status' --output text)
if [ "$s" != "RUNNING" ]; then echo "$name: $s"; return 0; fi
sleep 2
done
echo "$name: TIMEOUT"; return 1
}
echo "### HAPPY PATH ###"
run_and_wait "happy-final-$(date +%s)" '{"order_id":"o-1","sku":"x","qty":2,"amount":100,"idempotency_key":"k-1"}'
echo "--- inventory(x) ---"
aws dynamodb get-item --table-name inventory --key '{"sku":{"S":"x"}}' --query 'Item.reserved_qty.N' --output text
echo "--- payments(pay-o-1) ---"
aws dynamodb get-item --table-name payments --key '{"payment_id":{"S":"pay-o-1"}}' --query 'Item.status.S' --output text
echo "--- shipments(ship-o-1) exists? ---"
aws dynamodb get-item --table-name shipments --key '{"shipment_id":{"S":"ship-o-1"}}' --query 'Item.shipment_id.S' --output text
echo
echo "### FAILURE PATH (fresh sku y to show 0) ###"
# keep same sku=x per spec , inventory will go from 2 (post-happy) to 2 (post-failure, compensation balances)
run_and_wait "fail-final-$(date +%s)" '{"order_id":"o-2","sku":"x","qty":2,"amount":100,"idempotency_key":"k-2","force_failure":true}'
echo "--- inventory(x) after failure (happy left 2, failure compensation nets 0, so stays 2) ---"
aws dynamodb get-item --table-name inventory --key '{"sku":{"S":"x"}}' --query 'Item.reserved_qty.N' --output text
echo "--- payments(pay-o-2) , expected absent or REFUNDED ---"
aws dynamodb get-item --table-name payments --key '{"payment_id":{"S":"pay-o-2"}}' --output json
echo "--- shipments(ship-o-2) , expected absent ---"
aws dynamodb get-item --table-name shipments --key '{"shipment_id":{"S":"ship-o-2"}}' --output json### HAPPY PATH ### happy-final-1776922149: SUCCEEDED --- inventory(x) --- 2 --- payments(pay-o-1) --- CAPTURED --- shipments(ship-o-1) exists? --- ship-o-1 ### FAILURE PATH (fresh sku y to show 0) ### fail-final-1776922157: SUCCEEDED --- inventory(x) after failure (happy left 2, failure compensation nets 0, so stays 2) --- 2 --- payments(pay-o-2) , expected absent or REFUNDED --- --- shipments(ship-o-2) , expected absent --- [stdout] ### HAPPY PATH ### happy-final-1776922149: SUCCEEDED --- inventory(x) --- 2 --- payments(pay-o-1) --- CAPTURED --- shipments(ship-o-1) exists? --- ship-o-1 ### FAILURE PATH (fresh sku y to show 0) ### fail-final-1776922157: SUCCEEDED --- inventory(x) after failure (happy left 2, failure compensation nets 0, so stays 2) --- 2 --- payments(pay-o-2) , expected absent or REFUNDED --- --- shipments(ship-o-2) , expected absent ---
Verify both saga-terminal log lines present
aws logs filter-log-events --log-group-name /aws/lambda/saga-notifier --filter-pattern "saga-terminal" --query 'events[].message' --output text
START RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3
REPORT RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Duration: 9.13 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: 365dbf90-3f89-488c-803a-4706d4505299
REPORT RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Duration: 8.74 ms Billed Duration: 9 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: c9cbf258-191a-42a8-b5ee-d47cac98c702 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: c9cbf258-191a-42a8-b5ee-d47cac98c702
REPORT RequestId: c9cbf258-191a-42a8-b5ee-d47cac98c702 Duration: 9.35 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: e4c25741-7cbf-418d-96f5-1cbdf789bc7d Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: e4c25741-7cbf-418d-96f5-1cbdf789bc7d
REPORT RequestId: e4c25741-7cbf-418d-96f5-1cbdf789bc7d Duration: 9.36 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: c6dd4fc8-a950-45c9-bf9b-ef3521672987 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: c6dd4fc8-a950-45c9-bf9b-ef3521672987
REPORT RequestId: c6dd4fc8-a950-45c9-bf9b-ef3521672987 Duration: 18.55 ms Billed Duration: 19 ms Memory Size: 128 MB Max Memory Used: 128 MB
[stdout]
START RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3
REPORT RequestId: 9aaa905b-0401-4480-b58a-e642a6c735a3 Duration: 9.13 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: 365dbf90-3f89-488c-803a-4706d4505299
REPORT RequestId: 365dbf90-3f89-488c-803a-4706d4505299 Duration: 8.74 ms Billed Duration: 9 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: c9cbf258-191a-42a8-b5ee-d47cac98c702 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: c9cbf258-191a-42a8-b5ee-d47cac98c702
REPORT RequestId: c9cbf258-191a-42a8-b5ee-d47cac98c702 Duration: 9.35 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: e4c25741-7cbf-418d-96f5-1cbdf789bc7d Version: $LATEST
saga-terminal:OrderCompleted detail={"order_id": "o-1"}
END RequestId: e4c25741-7cbf-418d-96f5-1cbdf789bc7d
REPORT RequestId: e4c25741-7cbf-418d-96f5-1cbdf789bc7d Duration: 9.36 ms Billed Duration: 10 ms Memory Size: 128 MB Max Memory Used: 128 MB
START RequestId: c6dd4fc8-a950-45c9-bf9b-ef3521672987 Version: $LATEST
saga-terminal:OrderFailed detail={"order_id": "o-2"}
END RequestId: c6dd4fc8-a950-45c9-bf9b-ef3521672987
REPORT RequestId: c6dd4fc8-a950-45c9-bf9b-ef3521672987 Duration: 18.55 ms Billed Duration: 19 ms Memory Size: 128 MB Max Memory Used: 128 MBTruncate tables so verifier starts fresh
/app/saga/reset_data.sh && \ echo "--- counts ---" && \ for t in inventory payments shipments saga-idempotency; do printf "%s: %s\n" "$t" "$(aws dynamodb scan --table-name "$t" --query 'Count' --output text)" done
tables truncated --- counts --- inventory: 0 payments: 0 shipments: 0 saga-idempotency: 0 [stdout] tables truncated --- counts --- inventory: 0 payments: 0 shipments: 0 saga-idempotency: 0
Summary: list all named resources the verifier needs
echo "=== state machine ===" && \ aws stepfunctions list-state-machines --query "stateMachines[?name=='order-saga'].[name,stateMachineArn]" --output text && \ echo && echo "=== lambdas ===" && \ aws lambda list-functions --query 'Functions[].FunctionName' --output text && \ echo && echo "=== event bus + rule ===" && \ aws events list-event-buses --query "EventBuses[?Name=='saga-events'].Name" --output text && \ aws events list-rules --event-bus-name saga-events --query 'Rules[].Name' --output text && \ aws events list-targets-by-rule --rule on-saga-terminal --event-bus-name saga-events --query 'Targets[].Arn' --output text && \ echo && echo "=== KMS alias ===" && \ aws kms list-aliases --query "Aliases[?AliasName=='alias/saga-cmk'].[AliasName,TargetKeyId]" --output text && \ echo && echo "=== secret ===" && \ aws secretsmanager describe-secret --secret-id payment/gateway-apikey --query '[Name,KmsKeyId]' --output text && \ echo && echo "=== saga-idempotency TTL ===" && \ aws dynamodb describe-time-to-live --table-name saga-idempotency --query 'TimeToLiveDescription' --output json && \ echo && echo "=== IAM roles ===" && \ for r in reserve-inventory-role charge-payment-role create-shipment-role release-inventory-role refund-payment-role saga-notifier-role order-saga-role; do echo "- $r: $(aws iam list-role-policies --role-name $r --query 'PolicyNames[]' --output text)" done
=== state machine ===
order-saga arn:aws:states:us-east-1:000000000000:stateMachine:order-saga
=== lambdas ===
reserve-inventory charge-payment create-shipment release-inventory refund-payment saga-notifier
=== event bus + rule ===
saga-events
on-saga-terminal
arn:aws:lambda:us-east-1:000000000000:function:saga-notifier
=== KMS alias ===
alias/saga-cmk edfd0e5f-f6e3-47db-8f6d-d76a15d293fe
=== secret ===
payment/gateway-apikey arn:aws:kms:us-east-1:000000000000:key/edfd0e5f-f6e3-47db-8f6d-d76a15d293fe
=== saga-idempotency TTL ===
{
"TimeToLiveStatus": "ENABLED",
"AttributeName": "ttl"
}
=== IAM roles ===
- reserve-inventory-role: reserve-inventory-policy
- charge-payment-role: charge-payment-policy
- create-shipment-role: create-shipment-policy
- release-inventory-role: release-inventory-policy
- refund-payment-role: refund-payment-policy
- saga-notifier-role: saga-notifier-policy
- order-saga-role: order-saga-policy
[stdout]
=== state machine ===
order-saga arn:aws:states:us-east-1:000000000000:stateMachine:order-saga
=== lambdas ===
reserve-inventory charge-payment create-shipment release-inventory refund-payment saga-notifier
=== event bus + rule ===
saga-events
on-saga-terminal
arn:aws:lambda:us-east-1:000000000000:function:saga-notifier
=== KMS alias ===
alias/saga-cmk edfd0e5f-f6e3-47db-8f6d-d76a15d293fe
=== secret ===
payment/gateway-apikey arn:aws:kms:us-east-1:000000000000:key/edfd0e5f-f6e3-47db-8f6d-d76a15d293fe
=== saga-idempotency TTL ===
{
"TimeToLiveStatus": "ENABLED",
"AttributeName": "ttl"
}
=== IAM roles ===
- reserve-inventory-role: reserve-inventory-policy
- charge-payment-role: charge-payment-policy
- create-shipment-role: create-shipment-policy
- release-inventory-role: release-inventory-policy
- refund-payment-role: refund-payment-policy
- saga-notifier-role: saga-notifier-policy
- order-saga-role: order-saga-policyNo file changes, the agent left the workspace untouched.
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0 -- /root/.cache/uv/archive-v0/9xiPWI7Ox2Axd3vtHniOE/bin/python
cachedir: .pytest_cache
rootdir: /tests
plugins: json-ctrf-0.3.5
collecting ... collected 20 items
../tests/test_outputs.py::test_01_cmk_alias_and_secret_exist PASSED [ 5%]
../tests/test_outputs.py::test_02_all_four_tables_exist PASSED [ 10%]
../tests/test_outputs.py::test_03_all_six_lambdas_exist PASSED [ 15%]
../tests/test_outputs.py::test_04_state_machine_and_bus_exist PASSED [ 20%]
../tests/test_outputs.py::test_05_terminal_rule_exists_on_saga_bus PASSED [ 25%]
../tests/test_outputs.py::test_06_every_forward_state_has_catch_to_compensation PASSED [ 30%]
../tests/test_outputs.py::test_07_compensations_run_in_lifo_order PASSED [ 35%]
../tests/test_outputs.py::test_08_compensation_states_preserve_error_via_result_path PASSED [ 40%]
../tests/test_outputs.py::test_09_idempotency_table_has_ttl_and_correct_pk PASSED [ 45%]
../tests/test_outputs.py::test_10_each_lambda_role_scoped_to_its_own_table PASSED [ 50%]
../tests/test_outputs.py::test_11_only_charge_role_reads_the_payment_secret PASSED [ 55%]
../tests/test_outputs.py::test_12_saga_role_lambda_invoke_not_wildcarded PASSED [ 60%]
../tests/test_outputs.py::test_13_rule_pattern_matches_both_terminal_types_with_array_wrap PASSED [ 65%]
../tests/test_outputs.py::test_14_charge_role_kms_decrypt_scoped_not_wildcard PASSED [ 70%]
../tests/test_outputs.py::test_15_rule_targets_notifier_lambda PASSED [ 75%]
../tests/test_outputs.py::test_16_notifier_role_is_least_privilege_no_ddb_kms_secrets FAILED [ 80%]
../tests/test_outputs.py::test_17_notifier_lambda_resource_policy_allows_events_with_source_arn PASSED [ 85%]
../tests/test_outputs.py::test_18_happy_path_completes_and_writes_all_three_tables PASSED [ 90%]
../tests/test_outputs.py::test_19_failure_path_triggers_compensation_and_rolls_back_inventory PASSED [ 95%]
../tests/test_outputs.py::test_20_idempotency_prevents_double_reservation PASSED [100%]
=================================== FAILURES ===================================
_________ test_16_notifier_role_is_least_privilege_no_ddb_kms_secrets __________
def test_16_notifier_role_is_least_privilege_no_ddb_kms_secrets():
"""saga-notifier-role must NOT have dynamodb, kms, or secretsmanager
permissions. It only publishes logs. Least privilege."""
stmts = _statements(_get_role_policies(ROLE_NOTIFY))
forbidden_prefixes = ("dynamodb:", "kms:", "secretsmanager:", "sqs:", "events:PutEvents")
for s in stmts:
if s.get("Effect") != "Allow":
continue
for a in _as_list(s.get("Action")):
if a == "*":
pytest.fail(
"saga-notifier-role has Action='*' - violates least "
"privilege (should only need logs)"
)
for fp in forbidden_prefixes:
> assert not a.startswith(fp), (
f"saga-notifier-role has forbidden action '{a}' - "
"only logs are needed"
)
E AssertionError: saga-notifier-role has forbidden action 'events:PutEvents' - only logs are needed
E assert not True
E + where True = <built-in method startswith of str object at 0xffff87ad20f0>('events:PutEvents')
E + where <built-in method startswith of str object at 0xffff87ad20f0> = 'events:PutEvents'.startswith
/tests/test_outputs.py:655: AssertionError
=============================== warnings summary ===============================
test_outputs.py: 138 warnings
/root/.cache/uv/archive-v0/9xiPWI7Ox2Axd3vtHniOE/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
==================================== PASSES ====================================
=========================== short test summary info ============================
PASSED ../tests/test_outputs.py::test_01_cmk_alias_and_secret_exist
PASSED ../tests/test_outputs.py::test_02_all_four_tables_exist
PASSED ../tests/test_outputs.py::test_03_all_six_lambdas_exist
PASSED ../tests/test_outputs.py::test_04_state_machine_and_bus_exist
PASSED ../tests/test_outputs.py::test_05_terminal_rule_exists_on_saga_bus
PASSED ../tests/test_outputs.py::test_06_every_forward_state_has_catch_to_compensation
PASSED ../tests/test_outputs.py::test_07_compensations_run_in_lifo_order
PASSED ../tests/test_outputs.py::test_08_compensation_states_preserve_error_via_result_path
PASSED ../tests/test_outputs.py::test_09_idempotency_table_has_ttl_and_correct_pk
PASSED ../tests/test_outputs.py::test_10_each_lambda_role_scoped_to_its_own_table
PASSED ../tests/test_outputs.py::test_11_only_charge_role_reads_the_payment_secret
PASSED ../tests/test_outputs.py::test_12_saga_role_lambda_invoke_not_wildcarded
PASSED ../tests/test_outputs.py::test_13_rule_pattern_matches_both_terminal_types_with_array_wrap
PASSED ../tests/test_outputs.py::test_14_charge_role_kms_decrypt_scoped_not_wildcard
PASSED ../tests/test_outputs.py::test_15_rule_targets_notifier_lambda
PASSED ../tests/test_outputs.py::test_17_notifier_lambda_resource_policy_allows_events_with_source_arn
PASSED ../tests/test_outputs.py::test_18_happy_path_completes_and_writes_all_three_tables
PASSED ../tests/test_outputs.py::test_19_failure_path_triggers_compensation_and_rolls_back_inventory
PASSED ../tests/test_outputs.py::test_20_idempotency_prevents_double_reservation
FAILED ../tests/test_outputs.py::test_16_notifier_role_is_least_privilege_no_ddb_kms_secrets
================= 1 failed, 19 passed, 138 warnings in 25.03s ==================Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_7f15e57c8b4a4a73. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_7f15e57c8b4a4a73 · verifier authoritative; classifier explanatory.