tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.Check working directory and verify LocalStack connection
ls -la /app/ 2>&1 | head -30 && echo "---" && aws --endpoint-url=http://localstack:4566 sts get-caller-identity 2>&1
total 8
drwxr-xr-x 2 root root 4096 Apr 21 20:05 .
drwxr-xr-x 1 root root 4096 Apr 23 05:31 ..
---
{
"UserId": "AKIAIOSFODNN7EXAMPLE",
"Account": "000000000000",
"Arn": "arn:aws:iam::000000000000:root"
}
[stdout]
total 8
drwxr-xr-x 2 root root 4096 Apr 21 20:05 .
drwxr-xr-x 1 root root 4096 Apr 23 05:31 ..
---
{
"UserId": "AKIAIOSFODNN7EXAMPLE",
"Account": "000000000000",
"Arn": "arn:aws:iam::000000000000:root"
}Check LocalStack version and tooling
curl -s http://localstack:4566/_localstack/info | head -50; echo "---"; which aws python3 zip jq; python3 --version
{"version": "4.13.1:77ef41bc3", "edition": "community", "is_license_activated": false, "session_id": "a611e9a7-2cdc-4583-a384-62a165fcfa9e", "machine_id": "dkr_9520ba4a2d3d", "system": "linux", "is_docker": true, "server_time_utc": "2026-04-23T05:33:50", "uptime": 132}---
/usr/local/bin/aws
/opt/venv/bin/python3
/usr/bin/zip
/usr/bin/jq
Python 3.12.3
[stdout]
{"version": "4.13.1:77ef41bc3", "edition": "community", "is_license_activated": false, "session_id": "a611e9a7-2cdc-4583-a384-62a165fcfa9e", "machine_id": "dkr_9520ba4a2d3d", "system": "linux", "is_docker": true, "server_time_utc": "2026-04-23T05:33:50", "uptime": 132}---
/usr/local/bin/aws
/opt/venv/bin/python3
/usr/bin/zip
/usr/bin/jq
Python 3.12.3{
"query": "select:TodoWrite",
"max_results": 1
}
, output,
{"type": "tool_reference", "tool_name": "TodoWrite"}
Create working directory layout
mkdir -p /app/lambdas /app/policies /app/build /app/sm
(Bash completed with no output)
import os
import boto3
from botocore.config import Config
ENDPOINT = os.environ.get("AWS_ENDPOINT_URL", "http://localstack:4566")
_cfg = Config(retries={"max_attempts": 2, "mode": "standard"})
_ddb = boto3.client("dynamodb", endpoint_url=ENDPOINT, config=_cfg)
INV_TABLE = "inventory"
IDEM_TABLE = "saga-idempotency"
STEP = "reserve"
def _already_done(key: str) -> bool:
r = _ddb.get_item(
TableName=IDEM_TABLE,
Key={"idempotency_key": {"S": f"{STEP}:{key}"}},
ConsistentRead=True,
)
return "Item" in r
def _mark_done(key: str) -> None:
_ddb.put_item(
TableName=IDEM_TABLE,
Item={"idempotency_key": {"S": f"{STEP}:{key}"}, "step": {"S": STEP}},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
def handler(event, _ctx):
sku = event["sku"]
qty = int(event["qty"])
idem = event["idempotency_key"]
if _already_done(idem):
return {**event, "reserve_status": "RESERVED", "idempotent": True}
_ddb.update_item(
TableName=INV_TABLE,
Key={"sku": {"S": sku}},
UpdateExpression="ADD reserved_qty :q",
ExpressionAttributeValues={":q": {"N": str(qty)}},
)
_mark_done(idem)
return {**event, "reserve_status": "RESERVED", "idempotent": False}
import os
import boto3
from botocore.config import Config
ENDPOINT = os.environ.get("AWS_ENDPOINT_URL", "http://localstack:4566")
SECRET_ID = os.environ["PAYMENT_SECRET_ID"]
_cfg = Config(retries={"max_attempts": 2, "mode": "standard"})
_ddb = boto3.client("dynamodb", endpoint_url=ENDPOINT, config=_cfg)
_sm = boto3.client("secretsmanager", endpoint_url=ENDPOINT, config=_cfg)
PAY_TABLE = "payments"
IDEM_TABLE = "saga-idempotency"
STEP = "charge"
def _already_done(key: str):
r = _ddb.get_item(
TableName=IDEM_TABLE,
Key={"idempotency_key": {"S": f"{STEP}:{key}"}},
ConsistentRead=True,
)
return r.get("Item")
def _mark_done(key: str, payment_id: str) -> None:
_ddb.put_item(
TableName=IDEM_TABLE,
Item={
"idempotency_key": {"S": f"{STEP}:{key}"},
"step": {"S": STEP},
"payment_id": {"S": payment_id},
},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
def handler(event, _ctx):
if event.get("force_failure") is True:
raise RuntimeError("forced charge failure")
order_id = event["order_id"]
amount = int(event["amount"])
idem = event["idempotency_key"]
payment_id = f"pay-{order_id}"
done = _already_done(idem)
if done:
return {**event, "payment_id": payment_id, "charge_status": "CAPTURED", "idempotent": True}
# Prove we can read the secret. The payment gateway is fake , we only log that we fetched it.
secret = _sm.get_secret_value(SecretId=SECRET_ID)
print(f"charge-payment: fetched gateway api key (length={len(secret['SecretString'])})")
_ddb.put_item(
TableName=PAY_TABLE,
Item={
"payment_id": {"S": payment_id},
"order_id": {"S": order_id},
"amount": {"N": str(amount)},
"status": {"S": "CAPTURED"},
},
)
_mark_done(idem, payment_id)
return {**event, "payment_id": payment_id, "charge_status": "CAPTURED", "idempotent": False}
import os
import boto3
from botocore.config import Config
ENDPOINT = os.environ.get("AWS_ENDPOINT_URL", "http://localstack:4566")
_cfg = Config(retries={"max_attempts": 2, "mode": "standard"})
_ddb = boto3.client("dynamodb", endpoint_url=ENDPOINT, config=_cfg)
SHIP_TABLE = "shipments"
IDEM_TABLE = "saga-idempotency"
STEP = "ship"
def _already_done(key: str):
r = _ddb.get_item(
TableName=IDEM_TABLE,
Key={"idempotency_key": {"S": f"{STEP}:{key}"}},
ConsistentRead=True,
)
return r.get("Item")
def _mark_done(key: str, shipment_id: str) -> None:
_ddb.put_item(
TableName=IDEM_TABLE,
Item={
"idempotency_key": {"S": f"{STEP}:{key}"},
"step": {"S": STEP},
"shipment_id": {"S": shipment_id},
},
ConditionExpression="attribute_not_exists(idempotency_key)",
)
def handler(event, _ctx):
if event.get("force_ship_failure") is True:
raise RuntimeError("forced ship failure")
order_id = event["order_id"]
idem = event["idempotency_key"]
shipment_id = f"ship-{order_id}"
done = _already_done(idem)
if done:
return {**event, "shipment_id": shipment_id, "ship_status": "CREATED", "idempotent": True}
_ddb.put_item(
TableName=SHIP_TABLE,
Item={
"shipment_id": {"S": shipment_id},
"order_id": {"S": order_id},
"sku": {"S": event["sku"]},
"qty": {"N": str(int(event["qty"]))},
"status": {"S": "CREATED"},
},
)
_mark_done(idem, shipment_id)
return {**event, "shipment_id": shipment_id, "ship_status": "CREATED", "idempotent": False}
import os
import boto3
from botocore.config import Config
ENDPOINT = os.environ.get("AWS_ENDPOINT_URL", "http://localstack:4566")
_cfg = Config(retries={"max_attempts": 2, "mode": "standard"})
_ddb = boto3.client("dynamodb", endpoint_url=ENDPOINT, config=_cfg)
INV_TABLE = "inventory"
def handler(event, _ctx):
# Compensation runs only when the forward step reserved inventory. If we never reserved,
# the saga skips this state and this handler is never called.
sku = event["sku"]
qty = int(event["qty"])
order_id = event["order_id"]
marker = f"released:{order_id}"
# Idempotent release: only subtract once per order_id, tracked as a set attribute on the row.
try:
_ddb.update_item(
TableName=INV_TABLE,
Key={"sku": {"S": sku}},
UpdateExpression="ADD reserved_qty :neg, released_orders :m",
ConditionExpression="(NOT contains(released_orders, :mk))",
ExpressionAttributeValues={
":neg": {"N": str(-qty)},
":m": {"SS": [marker]},
":mk": {"S": marker},
},
)
return {**event, "release_status": "RELEASED", "idempotent": False}
except _ddb.exceptions.ConditionalCheckFailedException:
return {**event, "release_status": "RELEASED", "idempotent": True}
import os
import boto3
from botocore.config import Config
ENDPOINT = os.environ.get("AWS_ENDPOINT_URL", "http://localstack:4566")
_cfg = Config(retries={"max_attempts": 2, "mode": "standard"})
_ddb = boto3.client("dynamodb", endpoint_url=ENDPOINT, config=_cfg)
PAY_TABLE = "payments"
def handler(event, _ctx):
# No-op refund if charge never succeeded (payment_id was never written).
order_id = event["order_id"]
payment_id = event.get("payment_id") or f"pay-{order_id}"
existing = _ddb.get_item(
TableName=PAY_TABLE,
Key={"payment_id": {"S": payment_id}},
ConsistentRead=True,
).get("Item")
if not existing:
return {**event, "refund_status": "NOOP"}
if existing.get("status", {}).get("S") == "REFUNDED":
return {**event, "refund_status": "REFUNDED", "idempotent": True}
_ddb.update_item(
TableName=PAY_TABLE,
Key={"payment_id": {"S": payment_id}},
UpdateExpression="SET #s = :r",
ConditionExpression="#s = :c",
ExpressionAttributeNames={"#s": "status"},
ExpressionAttributeValues={":r": {"S": "REFUNDED"}, ":c": {"S": "CAPTURED"}},
)
return {**event, "refund_status": "REFUNDED", "idempotent": False}
import json
def handler(event, _ctx):
detail_type = event.get("detail-type", "Unknown")
detail = event.get("detail", {})
print(f"saga-terminal:{detail_type} {json.dumps(detail, sort_keys=True)}")
return {"ok": True}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"Service": "lambda.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"Service": "states.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
{
"Comment": "Order saga: reserve -> charge -> ship with reverse-order compensations",
"StartAt": "ReserveInventory",
"States": {
"ReserveInventory": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "__RESERVE_ARN__",
"Payload.$": "$"
},
"ResultSelector": {"state.$": "$.Payload"},
"ResultPath": "$.reserve",
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"ResultPath": "$.error",
"Next": "PublishFailed"
}
],
"Next": "ChargePayment"
},
"ChargePayment": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "__CHARGE_ARN__",
"Payload.$": "$.reserve.state"
},
"ResultSelector": {"state.$": "$.Payload"},
"ResultPath": "$.charge",
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"ResultPath": "$.error",
"Next": "ReleaseInventoryAfterChargeFail"
}
],
"Next": "CreateShipment"
},
"CreateShipment": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "__SHIP_ARN__",
"Payload.$": "$.charge.state"
},
"ResultSelector": {"state.$": "$.Payload"},
"ResultPath": "$.ship",
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"ResultPath": "$.error",
"Next": "RefundPaymentAfterShipFail"
}
],
"Next": "PublishCompleted"
},
"RefundPaymentAfterShipFail": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "__REFUND_ARN__",
"Payload.$": "$.charge.state"
},
"ResultSelector": {"state.$": "$.Payload"},
"ResultPath": "$.refund",
"Next": "ReleaseInventoryAfterShipFail"
},
"ReleaseInventoryAfterShipFail": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "__RELEASE_ARN__",
"Payload.$": "$.refund.state"
},
"ResultSelector": {"state.$": "$.Payload"},
"ResultPath": "$.release",
"Next": "PublishFailed"
},
"ReleaseInventoryAfterChargeFail": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": {
"FunctionName": "__RELEASE_ARN__",
"Payload.$": "$.reserve.state"
},
"ResultSelector": {"state.$": "$.Payload"},
"ResultPath": "$.release",
"Next": "PublishFailed"
},
"PublishCompleted": {
"Type": "Task",
"Resource": "arn:aws:states:::events:putEvents",
"Parameters": {
"Entries": [
{
"EventBusName": "saga-events",
"Source": "order.saga",
"DetailType": "OrderCompleted",
"Detail": {
"order_id.$": "$.ship.state.order_id",
"payment_id.$": "$.ship.state.payment_id",
"shipment_id.$": "$.ship.state.shipment_id"
}
}
]
},
"End": true
},
"PublishFailed": {
"Type": "Task",
"Resource": "arn:aws:states:::events:putEvents",
"Parameters": {
"Entries": [
{
"EventBusName": "saga-events",
"Source": "order.saga",
"DetailType": "OrderFailed",
"Detail": {
"order_id.$": "$.order_id",
"idempotency_key.$": "$.idempotency_key"
}
}
]
},
"End": true
}
}
}
#!/usr/bin/env bash
set -euo pipefail
export AWS_PAGER=""
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
ACCOUNT_ID=000000000000
REGION=us-east-1
ROOT=/app
POL=$ROOT/policies
BUILD=$ROOT/build
mkdir -p "$BUILD"
echo "== KMS CMK + alias"
KEY_POLICY=$(cat <<'JSON'
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "EnableRootAdmin",
"Effect": "Allow",
"Principal": {"AWS": "arn:aws:iam::000000000000:root"},
"Action": "kms:*",
"Resource": "*"
}
]
}
JSON
)
KEY_ID=$($AWS kms create-key \
--description "Saga customer-managed key" \
--policy "$KEY_POLICY" \
--query 'KeyMetadata.KeyId' --output text)
KEY_ARN="arn:aws:kms:${REGION}:${ACCOUNT_ID}:key/${KEY_ID}"
$AWS kms create-alias --alias-name alias/saga-cmk --target-key-id "$KEY_ID" || true
echo "KEY_ARN=$KEY_ARN"
echo "== Secret (CMK-encrypted)"
$AWS secretsmanager create-secret \
--name payment/gateway-apikey \
--kms-key-id "alias/saga-cmk" \
--secret-string '{"api_key":"sk_live_fake_gateway_key"}' >/dev/null
SECRET_ARN=$($AWS secretsmanager describe-secret --secret-id payment/gateway-apikey --query 'ARN' --output text)
echo "SECRET_ARN=$SECRET_ARN"
echo "== DynamoDB tables"
$AWS dynamodb create-table \
--table-name inventory \
--attribute-definitions AttributeName=sku,AttributeType=S \
--key-schema AttributeName=sku,KeyType=HASH \
--billing-mode PAY_PER_REQUEST >/dev/null
$AWS dynamodb create-table \
--table-name payments \
--attribute-definitions AttributeName=payment_id,AttributeType=S \
--key-schema AttributeName=payment_id,KeyType=HASH \
--billing-mode PAY_PER_REQUEST >/dev/null
$AWS dynamodb create-table \
--table-name shipments \
--attribute-definitions AttributeName=shipment_id,AttributeType=S \
--key-schema AttributeName=shipment_id,KeyType=HASH \
--billing-mode PAY_PER_REQUEST >/dev/null
$AWS dynamodb create-table \
--table-name saga-idempotency \
--attribute-definitions AttributeName=idempotency_key,AttributeType=S \
--key-schema AttributeName=idempotency_key,KeyType=HASH \
--billing-mode PAY_PER_REQUEST >/dev/null
$AWS dynamodb update-time-to-live \
--table-name saga-idempotency \
--time-to-live-specification 'Enabled=true,AttributeName=ttl' >/dev/null
for t in inventory payments shipments saga-idempotency; do
$AWS dynamodb wait table-exists --table-name "$t"
done
INV_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT_ID}:table/inventory"
PAY_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT_ID}:table/payments"
SHIP_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT_ID}:table/shipments"
IDEM_ARN="arn:aws:dynamodb:${REGION}:${ACCOUNT_ID}:table/saga-idempotency"
echo "== Per-lambda least-privilege policies"
# reserve: inventory + saga-idempotency RW, logs
cat > "$BUILD/policy-reserve.json" <<JSON
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["dynamodb:UpdateItem","dynamodb:GetItem","dynamodb:PutItem"], "Resource": ["$INV_ARN","$IDEM_ARN"]},
{"Effect": "Allow", "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], "Resource": "arn:aws:logs:${REGION}:${ACCOUNT_ID}:log-group:/aws/lambda/reserve-inventory:*"}
]
}
JSON
# charge: payments + saga-idempotency RW, secrets read on specific secret, kms decrypt on specific key, logs
cat > "$BUILD/policy-charge.json" <<JSON
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["dynamodb:GetItem","dynamodb:PutItem"], "Resource": ["$PAY_ARN","$IDEM_ARN"]},
{"Effect": "Allow", "Action": ["secretsmanager:GetSecretValue","secretsmanager:DescribeSecret"], "Resource": "$SECRET_ARN"},
{"Effect": "Allow", "Action": ["kms:Decrypt"], "Resource": "$KEY_ARN"},
{"Effect": "Allow", "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], "Resource": "arn:aws:logs:${REGION}:${ACCOUNT_ID}:log-group:/aws/lambda/charge-payment:*"}
]
}
JSON
# ship: shipments + saga-idempotency RW, logs
cat > "$BUILD/policy-ship.json" <<JSON
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["dynamodb:GetItem","dynamodb:PutItem"], "Resource": ["$SHIP_ARN","$IDEM_ARN"]},
{"Effect": "Allow", "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], "Resource": "arn:aws:logs:${REGION}:${ACCOUNT_ID}:log-group:/aws/lambda/create-shipment:*"}
]
}
JSON
# release: only inventory RW, logs
cat > "$BUILD/policy-release.json" <<JSON
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["dynamodb:UpdateItem"], "Resource": "$INV_ARN"},
{"Effect": "Allow", "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], "Resource": "arn:aws:logs:${REGION}:${ACCOUNT_ID}:log-group:/aws/lambda/release-inventory:*"}
]
}
JSON
# refund: only payments RW, logs
cat > "$BUILD/policy-refund.json" <<JSON
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["dynamodb:GetItem","dynamodb:UpdateItem"], "Resource": "$PAY_ARN"},
{"Effect": "Allow", "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], "Resource": "arn:aws:logs:${REGION}:${ACCOUNT_ID}:log-group:/aws/lambda/refund-payment:*"}
]
}
JSON
# notifier: logs only (event bus invokes it; no outbound resources)
cat > "$BUILD/policy-notifier.json" <<JSON
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], "Resource": "arn:aws:logs:${REGION}:${ACCOUNT_ID}:log-group:/aws/lambda/saga-notifier:*"}
]
}
JSON
echo "== Create IAM roles"
create_role() {
local name=$1 trust=$2 policy=$3 policy_name=$4
$AWS iam create-role --role-name "$name" --assume-role-policy-document "file://$trust" >/dev/null
$AWS iam put-role-policy --role-name "$name" --policy-name "$policy_name" --policy-document "file://$policy" >/dev/null
}
create_role reserve-inventory-role "$POL/trust-lambda.json" "$BUILD/policy-reserve.json" reserve-inline
create_role charge-payment-role "$POL/trust-lambda.json" "$BUILD/policy-charge.json" charge-inline
create_role create-shipment-role "$POL/trust-lambda.json" "$BUILD/policy-ship.json" ship-inline
create_role release-inventory-role "$POL/trust-lambda.json" "$BUILD/policy-release.json" release-inline
create_role refund-payment-role "$POL/trust-lambda.json" "$BUILD/policy-refund.json" refund-inline
create_role saga-notifier-role "$POL/trust-lambda.json" "$BUILD/policy-notifier.json" notifier-inline
echo "== Package lambdas"
package() {
local src=$1 out=$2
(cd "$ROOT/lambdas" && zip -q -j "$BUILD/$out" "$src")
}
package reserve_inventory.py reserve.zip
package charge_payment.py charge.zip
package create_shipment.py ship.zip
package release_inventory.py release.zip
package refund_payment.py refund.zip
package saga_notifier.py notifier.zip
echo "== Deploy lambdas"
deploy_lambda() {
local name=$1 role=$2 zip=$3 handler=$4 env_json=${5:-"{}"}
$AWS lambda create-function \
--function-name "$name" \
--runtime python3.11 \
--role "arn:aws:iam::${ACCOUNT_ID}:role/${role}" \
--handler "$handler" \
--zip-file "fileb://$BUILD/$zip" \
--environment "Variables=$env_json" \
--timeout 20 >/dev/null
$AWS lambda wait function-active --function-name "$name"
}
deploy_lambda reserve-inventory reserve-inventory-role reserve.zip reserve_inventory.handler '{}'
deploy_lambda charge-payment charge-payment-role charge.zip charge_payment.handler "{PAYMENT_SECRET_ID=payment/gateway-apikey}"
deploy_lambda create-shipment create-shipment-role ship.zip create_shipment.handler '{}'
deploy_lambda release-inventory release-inventory-role release.zip release_inventory.handler '{}'
deploy_lambda refund-payment refund-payment-role refund.zip refund_payment.handler '{}'
deploy_lambda saga-notifier saga-notifier-role notifier.zip saga_notifier.handler '{}'
RESERVE_ARN=$($AWS lambda get-function --function-name reserve-inventory --query 'Configuration.FunctionArn' --output text)
CHARGE_ARN=$($AWS lambda get-function --function-name charge-payment --query 'Configuration.FunctionArn' --output text)
SHIP_ARN_L=$($AWS lambda get-function --function-name create-shipment --query 'Configuration.FunctionArn' --output text)
RELEASE_ARN=$($AWS lambda get-function --function-name release-inventory --query 'Configuration.FunctionArn' --output text)
REFUND_ARN=$($AWS lambda get-function --function-name refund-payment --query 'Configuration.FunctionArn' --output text)
NOTIFIER_ARN=$($AWS lambda get-function --function-name saga-notifier --query 'Configuration.FunctionArn' --output text)
echo "== EventBridge bus/rule/target"
$AWS events create-event-bus --name saga-events >/dev/null
BUS_ARN="arn:aws:events:${REGION}:${ACCOUNT_ID}:event-bus/saga-events"
$AWS events put-rule \
--name on-saga-terminal \
--event-bus-name saga-events \
--event-pattern '{"source":["order.saga"],"detail-type":["OrderCompleted","OrderFailed"]}' \
--state ENABLED >/dev/null
$AWS events put-targets \
--rule on-saga-terminal \
--event-bus-name saga-events \
--targets "Id=1,Arn=${NOTIFIER_ARN}" >/dev/null
RULE_ARN="arn:aws:events:${REGION}:${ACCOUNT_ID}:rule/saga-events/on-saga-terminal"
$AWS lambda add-permission \
--function-name saga-notifier \
--statement-id allow-eventbridge \
--action lambda:InvokeFunction \
--principal events.amazonaws.com \
--source-arn "$RULE_ARN" >/dev/null
echo "== State machine role (scoped to actual function and bus ARNs only)"
cat > "$BUILD/policy-saga.json" <<JSON
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": [
"$RESERVE_ARN",
"$CHARGE_ARN",
"$SHIP_ARN_L",
"$RELEASE_ARN",
"$REFUND_ARN"
]
},
{
"Effect": "Allow",
"Action": "events:PutEvents",
"Resource": "$BUS_ARN"
}
]
}
JSON
$AWS iam create-role --role-name order-saga-role \
--assume-role-policy-document "file://$POL/trust-states.json" >/dev/null
$AWS iam put-role-policy --role-name order-saga-role \
--policy-name saga-inline \
--policy-document "file://$BUILD/policy-saga.json" >/dev/null
echo "== Materialise state machine definition"
SAGA_DEF=$BUILD/order-saga.json
sed \
-e "s|__RESERVE_ARN__|$RESERVE_ARN|g" \
-e "s|__CHARGE_ARN__|$CHARGE_ARN|g" \
-e "s|__SHIP_ARN__|$SHIP_ARN_L|g" \
-e "s|__RELEASE_ARN__|$RELEASE_ARN|g" \
-e "s|__REFUND_ARN__|$REFUND_ARN|g" \
"$ROOT/sm/order-saga.json.tmpl" > "$SAGA_DEF"
echo "== Create state machine"
SM_ARN=$($AWS stepfunctions create-state-machine \
--name order-saga \
--definition "file://$SAGA_DEF" \
--role-arn "arn:aws:iam::${ACCOUNT_ID}:role/order-saga-role" \
--query 'stateMachineArn' --output text)
echo "SM_ARN=$SM_ARN"
echo "$SM_ARN" > "$BUILD/sm_arn.txt"
echo "$NOTIFIER_ARN" > "$BUILD/notifier_arn.txt"
echo "DONE"
Run full deploy script
chmod +x /app/deploy.sh && bash /app/deploy.sh 2>&1 | tail -80
== KMS CMK + alias KEY_ARN=arn:aws:kms:us-east-1:000000000000:key/a82d57d2-5311-4114-9016-212910093270 == Secret (CMK-encrypted) SECRET_ARN=arn:aws:secretsmanager:us-east-1:000000000000:secret:payment/gateway-apikey-vrvuWc == DynamoDB tables == Per-lambda least-privilege policies == Create IAM roles == Package lambdas == Deploy lambdas == EventBridge bus/rule/target == State machine role (scoped to actual function and bus ARNs only) == Materialise state machine definition == Create state machine SM_ARN=arn:aws:states:us-east-1:000000000000:stateMachine:order-saga DONE [stdout] == KMS CMK + alias KEY_ARN=arn:aws:kms:us-east-1:000000000000:key/a82d57d2-5311-4114-9016-212910093270 == Secret (CMK-encrypted) SECRET_ARN=arn:aws:secretsmanager:us-east-1:000000000000:secret:payment/gateway-apikey-vrvuWc == DynamoDB tables == Per-lambda least-privilege policies == Create IAM roles == Package lambdas == Deploy lambdas == EventBridge bus/rule/target == State machine role (scoped to actual function and bus ARNs only) == Materialise state machine definition == Create state machine SM_ARN=arn:aws:states:us-east-1:000000000000:stateMachine:order-saga DONE
Run happy path execution
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
export AWS_PAGER=""
SM_ARN=$(cat /app/build/sm_arn.txt)
echo "== Start happy path"
EXEC_ARN=$($AWS stepfunctions start-execution \
--state-machine-arn "$SM_ARN" \
--name happy-$(date +%s) \
--input '{"order_id":"o-1","sku":"x","qty":2,"amount":100,"idempotency_key":"k-1"}' \
--query 'executionArn' --output text)
echo "exec=$EXEC_ARN"
for i in $(seq 1 30); do
STATUS=$($AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query 'status' --output text)
[ "$STATUS" != "RUNNING" ] && break
sleep 2
done
echo "status=$STATUS"
$AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query '{status:status,output:output}' --output json
== Start happy path
exec=arn:aws:states:us-east-1:000000000000:execution:order-saga:happy-1776922634
status=SUCCEEDED
{
"status": "SUCCEEDED",
"output": "{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"bec10122-3bcf-4b52-ba64-0613b819ce2a\"}]}"
}
[stdout]
== Start happy path
exec=arn:aws:states:us-east-1:000000000000:execution:order-saga:happy-1776922634
status=SUCCEEDED
{
"status": "SUCCEEDED",
"output": "{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"bec10122-3bcf-4b52-ba64-0613b819ce2a\"}]}"
}Verify happy-path DB state and notifier log
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
export AWS_PAGER=""
echo "-- inventory (sku=x)"
$AWS dynamodb get-item --table-name inventory --key '{"sku":{"S":"x"}}' --output json
echo "-- payments (pay-o-1)"
$AWS dynamodb get-item --table-name payments --key '{"payment_id":{"S":"pay-o-1"}}' --output json
echo "-- shipments (ship-o-1)"
$AWS dynamodb get-item --table-name shipments --key '{"shipment_id":{"S":"ship-o-1"}}' --output json
echo "-- notifier logs (saga-terminal)"
# Give eventbridge a moment to fan out
sleep 3
LG=/aws/lambda/saga-notifier
STREAMS=$($AWS logs describe-log-streams --log-group-name "$LG" --query 'logStreams[].logStreamName' --output text 2>&1 || echo "")
echo "streams=$STREAMS"
for s in $STREAMS; do
$AWS logs get-log-events --log-group-name "$LG" --log-stream-name "$s" --query 'events[].message' --output text | grep -i saga-terminal || true
done
-- inventory (sku=x)
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "2"
}
}
}
-- payments (pay-o-1)
{
"Item": {
"amount": {
"N": "100"
},
"order_id": {
"S": "o-1"
},
"payment_id": {
"S": "pay-o-1"
},
"status": {
"S": "CAPTURED"
}
}
}
-- shipments (ship-o-1)
{
"Item": {
"shipment_id": {
"S": "ship-o-1"
},
"sku": {
"S": "x"
},
"order_id": {
"S": "o-1"
},
"qty": {
"N": "2"
},
"status": {
"S": "CREATED"
}
}
}
-- notifier logs (saga-terminal)
streams=2026/04/23/[$LATEST]1cbc65049953a5dfc92ea629a2fb6323
saga-terminal:OrderCompleted {"order_id": "o-1", "payment_id": "pay-o-1", "shipment_id": "ship-o-1"}
[stdout]
-- inventory (sku=x)
{
"Item": {
"sku": {
"S": "x"
},
"reserved_qty": {
"N": "2"
}
}
}
-- payments (pay-o-1)
{
"Item": {
"amount": {
"N": "100"
},
"order_id": {
"S": "o-1"
},
"payment_id": {
"S": "pay-o-1"
},
"status": {
"S": "CAPTURED"
}
}
}
-- shipments (ship-o-1)
{
"Item": {
"shipment_id": {
"S": "ship-o-1"
},
"sku": {
"S": "x"
},
"order_id": {
"S": "o-1"
},
"qty": {
"N": "2"
},
"status": {
"S": "CREATED"
}
}
}
-- notifier logs (saga-terminal)
streams=2026/04/23/[$LATEST]1cbc65049953a5dfc92ea629a2fb6323
saga-terminal:OrderCompleted {"order_id": "o-1", "payment_id": "pay-o-1", "shipment_id": "ship-o-1"}Run failure path and assert rollback
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
export AWS_PAGER=""
SM_ARN=$(cat /app/build/sm_arn.txt)
echo "== Start failure path"
EXEC_ARN=$($AWS stepfunctions start-execution \
--state-machine-arn "$SM_ARN" \
--name fail-$(date +%s) \
--input '{"order_id":"o-2","sku":"x","qty":2,"amount":100,"idempotency_key":"k-2","force_failure":true}' \
--query 'executionArn' --output text)
echo "exec=$EXEC_ARN"
for i in $(seq 1 30); do
STATUS=$($AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query 'status' --output text)
[ "$STATUS" != "RUNNING" ] && break
sleep 2
done
echo "status=$STATUS"
$AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query '{status:status,output:output}' --output json
echo "-- inventory (should be back to 0 after release)"
$AWS dynamodb get-item --table-name inventory --key '{"sku":{"S":"x"}}' --output json
echo "-- payments (pay-o-2 should NOT exist since charge failed before writing)"
$AWS dynamodb get-item --table-name payments --key '{"payment_id":{"S":"pay-o-2"}}' --output json
echo "-- shipments (ship-o-2 should NOT exist)"
$AWS dynamodb get-item --table-name shipments --key '{"shipment_id":{"S":"ship-o-2"}}' --output json
echo "-- notifier logs"
sleep 3
LG=/aws/lambda/saga-notifier
STREAMS=$($AWS logs describe-log-streams --log-group-name "$LG" --query 'logStreams[].logStreamName' --output text)
for s in $STREAMS; do
$AWS logs get-log-events --log-group-name "$LG" --log-stream-name "$s" --query 'events[].message' --output text | grep -i saga-terminal || true
done
== Start failure path
exec=arn:aws:states:us-east-1:000000000000:execution:order-saga:fail-1776922661
status=SUCCEEDED
{
"status": "SUCCEEDED",
"output": "{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"86590887-10d5-45af-b99f-4a8538a61942\"}]}"
}
-- inventory (should be back to 0 after release)
{
"Item": {
"sku": {
"S": "x"
},
"released_orders": {
"SS": [
"released:o-2"
]
},
"reserved_qty": {
"N": "2"
}
}
}
-- payments (pay-o-2 should NOT exist since charge failed before writing)
-- shipments (ship-o-2 should NOT exist)
-- notifier logs
saga-terminal:OrderCompleted {"order_id": "o-1", "payment_id": "pay-o-1", "shipment_id": "ship-o-1"}
saga-terminal:OrderFailed {"idempotency_key": "k-2", "order_id": "o-2"}
[stdout]
== Start failure path
exec=arn:aws:states:us-east-1:000000000000:execution:order-saga:fail-1776922661
status=SUCCEEDED
{
"status": "SUCCEEDED",
"output": "{\"FailedEntryCount\":0,\"Entries\":[{\"EventId\":\"86590887-10d5-45af-b99f-4a8538a61942\"}]}"
}
-- inventory (should be back to 0 after release)
{
"Item": {
"sku": {
"S": "x"
},
"released_orders": {
"SS": [
"released:o-2"
]
},
"reserved_qty": {
"N": "2"
}
}
}
-- payments (pay-o-2 should NOT exist since charge failed before writing)
-- shipments (ship-o-2 should NOT exist)
-- notifier logs
saga-terminal:OrderCompleted {"order_id": "o-1", "payment_id": "pay-o-1", "shipment_id": "ship-o-1"}
saga-terminal:OrderFailed {"idempotency_key": "k-2", "order_id": "o-2"}Isolated failure path on fresh SKU
set -e
AWS="aws --endpoint-url=http://localstack:4566 --region us-east-1"
export AWS_PAGER=""
SM_ARN=$(cat /app/build/sm_arn.txt)
echo "== Failure path against a fresh sku to prove isolation"
EXEC_ARN=$($AWS stepfunctions start-execution \
--state-machine-arn "$SM_ARN" \
--name failfresh-$(date +%s) \
--input '{"order_id":"o-3","sku":"y","qty":5,"amount":50,"idempotency_key":"k-3","force_failure":true}' \
--query 'executionArn' --output text)
for i in $(seq 1 30); do
STATUS=$($AWS stepfunctions describe-execution --execution-arn "$EXEC_ARN" --query 'status' --output text)
[ "$STATUS" != "RUNNING" ] && break
sleep 2
done
echo "status=$STATUS"
echo "-- inventory sku=y (compensation should leave reserved_qty=0)"
$AWS dynamodb get-item --table-name inventory --key '{"sku":{"S":"y"}}' --output json
echo "-- payments pay-o-3 (absent: charge failed before write)"
$AWS dynamodb get-item --table-name payments --key '{"payment_id":{"S":"pay-o-3"}}' --output json
echo "(empty = absent as expected)"
echo "-- shipments ship-o-3 (absent)"
$AWS dynamodb get-item --table-name shipments --key '{"shipment_id":{"S":"ship-o-3"}}' --output json
echo "(empty = absent as expected)"
== Failure path against a fresh sku to prove isolation
status=SUCCEEDED
-- inventory sku=y (compensation should leave reserved_qty=0)
{
"Item": {
"sku": {
"S": "y"
},
"released_orders": {
"SS": [
"released:o-3"
]
},
"reserved_qty": {
"N": "0"
}
}
}
-- payments pay-o-3 (absent: charge failed before write)
(empty = absent as expected)
-- shipments ship-o-3 (absent)
(empty = absent as expected)
[stdout]
== Failure path against a fresh sku to prove isolation
status=SUCCEEDED
-- inventory sku=y (compensation should leave reserved_qty=0)
{
"Item": {
"sku": {
"S": "y"
},
"released_orders": {
"SS": [
"released:o-3"
]
},
"reserved_qty": {
"N": "0"
}
}
}
-- payments pay-o-3 (absent: charge failed before write)
(empty = absent as expected)
-- shipments ship-o-3 (absent)
(empty = absent as expected)No file changes, the agent left the workspace untouched.
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0 -- /root/.cache/uv/archive-v0/SWnVkAM0THwiEH_TsUKcY/bin/python
cachedir: .pytest_cache
rootdir: /tests
plugins: json-ctrf-0.3.5
collecting ... collected 20 items
../tests/test_outputs.py::test_01_cmk_alias_and_secret_exist PASSED [ 5%]
../tests/test_outputs.py::test_02_all_four_tables_exist PASSED [ 10%]
../tests/test_outputs.py::test_03_all_six_lambdas_exist PASSED [ 15%]
../tests/test_outputs.py::test_04_state_machine_and_bus_exist PASSED [ 20%]
../tests/test_outputs.py::test_05_terminal_rule_exists_on_saga_bus PASSED [ 25%]
../tests/test_outputs.py::test_06_every_forward_state_has_catch_to_compensation PASSED [ 30%]
../tests/test_outputs.py::test_07_compensations_run_in_lifo_order PASSED [ 35%]
../tests/test_outputs.py::test_08_compensation_states_preserve_error_via_result_path PASSED [ 40%]
../tests/test_outputs.py::test_09_idempotency_table_has_ttl_and_correct_pk PASSED [ 45%]
../tests/test_outputs.py::test_10_each_lambda_role_scoped_to_its_own_table PASSED [ 50%]
../tests/test_outputs.py::test_11_only_charge_role_reads_the_payment_secret PASSED [ 55%]
../tests/test_outputs.py::test_12_saga_role_lambda_invoke_not_wildcarded PASSED [ 60%]
../tests/test_outputs.py::test_13_rule_pattern_matches_both_terminal_types_with_array_wrap PASSED [ 65%]
../tests/test_outputs.py::test_14_charge_role_kms_decrypt_scoped_not_wildcard PASSED [ 70%]
../tests/test_outputs.py::test_15_rule_targets_notifier_lambda PASSED [ 75%]
../tests/test_outputs.py::test_16_notifier_role_is_least_privilege_no_ddb_kms_secrets PASSED [ 80%]
../tests/test_outputs.py::test_17_notifier_lambda_resource_policy_allows_events_with_source_arn PASSED [ 85%]
../tests/test_outputs.py::test_18_happy_path_completes_and_writes_all_three_tables PASSED [ 90%]
../tests/test_outputs.py::test_19_failure_path_triggers_compensation_and_rolls_back_inventory PASSED [ 95%]
../tests/test_outputs.py::test_20_idempotency_prevents_double_reservation PASSED [100%]
=============================== warnings summary ===============================
test_outputs.py: 135 warnings
/root/.cache/uv/archive-v0/SWnVkAM0THwiEH_TsUKcY/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
==================================== PASSES ====================================
=========================== short test summary info ============================
PASSED ../tests/test_outputs.py::test_01_cmk_alias_and_secret_exist
PASSED ../tests/test_outputs.py::test_02_all_four_tables_exist
PASSED ../tests/test_outputs.py::test_03_all_six_lambdas_exist
PASSED ../tests/test_outputs.py::test_04_state_machine_and_bus_exist
PASSED ../tests/test_outputs.py::test_05_terminal_rule_exists_on_saga_bus
PASSED ../tests/test_outputs.py::test_06_every_forward_state_has_catch_to_compensation
PASSED ../tests/test_outputs.py::test_07_compensations_run_in_lifo_order
PASSED ../tests/test_outputs.py::test_08_compensation_states_preserve_error_via_result_path
PASSED ../tests/test_outputs.py::test_09_idempotency_table_has_ttl_and_correct_pk
PASSED ../tests/test_outputs.py::test_10_each_lambda_role_scoped_to_its_own_table
PASSED ../tests/test_outputs.py::test_11_only_charge_role_reads_the_payment_secret
PASSED ../tests/test_outputs.py::test_12_saga_role_lambda_invoke_not_wildcarded
PASSED ../tests/test_outputs.py::test_13_rule_pattern_matches_both_terminal_types_with_array_wrap
PASSED ../tests/test_outputs.py::test_14_charge_role_kms_decrypt_scoped_not_wildcard
PASSED ../tests/test_outputs.py::test_15_rule_targets_notifier_lambda
PASSED ../tests/test_outputs.py::test_16_notifier_role_is_least_privilege_no_ddb_kms_secrets
PASSED ../tests/test_outputs.py::test_17_notifier_lambda_resource_policy_allows_events_with_source_arn
PASSED ../tests/test_outputs.py::test_18_happy_path_completes_and_writes_all_three_tables
PASSED ../tests/test_outputs.py::test_19_failure_path_triggers_compensation_and_rolls_back_inventory
PASSED ../tests/test_outputs.py::test_20_idempotency_prevents_double_reservation
====================== 20 passed, 135 warnings in 23.20s =======================Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_2ddb0cc065b14777. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_2ddb0cc065b14777 · verifier authoritative; classifier explanatory.