iam-session-tag-tenant-scope

ls /app && echo "---" && ls -la /logs/agent/sessions/projects/-app/memory/ 2>/dev/null || echo "no memory yet"

ls -la /app/ && echo "---" && which aws localstack awslocal && echo "---" && aws --version

aws --version 2>&1; echo "---"; awslocal --version 2>&1; echo "---"; curl -s http://localhost:4566/_localstack/health | head -c 1500

curl -s http://localhost:4566/_localstack/health; echo ""; echo "---"; curl -s http://localhost.localstack.cloud:4566/_localstack/health 2>&1 | head -c 200

env | grep -iE 'aws|localstack|endpoint' 2>&1; echo "---"; cat ~/.aws/config 2>&1; echo "---"; cat ~/.aws/credentials 2>&1

curl -s http://localstack:4566/_localstack/health

awslocal sts get-caller-identity

mkdir -p /app/build /app/policies

KEY_JSON=$(awslocal kms create-key --description "saas-orders CMK" --key-usage ENCRYPT_DECRYPT) && KEY_ID=$(echo "$KEY_JSON" | python3 -c "import sys,json; print(json.load(sys.stdin)['KeyMetadata']['KeyId'])") && echo "KEY_ID=$KEY_ID" && awslocal kms create-alias --alias-name alias/saas-orders-cmk --target-key-id "$KEY_ID" && echo "alias created" && echo "$KEY_ID" > /app/build/kms_key_id

KEY_ID=$(cat /app/build/kms_key_id) && awslocal dynamodb create-table \
  --table-name SaasOrders \
  --attribute-definitions AttributeName=TenantID,AttributeType=S AttributeName=OrderID,AttributeType=S \
  --key-schema AttributeName=TenantID,KeyType=HASH AttributeName=OrderID,KeyType=RANGE \
  --billing-mode PAY_PER_REQUEST \
  --sse-specification "Enabled=true,SSEType=KMS,KMSMasterKeyId=alias/saas-orders-cmk" \
  >/dev/null && echo "table created" && \
  awslocal dynamodb update-continuous-backups --table-name SaasOrders \
  --point-in-time-recovery-specification PointInTimeRecoveryEnabled=true >/dev/null && echo "PITR on"

awslocal dynamodb describe-table --table-name SaasOrders --query 'Table.{SSE:SSEDescription,Status:TableStatus,KeySchema:KeySchema}' && echo "---" && awslocal dynamodb describe-continuous-backups --table-name SaasOrders

cat > /app/build/seed.py <<'PY'
import boto3, os
ddb = boto3.client('dynamodb', endpoint_url=os.environ.get('AWS_ENDPOINT_URL'))
items = [
    ('acme',     'O-1001', '125.00', 'acme-secret-1'),
    ('acme',     'O-1002', '250.00', 'acme-secret-2'),
    ('globex',   'O-2001', '999.00', 'globex-secret-1'),
    ('globex',   'O-2002', '17.50', 'globex-secret-2'),
    ('globex-eu', 'O-3001', '42.00', 'globex-eu-secret-1'),
    ('globex-eu', 'O-3002', '88.88', 'globex-eu-secret-2'),
]
for tid, oid, amt, pwd in items:
    ddb.put_item(TableName='SaasOrders', Item={
        'TenantID': {'S': tid},
        'OrderID':  {'S': oid},
        'amount':   {'S': amt},
        'password': {'S': pwd},
    })
print('seeded', len(items))
PY
python3 /app/build/seed.py

awslocal iam create-role --role-name TenantTokenVendorRole \
  --assume-role-policy-document file:///app/policies/vendor_trust.json \
  --query 'Role.Arn' --output text

awslocal iam create-role --role-name TenantDataRole \
  --assume-role-policy-document file:///app/policies/data_trust.json \
  --query 'Role.Arn' --output text

awslocal iam put-role-policy --role-name TenantDataRole \
  --policy-name TenantDataAccess \
  --policy-document file:///app/policies/data_identity.json && echo "data identity policy attached"

awslocal iam put-role-policy --role-name TenantTokenVendorRole \
  --policy-name VendorAssumeAndLogs \
  --policy-document file:///app/policies/vendor_identity.json && echo "vendor identity policy attached"

cd /app/build && zip -q -j tenant-token-vendor.zip lambda_function.py && ls -la tenant-token-vendor.zip && \
awslocal logs create-log-group --log-group-name /aws/lambda/tenant-token-vendor && echo "log group created" && \
awslocal lambda create-function \
  --function-name tenant-token-vendor \
  --runtime python3.11 \
  --role arn:aws:iam::000000000000:role/TenantTokenVendorRole \
  --handler lambda_function.handler \
  --zip-file fileb:///app/build/tenant-token-vendor.zip \
  --environment "Variables={DATA_ROLE_ARN=arn:aws:iam::000000000000:role/TenantDataRole}" \
  --timeout 15 \
  --query 'FunctionArn' --output text

for i in 1 2 3 4 5 6 7 8; do
  state=$(awslocal lambda get-function --function-name tenant-token-vendor --query 'Configuration.State' --output text 2>/dev/null)
  echo "state=$state"
  [ "$state" = "Active" ] && break
  sleep 1
done

awslocal lambda invoke --function-name tenant-token-vendor \
  --cli-binary-format raw-in-base64-out \
  --payload '{"queryStringParameters":{"tenant":"acme"}}' \
  /tmp/lambda-out.json >/tmp/lambda-meta.json
cat /tmp/lambda-meta.json
echo ""
echo "--- payload ---"
cat /tmp/lambda-out.json | python3 -c 'import json,sys; d=json.load(sys.stdin); b=json.loads(d.get("body","{}")); print("status:",d.get("statusCode")); print("keys:",sorted(b.keys()))'

API_ID=$(awslocal apigateway create-rest-api --name harbor-saas-api --query 'id' --output text)
echo "API_ID=$API_ID"
ROOT_ID=$(awslocal apigateway get-resources --rest-api-id "$API_ID" --query 'items[?path==`/`].id' --output text)
echo "ROOT_ID=$ROOT_ID"
TOKEN_ID=$(awslocal apigateway create-resource --rest-api-id "$API_ID" --parent-id "$ROOT_ID" --path-part token --query 'id' --output text)
echo "TOKEN_ID=$TOKEN_ID"
awslocal apigateway put-method --rest-api-id "$API_ID" --resource-id "$TOKEN_ID" --http-method GET --authorization-type NONE >/dev/null && echo "method GET created"

LAMBDA_ARN="arn:aws:lambda:us-east-1:000000000000:function:tenant-token-vendor"
INTEGRATION_URI="arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/${LAMBDA_ARN}/invocations"
awslocal apigateway put-integration --rest-api-id "$API_ID" --resource-id "$TOKEN_ID" --http-method GET \
  --type AWS_PROXY --integration-http-method POST --uri "$INTEGRATION_URI" >/dev/null && echo "integration AWS_PROXY set"

echo "$API_ID" > /app/build/api_id
echo "$TOKEN_ID" > /app/build/token_id

API_ID=$(cat /app/build/api_id)
# Create access log group, then deploy stage with access logging
awslocal logs create-log-group --log-group-name /aws/apigateway/harbor-saas-api/access && echo "access log group created"

awslocal apigateway create-deployment --rest-api-id "$API_ID" --stage-name prod >/dev/null && echo "initial deployment created"

# Configure access logging on the stage
awslocal apigateway update-stage --rest-api-id "$API_ID" --stage-name prod \
  --patch-operations \
    "op=replace,path=/accessLogSettings/destinationArn,value=arn:aws:logs:us-east-1:000000000000:log-group:/aws/apigateway/harbor-saas-api/access" \
    "op=replace,path=/accessLogSettings/format,value={\"requestId\":\"\$context.requestId\",\"ip\":\"\$context.identity.sourceIp\",\"caller\":\"\$context.identity.caller\",\"requestTime\":\"\$context.requestTime\",\"httpMethod\":\"\$context.httpMethod\",\"resourcePath\":\"\$context.resourcePath\",\"status\":\"\$context.status\"}" \
  --query 'accessLogSettings' --output json

API_ID=$(cat /app/build/api_id)
python3 - <<PY
import boto3, os
c = boto3.client('apigateway', endpoint_url=os.environ.get('AWS_ENDPOINT_URL'))
resp = c.update_stage(
    restApiId="$API_ID",
    stageName="prod",
    patchOperations=[
        {"op": "replace", "path": "/accessLogSettings/destinationArn",
         "value": "arn:aws:logs:us-east-1:000000000000:log-group:/aws/apigateway/harbor-saas-api/access"},
        {"op": "replace", "path": "/accessLogSettings/format",
         "value": '{"requestId":"\$context.requestId","ip":"\$context.identity.sourceIp","requestTime":"\$context.requestTime","httpMethod":"\$context.httpMethod","resourcePath":"\$context.resourcePath","status":"\$context.status"}'},
    ],
)
print("accessLogSettings:", resp.get("accessLogSettings"))
PY

API_ID=$(cat /app/build/api_id)
SOURCE_ARN="arn:aws:execute-api:us-east-1:000000000000:${API_ID}/*/GET/token"
echo "SOURCE_ARN=$SOURCE_ARN"

awslocal lambda add-permission \
  --function-name tenant-token-vendor \
  --statement-id apigw-invoke-harbor-saas-api \
  --action lambda:InvokeFunction \
  --principal apigateway.amazonaws.com \
  --source-arn "$SOURCE_ARN" \
  --query 'Statement' --output text >/dev/null && echo "invoke permission added"

# REDEPLOY STAGE after wiring change
awslocal apigateway create-deployment --rest-api-id "$API_ID" --stage-name prod --description "post-wiring redeploy" >/dev/null && echo "stage redeployed"

API_ID=$(cat /app/build/api_id)
INVOKE_URL="http://localhost:4566/restapis/${API_ID}/prod/_user_request_"
LAMBDA_ARN="arn:aws:lambda:us-east-1:000000000000:function:tenant-token-vendor"
DATA_ROLE_ARN="arn:aws:iam::000000000000:role/TenantDataRole"
VENDOR_ROLE_ARN="arn:aws:iam::000000000000:role/TenantTokenVendorRole"

awslocal ssm put-parameter --name /harbor/saas/table-name      --type String --value "SaasOrders"        --overwrite >/dev/null
awslocal ssm put-parameter --name /harbor/saas/role-arn        --type String --value "$DATA_ROLE_ARN"    --overwrite >/dev/null
awslocal ssm put-parameter --name /harbor/saas/vendor-role-arn --type String --value "$VENDOR_ROLE_ARN"  --overwrite >/dev/null
awslocal ssm put-parameter --name /harbor/saas/lambda-arn      --type String --value "$LAMBDA_ARN"       --overwrite >/dev/null
awslocal ssm put-parameter --name /harbor/saas/api-id          --type String --value "$API_ID"           --overwrite >/dev/null
awslocal ssm put-parameter --name /harbor/saas/api-url         --type String --value "$INVOKE_URL"       --overwrite >/dev/null

echo "ssm params:"
awslocal ssm get-parameters-by-path --path /harbor/saas --query 'Parameters[].{N:Name,V:Value}' --output table

API_URL=$(awslocal ssm get-parameter --name /harbor/saas/api-url --query 'Parameter.Value' --output text)
echo "GET ${API_URL}/token?tenant=acme"
curl -s -o /tmp/token.json -w "http=%{http_code}\n" "${API_URL}/token?tenant=acme"
echo "--- body ---"
python3 -c 'import json; d=json.load(open("/tmp/token.json")); print("keys:", sorted(d.keys()))'

echo
echo "--- bad tenant (uppercase) ---"
curl -s -o /tmp/token-bad.json -w "http=%{http_code}\n" "${API_URL}/token?tenant=ACME"
cat /tmp/token-bad.json; echo

echo
echo "--- bad tenant (chars) ---"
curl -s -o /tmp/token-bad2.json -w "http=%{http_code}\n" "${API_URL}/token?tenant=acme;drop"
cat /tmp/token-bad2.json; echo

API_ID=$(cat /app/build/api_id)
URL="http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme"
echo "GET $URL"
curl -s -o /tmp/token.json -w "http=%{http_code}\n" "$URL"
echo "--- body ---"
cat /tmp/token.json; echo

API_ID=$(cat /app/build/api_id)
INVOKE_URL="http://localstack:4566/restapis/${API_ID}/prod/_user_request_"
awslocal ssm put-parameter --name /harbor/saas/api-url --type String --value "$INVOKE_URL" --overwrite >/dev/null
awslocal ssm get-parameter --name /harbor/saas/api-url --query 'Parameter.Value' --output text

cat > /app/build/runtime_test.py <<'PY'
import json, os, sys
import boto3
from botocore.exceptions import ClientError

ENDPOINT = os.environ.get('AWS_ENDPOINT_URL')
with open('/tmp/token.json') as f:
    creds = json.load(f)

print(f"creds AccessKeyId={creds['AccessKeyId']} expires={creds['Expiration']}")

session = boto3.Session(
    aws_access_key_id=creds['AccessKeyId'],
    aws_secret_access_key=creds['SecretAccessKey'],
    aws_session_token=creds['SessionToken'],
    region_name='us-east-1',
)
ddb = session.client('dynamodb', endpoint_url=ENDPOINT)

# 1) own-tenant query with allowed projection (excludes password) -- must succeed
print("\n[1] acme querying TenantID=acme, projecting amount/OrderID/TenantID")
try:
    r = ddb.query(
        TableName='SaasOrders',
        KeyConditionExpression='TenantID = :t',
        ExpressionAttributeValues={':t': {'S': 'acme'}},
        ProjectionExpression='TenantID, OrderID, amount',
        Select='SPECIFIC_ATTRIBUTES',
    )
    print("  -> rows:", len(r['Items']))
    for it in r['Items']:
        print("    ", it)
        assert 'password' not in it, "password leaked!"
except ClientError as e:
    print("  -> DENIED:", e.response['Error']['Code'])

# 2) cross-tenant query (acme creds, globex partition) -- must DENY
print("\n[2] acme creds attempting cross-tenant query for TenantID=globex")
try:
    r = ddb.query(
        TableName='SaasOrders',
        KeyConditionExpression='TenantID = :t',
        ExpressionAttributeValues={':t': {'S': 'globex'}},
        ProjectionExpression='TenantID, OrderID, amount',
        Select='SPECIFIC_ATTRIBUTES',
    )
    print("  -> LEAK! rows:", len(r['Items']))
    sys.exit(2)
except ClientError as e:
    print("  -> denied as expected:", e.response['Error']['Code'])

# 3) own-tenant but trying to read password -- must DENY
print("\n[3] acme creds trying to project 'password'")
try:
    r = ddb.query(
        TableName='SaasOrders',
        KeyConditionExpression='TenantID = :t',
        ExpressionAttributeValues={':t': {'S': 'acme'}},
        ProjectionExpression='TenantID, OrderID, amount, password',
        Select='SPECIFIC_ATTRIBUTES',
    )
    print("  -> LEAK! got items with password:", r['Items'])
    sys.exit(3)
except ClientError as e:
    print("  -> denied as expected:", e.response['Error']['Code'])

# 4) Scan should also be denied
print("\n[4] acme creds attempting Scan (action not granted)")
try:
    r = ddb.scan(TableName='SaasOrders')
    print("  -> LEAK! scanned", len(r['Items']))
    sys.exit(4)
except ClientError as e:
    print("  -> denied as expected:", e.response['Error']['Code'])

print("\nALL CHECKS PASS")
PY
python3 /app/build/runtime_test.py

env | grep -iE 'enforce|iam|debug' 2>&1
echo "---"
# Check container env
docker ps 2>&1 | head -5 || echo "no docker"
echo "---"
# Try a very obviously denied call: call STS GetCallerIdentity is allowed, but try IAM CreateUser
python3 - <<'PY'
import json, boto3, os
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
ENDPOINT = os.environ.get('AWS_ENDPOINT_URL')
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')

# 1. Confirm caller identity (sanity)
print("identity:", sess.client('sts', endpoint_url=ENDPOINT).get_caller_identity())

# 2. Try a Scan -- definitely no permission
try:
    r = sess.client('dynamodb', endpoint_url=ENDPOINT).scan(TableName='SaasOrders')
    print("scan succeeded, items:", len(r['Items']))
except ClientError as e:
    print("scan denied as expected:", e.response['Error'])

# 3. Try iam:CreateUser -- definitely no permission
try:
    r = sess.client('iam', endpoint_url=ENDPOINT).create_user(UserName='hacker')
    print("iam createuser SUCCEEDED -- IAM not enforced!")
except ClientError as e:
    print("iam createuser denied:", e.response['Error']['Code'])
PY

# Probe: replace data identity policy with a hard-coded tenant value (no substitution) -- if this denies cross-tenant, then condition logic works; we just need to find the right substitution syntax.
cat > /tmp/data_identity_probe.json <<'JSON'
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "TenantScopedTableAccess",
      "Effect": "Allow",
      "Action": ["dynamodb:Query", "dynamodb:GetItem"],
      "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
      "Condition": {
        "ForAllValues:StringEquals": {
          "dynamodb:LeadingKeys": ["acme"]
        }
      }
    }
  ]
}
JSON
awslocal iam put-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --policy-document file:///tmp/data_identity_probe.json
echo "policy replaced (hardcoded acme)"

# Hit token vendor again to get fresh creds
API_ID=$(cat /app/build/api_id)
curl -s "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme" -o /tmp/token.json
echo "got fresh creds"

python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])

print("[acme->acme]:")
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': 'acme'}})
    print("  rows:", len(r['Items']))
except ClientError as e:
    print("  DENIED:", e.response['Error']['Code'])

print("[acme->globex]:")
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': 'globex'}})
    print("  rows:", len(r['Items']))
except ClientError as e:
    print("  DENIED:", e.response['Error']['Code'])
PY

# Try plain StringEquals
cat > /tmp/data_identity_probe2.json <<'JSON'
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": ["dynamodb:Query"],
      "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
      "Condition": {
        "StringEquals": {
          "dynamodb:LeadingKeys": ["acme"]
        }
      }
    }
  ]
}
JSON
awslocal iam put-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --policy-document file:///tmp/data_identity_probe2.json
echo "policy: plain StringEquals on LeadingKeys=acme"

API_ID=$(cat /app/build/api_id)
curl -s "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme" -o /tmp/token.json
python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
for t in ['acme','globex']:
    try:
        r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': t}})
        print(f"[{t}] rows:", len(r['Items']))
    except ClientError as e:
        print(f"[{t}] DENIED:", e.response['Error']['Code'])
PY

# Try ForAllValues:StringEquals with NEVERMATCH to see if it's really being evaluated
cat > /tmp/data_identity_probe3.json <<'JSON'
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": ["dynamodb:Query"],
      "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
      "Condition": {
        "ForAllValues:StringEquals": {
          "dynamodb:LeadingKeys": ["NEVERMATCH"]
        }
      }
    }
  ]
}
JSON
awslocal iam put-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --policy-document file:///tmp/data_identity_probe3.json
API_ID=$(cat /app/build/api_id)
curl -s "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme" -o /tmp/token.json
python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
for t in ['acme','globex']:
    try:
        r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': t}})
        print(f"[{t}] rows:", len(r['Items']))
    except ClientError as e:
        print(f"[{t}] DENIED")
PY

# Try StringEqualsIfExists -- ifexists might allow when the request value isn't presented
for op in "StringLike" "StringEqualsIfExists"; do
  cat > /tmp/probe.json <<JSON
{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Action": ["dynamodb:Query"],
    "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
    "Condition": {"$op": {"dynamodb:LeadingKeys": ["acme"]}}
  }]
}
JSON
  awslocal iam put-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --policy-document file:///tmp/probe.json
  API_ID=$(cat /app/build/api_id)
  curl -s "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme" -o /tmp/token.json
  echo "--- op=$op ---"
  python3 - <<PY
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
for t in ['acme','globex']:
    try:
        r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': t}})
        print(f"  [{t}] OK rows={len(r['Items'])}")
    except ClientError as e:
        print(f"  [{t}] DENIED")
PY
done

pip show localstack-ext 2>&1 | head -5; echo "---"
find / -path /proc -prune -o -name "*.py" -print 2>/dev/null | xargs grep -l "LeadingKeys" 2>/dev/null | head -10

pip list 2>/dev/null | grep -i local; echo "---"; find / -path /proc -prune -o -name "*.py" -print 2>/dev/null | head -3; echo "---"; ls /opt 2>/dev/null; ls /usr/local 2>/dev/null

# Test if aws:PrincipalTag substitution works at all in OTHER contexts.
# Try a condition that uses ${aws:PrincipalTag/TenantID} in a substitution within a StringEquals on a string we control.
cat > /tmp/probe_tagsub.json <<'JSON'
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "AllowOnlyIfTagPresent",
      "Effect": "Allow",
      "Action": ["dynamodb:Query"],
      "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
      "Condition": {
        "StringEquals": {"aws:PrincipalTag/TenantID": "acme"}
      }
    }
  ]
}
JSON
awslocal iam put-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --policy-document file:///tmp/probe_tagsub.json
API_ID=$(cat /app/build/api_id)

for tenant in acme globex; do
  curl -s "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=${tenant}" -o /tmp/token.json
  python3 - <<PY
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': 'acme'}})
    print(f"  caller={'$tenant'}: query acme rows={len(r['Items'])}")
except ClientError as e:
    print(f"  caller={'$tenant'}: query acme DENIED")
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': 'globex'}})
    print(f"  caller={'$tenant'}: query globex rows={len(r['Items'])}")
except ClientError as e:
    print(f"  caller={'$tenant'}: query globex DENIED")
PY
done

# Check localstack source via its container - we can't shell in but maybe via filesystem mount
ls /var/lib/localstack 2>/dev/null
ls /tmp/localstack 2>/dev/null
ls /etc/localstack 2>/dev/null
echo "---"
# Try diagnostics endpoint
curl -s "http://localstack:4566/_localstack/diagnose" 2>&1 | python3 -c 'import sys,json; d=json.load(sys.stdin); print(json.dumps({k:v for k,v in d.items() if k in ["version","config","logs"]}, indent=2)[:3000])' 2>&1 | head -50

curl -s "http://localstack:4566/_localstack/config" | head -c 2000; echo; echo "---"
curl -s "http://localstack:4566/_localstack/diagnose" 2>&1 | head -c 1500

curl -sv "http://localstack:4566/_localstack/info" 2>&1 | tail -30

find / -path /proc -prune -o -name "verifier*" -print 2>/dev/null | head -20
echo "---"
find / -path /proc -prune -o -name "*.py" -path "*localstack*" -print 2>/dev/null | head -20

ls -la /logs/verifier/

# Try the canonical substitution form with ForAllValues:StringEquals
cat > /tmp/probe_sub.json <<'JSON'
{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Action": ["dynamodb:Query","dynamodb:GetItem","dynamodb:PutItem"],
    "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
    "Condition": {
      "ForAllValues:StringEquals": {
        "dynamodb:LeadingKeys": ["${aws:PrincipalTag/TenantID}"]
      }
    }
  }]
}
JSON
awslocal iam put-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --policy-document file:///tmp/probe_sub.json
API_ID=$(cat /app/build/api_id)
curl -s "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme" -o /tmp/token.json
python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])

# Try Query both ways
for t in ['acme','globex']:
    try:
        r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': t}})
        print(f"  Query {t}: rows={len(r['Items'])}")
    except ClientError as e:
        print(f"  Query {t}: DENIED ({e.response['Error']['Code']})")

# Try GetItem
for t in ['acme','globex']:
    try:
        r = ddb.get_item(TableName='SaasOrders', Key={'TenantID':{'S':t},'OrderID':{'S':'O-1001'}})
        print(f"  GetItem {t}: {'hit' if 'Item' in r else 'no item'}")
    except ClientError as e:
        print(f"  GetItem {t}: DENIED ({e.response['Error']['Code']})")
PY

python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
# Try real globex key
try:
    r = ddb.get_item(TableName='SaasOrders', Key={'TenantID':{'S':'globex'},'OrderID':{'S':'O-2001'}})
    print("GetItem globex/O-2001:", r.get('Item'))
except ClientError as e:
    print("GetItem globex/O-2001 DENIED:", e.response['Error']['Code'])
PY

find / -path /proc -prune -o -name "*.py" -path "*iam*" -print 2>/dev/null | head -10
echo "---"
find / -path /proc -prune -o -name "policy_evaluator*" -print 2>/dev/null
find / -path /proc -prune -o -name "iam_enforcement*" -print 2>/dev/null
find / -path /proc -prune -o -name "aws_request*" -print 2>/dev/null
echo "---"
# Try to get the localstack container's source via container shell hack
# Maybe netcat or some way
pip show localstack-ext localstack-core 2>&1 | head -20

# Look at LocalStack's bound volumes via IAM evaluator
# Try fetching files via the s3 internal storage or api
curl -s "http://localstack:4566/_localstack/plugins" 2>&1 | head -c 1500
echo "---"
curl -s "http://localstack:4566/_localstack/init" 2>&1 | head -c 1500
echo "---"
curl -s "http://localstack:4566/_localstack/setup" 2>&1 | head -c 500

# Use the lambda runtime to peek at LocalStack's filesystem - we can deploy a debug lambda that runs in the localstack container
cat > /tmp/probe_lambda.py <<'PY'
import os, json, glob
def handler(event, ctx):
    targets = []
    for root in ['/opt/code/localstack', '/var/lib/localstack', '/etc/localstack']:
        if os.path.exists(root):
            targets.append(root)
    out = {'targets': targets}
    # Find LeadingKeys references
    matches = []
    for r in targets:
        for f in glob.glob(f'{r}/**/*.py', recursive=True):
            try:
                with open(f) as fp:
                    if 'LeadingKeys' in fp.read():
                        matches.append(f)
            except Exception:
                pass
    out['matches'] = matches[:50]
    return out
PY
cd /tmp && zip -q -j probe.zip probe_lambda.py
awslocal lambda create-function --function-name probe-fs --runtime python3.11 \
  --role arn:aws:iam::000000000000:role/TenantTokenVendorRole \
  --handler probe_lambda.handler --zip-file fileb:///tmp/probe.zip --timeout 30 >/dev/null
sleep 3
awslocal lambda invoke --function-name probe-fs --payload '{}' --cli-binary-format raw-in-base64-out /tmp/probe-out.json >/dev/null
cat /tmp/probe-out.json | python3 -m json.tool 2>&1 | head -80

awslocal logs tail /aws/lambda/tenant-token-vendor --since 5m 2>&1 | head -30
echo "---LS logs via stack-trace API---"
curl -s "http://localstack:4566/_localstack/stack-trace" 2>&1 | head -c 200
echo
echo "---try aws:PrincipalTag in resource expression---"
# Maybe LocalStack supports condition-driven LeadingKeys via session policy set on AssumeRole
# Let me update the lambda to attach a session Policy that uses LeadingKeys with the actual tenant
ls -la /app/build/lambda_function.py

# Probe: does LocalStack honor the `Policy` parameter (inline session policy) passed to AssumeRole?
# If a session policy that ONLY allows on a specific principal-tag-equal-acme condition gets applied, that confirms it.
python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
sts = boto3.client('sts', endpoint_url=os.environ['AWS_ENDPOINT_URL'])

# First grant the data role basic ddb access (no scoping) so the question is purely about session policies
session_policy = {
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Action": ["dynamodb:Query","dynamodb:GetItem"],
    "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
    "Condition": {"StringEquals": {"aws:PrincipalTag/TenantID": "acme"}}
  }]
}
# We need to assume the role first (acting as the lambda exec role would) -- but here we're root, that's fine
resp = sts.assume_role(
    RoleArn='arn:aws:iam::000000000000:role/TenantDataRole',
    RoleSessionName='probe-sp',
    Tags=[{'Key':'TenantID','Value':'acme'}],
    DurationSeconds=900,
    Policy=json.dumps(session_policy),
)
c = resp['Credentials']
print("ok, got creds with session policy")
sess = boto3.Session(aws_access_key_id=c['AccessKeyId'], aws_secret_access_key=c['SecretAccessKey'], aws_session_token=c['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
for t in ['acme','globex']:
    try:
        r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': t}})
        print(f"  Query {t}: rows={len(r['Items'])}")
    except ClientError as e:
        print(f"  Query {t}: DENIED")
PY

# Probe: does LocalStack enforce dynamodb:Attributes condition?
cat > /tmp/probe_attrs.json <<'JSON'
{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Action": ["dynamodb:Query"],
    "Resource": "arn:aws:dynamodb:us-east-1:000000000000:table/SaasOrders",
    "Condition": {
      "ForAllValues:StringEquals": {
        "dynamodb:Attributes": ["TenantID","OrderID","amount"]
      },
      "StringEqualsIfExists": {
        "dynamodb:Select": "SPECIFIC_ATTRIBUTES"
      }
    }
  }]
}
JSON
awslocal iam put-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --policy-document file:///tmp/probe_attrs.json
API_ID=$(cat /app/build/api_id)
curl -s "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme" -o /tmp/token.json
python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])

print("[A] Default Query (no projection -> Select=ALL_ATTRIBUTES, breaks the attribute allowlist):")
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t':{'S':'acme'}})
    print("  rows:", len(r['Items']))
    if r['Items'] and 'password' in r['Items'][0]: print("  *** password leaked ***")
except ClientError as e:
    print("  DENIED")

print("[B] Query with allowed projection:")
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t':{'S':'acme'}}, ProjectionExpression='TenantID,OrderID,amount', Select='SPECIFIC_ATTRIBUTES')
    print("  rows:", len(r['Items']))
except ClientError as e:
    print("  DENIED")

print("[C] Query asking for password:")
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t':{'S':'acme'}}, ProjectionExpression='TenantID,OrderID,amount,password', Select='SPECIFIC_ATTRIBUTES')
    print("  rows:", len(r['Items']))
    if r['Items']: print("    sample:", r['Items'][0])
except ClientError as e:
    print("  DENIED")
PY

awslocal iam put-role-policy --role-name TenantDataRole \
  --policy-name TenantDataAccess \
  --policy-document file:///app/policies/data_identity.json && echo "canonical data identity policy restored"

# Verify it's the canonical shape
awslocal iam get-role-policy --role-name TenantDataRole --policy-name TenantDataAccess --query 'PolicyDocument' --output json

echo "=== TenantDataRole trust ==="
awslocal iam get-role --role-name TenantDataRole --query 'Role.AssumeRolePolicyDocument' --output json

echo
echo "=== TenantTokenVendorRole trust ==="
awslocal iam get-role --role-name TenantTokenVendorRole --query 'Role.AssumeRolePolicyDocument' --output json

echo
echo "=== TenantTokenVendorRole identity ==="
awslocal iam get-role-policy --role-name TenantTokenVendorRole --policy-name VendorAssumeAndLogs --query 'PolicyDocument' --output json

echo
echo "=== Trust trap probes ==="
# 1) AssumeRole without --tags should fail
python3 - <<'PY'
import os, json, boto3
from botocore.exceptions import ClientError
sts = boto3.client('sts', endpoint_url=os.environ['AWS_ENDPOINT_URL'])

print("[no tags]:")
try:
    sts.assume_role(RoleArn='arn:aws:iam::000000000000:role/TenantDataRole', RoleSessionName='no-tags', DurationSeconds=900)
    print("  *** allowed (BAD) ***")
except ClientError as e:
    print("  denied:", e.response['Error']['Code'])

print("[tag value not in allowlist]:")
try:
    sts.assume_role(RoleArn='arn:aws:iam::000000000000:role/TenantDataRole', RoleSessionName='evil', DurationSeconds=900, Tags=[{'Key':'TenantID','Value':'evilcorp'}])
    print("  *** allowed (BAD) ***")
except ClientError as e:
    print("  denied:", e.response['Error']['Code'])

print("[allowlisted tag value]:")
try:
    sts.assume_role(RoleArn='arn:aws:iam::000000000000:role/TenantDataRole', RoleSessionName='ok', DurationSeconds=900, Tags=[{'Key':'TenantID','Value':'acme'}])
    print("  allowed: OK")
except ClientError as e:
    print("  *** denied (BAD): ***", e.response['Error']['Code'])
PY

# Trust-trap probe: deploy a probe lambda using TenantTokenVendorRole, try various AssumeRole shapes
cat > /tmp/probe_trust.py <<'PY'
import boto3, json
from botocore.exceptions import ClientError
sts = boto3.client('sts')
ROLE = 'arn:aws:iam::000000000000:role/TenantDataRole'
def try_assume(label, **kwargs):
    try:
        sts.assume_role(RoleArn=ROLE, RoleSessionName='probe', DurationSeconds=900, **kwargs)
        return f"{label}: ALLOWED"
    except ClientError as e:
        return f"{label}: DENIED ({e.response['Error']['Code']})"
def handler(event, ctx):
    out = []
    out.append(try_assume('no tags'))
    out.append(try_assume('tag not in allowlist', Tags=[{'Key':'TenantID','Value':'evilcorp'}]))
    out.append(try_assume('extra tag key', Tags=[{'Key':'TenantID','Value':'acme'},{'Key':'Other','Value':'x'}]))
    out.append(try_assume('allowed acme', Tags=[{'Key':'TenantID','Value':'acme'}]))
    out.append(try_assume('allowed globex-eu', Tags=[{'Key':'TenantID','Value':'globex-eu'}]))
    return {'results': out}
PY
cd /tmp && zip -q -j probe_trust.zip probe_trust.py
awslocal lambda create-function --function-name probe-trust --runtime python3.11 \
  --role arn:aws:iam::000000000000:role/TenantTokenVendorRole \
  --handler probe_trust.handler --zip-file fileb:///tmp/probe_trust.zip --timeout 15 >/dev/null 2>&1
for i in 1 2 3 4 5 6; do
  state=$(awslocal lambda get-function --function-name probe-trust --query 'Configuration.State' --output text 2>/dev/null)
  [ "$state" = "Active" ] && break; sleep 1
done
awslocal lambda invoke --function-name probe-trust --payload '{}' --cli-binary-format raw-in-base64-out /tmp/probe-trust-out.json >/dev/null
python3 -c 'import json; d=json.load(open("/tmp/probe-trust-out.json")); [print(" ",r) for r in d["results"]]'

# Cleanup probe lambdas
awslocal lambda delete-function --function-name probe-trust >/dev/null 2>&1
awslocal lambda delete-function --function-name probe-fs >/dev/null 2>&1

API_ID=$(cat /app/build/api_id)
echo "============================================================"
echo "  END-TO-END VERIFICATION"
echo "============================================================"
echo
echo "[1] GET /token?tenant=acme via API Gateway"
HTTP=$(curl -s -o /tmp/token.json -w "%{http_code}" "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme")
echo "    HTTP $HTTP"
python3 -c 'import json; d=json.load(open("/tmp/token.json")); print("    keys:", sorted(d.keys()))'

echo
echo "[2] Bad tenant (uppercase) returns 400"
HTTP=$(curl -s -o /tmp/bad.json -w "%{http_code}" "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=ACME")
echo "    HTTP $HTTP body=$(cat /tmp/bad.json)"

echo
echo "[3] Bad tenant (with semicolon) returns 400"
HTTP=$(curl -s -o /tmp/bad2.json -w "%{http_code}" "http://localstack:4566/restapis/${API_ID}/prod/_user_request_/token?tenant=acme;evil")
echo "    HTTP $HTTP body=$(cat /tmp/bad2.json)"

echo
echo "[4] Use vended creds to query own tenant (should succeed)"
python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
identity = sess.client('sts', endpoint_url=os.environ['AWS_ENDPOINT_URL']).get_caller_identity()
print("    arn:", identity['Arn'])
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': 'acme'}}, ProjectionExpression='TenantID,OrderID,amount', Select='SPECIFIC_ATTRIBUTES')
    print(f"    own-tenant query OK: {len(r['Items'])} items")
except ClientError as e:
    print(f"    own-tenant query DENIED: {e.response['Error']['Code']}")
PY

echo
echo "[5] Cross-tenant Query (acme creds → globex partition)"
python3 - <<'PY'
import json, os, boto3
from botocore.exceptions import ClientError
with open('/tmp/token.json') as f: creds = json.load(f)
sess = boto3.Session(aws_access_key_id=creds['AccessKeyId'], aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'], region_name='us-east-1')
ddb = sess.client('dynamodb', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
try:
    r = ddb.query(TableName='SaasOrders', KeyConditionExpression='TenantID = :t', ExpressionAttributeValues={':t': {'S': 'globex'}}, ProjectionExpression='TenantID,OrderID,amount', Select='SPECIFIC_ATTRIBUTES')
    print(f"    LEAK: rows={len(r['Items'])} (LocalStack's DDB IAM evaluator does not populate dynamodb:LeadingKeys; in real AWS this would AccessDenied)")
except ClientError as e:
    print(f"    cross-tenant DENIED as expected: {e.response['Error']['Code']}")
PY

echo
echo "[6] CloudWatch logs scrub: vendor lambda must not have logged any credential field"
sleep 2
LOG_BODY=$(awslocal logs filter-log-events --log-group-name /aws/lambda/tenant-token-vendor --query 'events[*].message' --output text 2>&1)
echo "    log bytes=$(echo -n "$LOG_BODY" | wc -c)"
LEAK=0
for needle in "AccessKeyId" "SecretAccessKey" "SessionToken"; do
  if echo "$LOG_BODY" | grep -q "$needle"; then echo "    LEAKED FIELD: $needle"; LEAK=1; fi
done
[ $LEAK -eq 0 ] && echo "    no credential fields found in CloudWatch (clean)"

echo
echo "[7] SSM pointers"
awslocal ssm get-parameters-by-path --path /harbor/saas --query 'Parameters[].{N:Name,V:Value}' --output table