tests/test.sh exit code (0 → resolved); the classification below is post-hoc and cannot change it.No file changes, the agent left the workspace untouched.
downloading uv 0.9.7 aarch64-unknown-linux-gnu
Downloading botocore (12.7MiB)
Downloading pygments (1.2MiB)
Downloading pygments
Downloading botocore
Installed 13 packages in 35ms
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0 -- /root/.cache/uv/archive-v0/n-s162B1cOUVjM3gPMUjk/bin/python
cachedir: .pytest_cache
rootdir: /tests
plugins: json-ctrf-0.3.5
collecting ... collected 27 items
../tests/test_outputs.py::test_01_analyst_user_exists PASSED [ 3%]
../tests/test_outputs.py::test_02_identity_policy_attached_to_user PASSED [ 7%]
../tests/test_outputs.py::test_03_boundary_attached_via_get_user FAILED [ 11%]
../tests/test_outputs.py::test_04_vault_bucket_exists_with_public_access_block PASSED [ 14%]
../tests/test_outputs.py::test_05_kms_alias_and_ssm_pointers_resolve FAILED [ 18%]
../tests/test_outputs.py::test_06_identity_policy_grants_full_s3_and_full_kms PASSED [ 22%]
../tests/test_outputs.py::test_07_identity_policy_has_no_explicit_deny_anywhere FAILED [ 25%]
../tests/test_outputs.py::test_08_user_has_no_admin_managed_policy_attached PASSED [ 29%]
../tests/test_outputs.py::test_09_boundary_effect_allow_only FAILED [ 33%]
../tests/test_outputs.py::test_10_boundary_has_no_NotResource_or_NotAction FAILED [ 37%]
../tests/test_outputs.py::test_11_boundary_object_actions_scoped_to_confidential_prefix FAILED [ 40%]
../tests/test_outputs.py::test_12_boundary_listbucket_uses_s3_prefix_condition FAILED [ 44%]
../tests/test_outputs.py::test_13_boundary_kms_actions_scoped_to_cmk_arn PASSED [ 48%]
../tests/test_outputs.py::test_14_boundary_does_not_grant_kms_create_key_or_wildcard_kms FAILED [ 51%]
../tests/test_outputs.py::test_15_boundary_does_not_grant_iam_self_modify PASSED [ 55%]
../tests/test_outputs.py::test_16_boundary_size_under_managed_policy_limit PASSED [ 59%]
../tests/test_outputs.py::test_17_boundary_allows_confidential_put_via_simulator FAILED [ 62%]
../tests/test_outputs.py::test_18_boundary_denies_public_put_via_simulator PASSED [ 66%]
../tests/test_outputs.py::test_19_boundary_allows_kms_decrypt_via_simulator PASSED [ 70%]
../tests/test_outputs.py::test_20_boundary_denies_kms_create_key_via_simulator PASSED [ 74%]
../tests/test_outputs.py::test_21_analyst_can_put_to_confidential_prefix PASSED [ 77%]
../tests/test_outputs.py::test_22_analyst_cannot_put_to_public_prefix PASSED [ 81%]
../tests/test_outputs.py::test_23_analyst_cannot_create_kms_key FAILED [ 85%]
../tests/test_outputs.py::test_24_analyst_cannot_put_to_a_different_bucket FAILED [ 88%]
../tests/test_outputs.py::test_25_boundary_object_resource_prefix_is_lowercase_confidential FAILED [ 92%]
../tests/test_outputs.py::test_26_ssm_policy_arn_resolves_to_a_live_boundary_managed_policy FAILED [ 96%]
../tests/test_outputs.py::test_27_boundary_default_version_reflects_the_runtime_intended_shape FAILED [100%]
=================================== FAILURES ===================================
____________________ test_03_boundary_attached_via_get_user ____________________
def test_03_boundary_attached_via_get_user():
user = _get_user()
pb = user.get("PermissionsBoundary") or {}
arn = pb.get("PermissionsBoundaryArn")
> assert arn == BOUNDARY_ARN_EXPECTED, (
f"GetUser.User.PermissionsBoundary.PermissionsBoundaryArn={arn!r}; "
f"expected {BOUNDARY_ARN_EXPECTED!r} , the boundary managed policy is defined "
f"but not actually attached to the user as a permissions boundary"
)
E AssertionError: GetUser.User.PermissionsBoundary.PermissionsBoundaryArn=None; expected 'arn:aws:iam::000000000000:policy/AnalystBoundary' , the boundary managed policy is defined but not actually attached to the user as a permissions boundary
E assert None == 'arn:aws:iam::000000000000:policy/AnalystBoundary'
/tests/test_outputs.py:181: AssertionError
__________________ test_05_kms_alias_and_ssm_pointers_resolve __________________
def test_05_kms_alias_and_ssm_pointers_resolve():
kms = _client("kms")
info = kms.describe_key(KeyId="alias/vault-kms")["KeyMetadata"]
assert info["Arn"].startswith(f"arn:aws:kms:{REGION}:{ACCOUNT_ID}:key/")
> assert _ssm(SSM_BOUNDARY_ARN) == BOUNDARY_ARN_EXPECTED
E AssertionError: assert 'arn:aws:iam:...oundaryPolicy' == 'arn:aws:iam:...alystBoundary'
E
E - arn:aws:iam::000000000000:policy/AnalystBoundary
E + arn:aws:iam::000000000000:policy/AnalystBoundaryPolicy
E ? ++++++
/tests/test_outputs.py:200: AssertionError
____________ test_07_identity_policy_has_no_explicit_deny_anywhere _____________
def test_07_identity_policy_has_no_explicit_deny_anywhere():
for s in _stmts(_identity_doc()):
> assert s.get("Effect") != "Deny", (
f"identity policy contains an Effect:Deny statement {s!r} , the design requires the boundary "
f"alone to do the gating; an explicit Deny in the identity policy means the boundary's "
f"semantics aren't being exercised, and the auditor flags the design as broken"
)
E AssertionError: identity policy contains an Effect:Deny statement {'Sid': 'AnalystGuard', 'Effect': 'Deny', 'Action': 's3:PutObject', 'Resource': 'arn:aws:s3:::vault-bucket/public/*'} , the design requires the boundary alone to do the gating; an explicit Deny in the identity policy means the boundary's semantics aren't being exercised, and the auditor flags the design as broken
E assert 'Deny' != 'Deny'
E + where 'Deny' = <built-in method get of dict object at 0xffff96020bc0>('Effect')
E + where <built-in method get of dict object at 0xffff96020bc0> = {'Action': 's3:PutObject', 'Effect': 'Deny', 'Resource': 'arn:aws:s3:::vault-bucket/public/*', 'Sid': 'AnalystGuard'}.get
/tests/test_outputs.py:228: AssertionError
______________________ test_09_boundary_effect_allow_only ______________________
def test_09_boundary_effect_allow_only():
for s in _stmts(_boundary_doc()):
eff = s.get("Effect")
> assert eff == "Allow", (
f"boundary has a non-Allow statement: {s!r} , a permissions boundary is a CEILING, "
f"not a deny list; anything not listed is implicitly denied by the boundary itself, "
f"and the auditor rejects an explicit Deny in the boundary as a design smell"
)
E AssertionError: boundary has a non-Allow statement: {'Sid': 'PublicGuard', 'Effect': 'Deny', 'Action': 's3:*', 'Resource': 'arn:aws:s3:::vault-bucket/public/*'} , a permissions boundary is a CEILING, not a deny list; anything not listed is implicitly denied by the boundary itself, and the auditor rejects an explicit Deny in the boundary as a design smell
E assert 'Deny' == 'Allow'
E
E - Allow
E + Deny
/tests/test_outputs.py:251: AssertionError
_______________ test_10_boundary_has_no_NotResource_or_NotAction _______________
def test_10_boundary_has_no_NotResource_or_NotAction():
for s in _stmts(_boundary_doc()):
> assert "NotResource" not in s, (
f"boundary statement uses NotResource: {s!r} , NotResource does not invert into an allow-elsewhere, "
f"it inverts the resource match in a way that's almost never what the author meant; the auditor "
f"forbids it in boundary documents"
)
E AssertionError: boundary statement uses NotResource: {'Sid': 'BucketListing', 'Effect': 'Allow', 'Action': ['s3:ListBucket', 's3:GetBucketLocation'], 'NotResource': 'arn:aws:s3:::vault-bucket/public/*'} , NotResource does not invert into an allow-elsewhere, it inverts the resource match in a way that's almost never what the author meant; the auditor forbids it in boundary documents
E assert 'NotResource' not in {'Action': ['s3:ListBucket', 's3:GetBucketLocation'], 'Effect': 'Allow', 'NotResource': 'arn:aws:s3:::vault-bucket/public/*', 'Sid': 'BucketListing'}
/tests/test_outputs.py:260: AssertionError
________ test_11_boundary_object_actions_scoped_to_confidential_prefix _________
def test_11_boundary_object_actions_scoped_to_confidential_prefix():
bucket = _bucket()
expected = f"arn:aws:s3:::{bucket}/confidential/*"
object_actions = {"s3:GetObject", "s3:PutObject", "s3:DeleteObject"}
found_any = False
for s in _stmts(_boundary_doc()):
if s.get("Effect") != "Allow":
continue
actions = set(_as_list(s.get("Action")))
if not (actions & object_actions):
continue
found_any = True
resources = _as_list(s.get("Resource"))
assert "*" not in resources, (
f"boundary object-level Allow statement has Resource '*' (no scope): {s!r} , must be narrowed "
f"to {expected!r}"
)
bad = [r for r in resources if r != expected]
> assert not bad, (
f"boundary object-level Allow statement has Resource={resources!r}; expected exactly [{expected!r}]"
)
E AssertionError: boundary object-level Allow statement has Resource=['arn:aws:s3:::vault-bucket/Confidential/*']; expected exactly ['arn:aws:s3:::vault-bucket/confidential/*']
E assert not ['arn:aws:s3:::vault-bucket/Confidential/*']
/tests/test_outputs.py:286: AssertionError
_____________ test_12_boundary_listbucket_uses_s3_prefix_condition _____________
def test_12_boundary_listbucket_uses_s3_prefix_condition():
bucket = _bucket()
bucket_arn = f"arn:aws:s3:::{bucket}"
found = False
for s in _stmts(_boundary_doc()):
if s.get("Effect") != "Allow":
continue
actions = set(_as_list(s.get("Action")))
if "s3:ListBucket" not in actions:
continue
found = True
resources = _as_list(s.get("Resource"))
> assert bucket_arn in resources, (
f"boundary s3:ListBucket Resource={resources!r} does not include the bucket ARN {bucket_arn!r}; "
f"ListBucket is a bucket-level action and must target the bucket ARN, not an object-prefix ARN"
)
E AssertionError: boundary s3:ListBucket Resource=[] does not include the bucket ARN 'arn:aws:s3:::vault-bucket'; ListBucket is a bucket-level action and must target the bucket ARN, not an object-prefix ARN
E assert 'arn:aws:s3:::vault-bucket' in []
/tests/test_outputs.py:307: AssertionError
________ test_14_boundary_does_not_grant_kms_create_key_or_wildcard_kms ________
def test_14_boundary_does_not_grant_kms_create_key_or_wildcard_kms():
actions = _all_actions(_stmts(_boundary_doc()))
assert "kms:CreateKey" not in actions, (
"boundary grants kms:CreateKey , the analyst could create their own keys outside the design"
)
> assert "kms:*" not in actions, (
"boundary grants kms:* , that's a wildcard over every kms action including CreateKey, "
"and the design forbids it; the boundary's kms grants must be limited to the data-key "
"actions needed to use the existing cmk"
)
E AssertionError: boundary grants kms:* , that's a wildcard over every kms action including CreateKey, and the design forbids it; the boundary's kms grants must be limited to the data-key actions needed to use the existing cmk
E assert 'kms:*' not in ['s3:ListBucket', 's3:GetBucketLocation', 's3:GetObject', 's3:PutObject', 's3:DeleteObject', 's3:*', ...]
/tests/test_outputs.py:350: AssertionError
____________ test_17_boundary_allows_confidential_put_via_simulator ____________
def test_17_boundary_allows_confidential_put_via_si
… (truncated at 12,000 chars, full verifier log is in the trial artifacts)Reproduce this trial: git checkout 2f94510 && PYTHONPATH=src python3 scripts/build_site.py , then open trial/trial_78f9a6c81c764b54. Re-running the agent live requires EVAL_PLATFORM_ENABLE_OAUTH_SMOKE=1 and is non-deterministic.
Trial trial_78f9a6c81c764b54 · verifier authoritative; classifier explanatory.