glue-etl-catalog-security-configuration-kms

{
  "command": "echo \"AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL\"; echo \"ENFORCE_IAM=$ENFORCE_IAM\"; aws --version; aws sts get-caller-identity",
  "description": "Check environment and AWS access"
}

{
  "command": "ls /app/",
  "description": "List app directory"
}

{
  "command": "mkdir -p /app/build && cd /app/build && SUFFIX=$(openssl rand -hex 3) && echo \"SUFFIX=$SUFFIX\" > /app/build/vars.sh && cat /app/build/vars.sh",
  "description": "Generate 6-hex suffix"
}

{
  "command": "cat >> /app/build/vars.sh <<'EOF'\nACCOUNT=000000000000\nREGION=us-east-1\nSOURCE_BUCKET=harbor-glue-source-e8c5f0\nTARGET_BUCKET=harbor-glue-target-e8c5f0\nSCRIPTS_BUCKET=harbor-glue-scripts-e8c5f0\nROLE_NAME=harbor-glue-svc-role\nROLE_ARN=arn:aws:iam::000000000000:role/harbor-glue-svc-role\nSEC_CONFIG=harbor-glue-sec-config\nDB_NAME=harbor_etl_db\nTABLE_NAME=orders_raw\nJOB_NAME=harbor-orders-etl\nEOF\ncat /app/build/vars.sh",
  "description": "Persist names"
}

{
  "file_path": "/app/build/trust.json",
  "content": "{\n  \"Version\": \"2012-10-17\",\n  \"Statement\": [\n    {\n      \"Effect\": \"Allow\",\n      \"Principal\": { \"Service\": \"glue.amazonaws.com\" },\n      \"Action\": \"sts:AssumeRole\"\n    }\n  ]\n}\n"
}

{
  "command": ". /app/build/vars.sh && aws iam create-role --role-name \"$ROLE_NAME\" --assume-role-policy-document file:///app/build/trust.json --description \"Glue ETL service role for harbor data platform\" --query 'Role.Arn' --output text",
  "description": "Create IAM service role"
}

{
  "command": ". /app/build/vars.sh && aws iam attach-role-policy --role-name \"$ROLE_NAME\" --policy-arn arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole && echo OK",
  "description": "Attach AWSGlueServiceRole managed policy"
}

{
  "file_path": "/app/build/key-policy.json",
  "content": "{\n  \"Version\": \"2012-10-17\",\n  \"Id\": \"harbor-glue-cmk-policy\",\n  \"Statement\": [\n    {\n      \"Sid\": \"RootAccountAdmin\",\n      \"Effect\": \"Allow\",\n      \"Principal\": { \"AWS\": \"arn:aws:iam::000000000000:root\" },\n      \"Action\": \"kms:*\",\n      \"Resource\": \"*\"\n    },\n    {\n      \"Sid\": \"GlueServiceEnvelopeUse\",\n      \"Effect\": \"Allow\",\n      \"Principal\": { \"Service\": \"glue.amazonaws.com\" },\n      \"Action\": [\n        \"kms:Encrypt\",\n        \"kms:Decrypt\",\n        \"kms:ReEncrypt*\",\n        \"kms:GenerateDataKey*\",\n        \"kms:DescribeKey\",\n        \"kms:CreateGrant\"\n      ],\n      \"Resource\": \"*\",\n      \"Condition\": {\n        \"StringEquals\": {\n          \"kms:ViaService\": \"glue.us-east-1.amazonaws.com\",\n          \"aws:SourceAccount\": \"000000000000\"\n        }\n      }\n    },\n    {\n      \"Sid\": \"GlueRoleS3ScopedUse\",\n      \"Effect\": \"Allow\",\n      \"Principal\": { \"AWS\": \"arn:aws:iam::000000000000:role/harbor-glue-svc-role\" },\n      \"Action\": [\n        \"kms:Encrypt\",\n        \"kms:Decrypt\",\n        \"kms:ReEncrypt*\",\n        \"kms:GenerateDataKey*\",\n        \"kms:DescribeKey\"\n      ],\n      \"Resource\": \"*\",\n      \"Condition\": {\n        \"StringEquals\": {\n          \"aws:SourceAccount\": \"000000000000\"\n        },\n        \"ForAllValues:StringEquals\": {\n          \"kms:EncryptionContext:aws:s3:arn\": [\n            \"arn:aws:s3:::harbor-glue-source-e8c5f0/*\",\n            \"arn:aws:s3:::harbor-glue-target-e8c5f0/*\",\n            \"arn:aws:s3:::harbor-glue-scripts-e8c5f0/*\"\n          ]\n        }\n      }\n    }\n  ]\n}\n"
}

{
  "command": ". /app/build/vars.sh && CMK_OUT=$(aws kms create-key --description \"Harbor Glue ETL CMK , single-key encryption surface for catalog, security config, and S3\" --key-usage ENCRYPT_DECRYPT --key-spec SYMMETRIC_DEFAULT --policy file:///app/build/key-policy.json) && echo \"$CMK_OUT\" | tee /app/build/cmk.json && CMK_ID=$(echo \"$CMK_OUT\" | python3 -c 'import json,sys;print(json.load(sys.stdin)[\"KeyMetadata\"][\"KeyId\"])') && CMK_ARN=$(echo \"$CMK_OUT\" | python3 -c 'import json,sys;print(json.load(sys.stdin)[\"KeyMetadata\"][\"Arn\"])') && echo \"CMK_ID=$CMK_ID\" >> /app/build/vars.sh && echo \"CMK_ARN=$CMK_ARN\" >> /app/build/vars.sh && echo && echo \"CMK_ID=$CMK_ID\" && echo \"CMK_ARN=$CMK_ARN\"",
  "description": "Create the CMK"
}

{
  "command": ". /app/build/vars.sh && aws kms enable-key-rotation --key-id \"$CMK_ID\" 2>&1 | head -5; aws kms create-alias --alias-name alias/harbor-glue-cmk --target-key-id \"$CMK_ID\" && echo \"alias OK\"",
  "description": "Enable rotation and create alias"
}

{
  "query": "select:TodoWrite",
  "max_results": 1
}

{
  "todos": [
    {
      "content": "Generate suffix and create IAM service role with trust policy",
      "activeForm": "Setting up suffix and IAM role",
      "status": "completed"
    },
    {
      "content": "Create CMK with hardened key policy and alias",
      "activeForm": "Creating CMK and alias",
      "status": "completed"
    },
    {
      "content": "Create three S3 buckets, default-encrypt with CMK, apply hardened bucket policies",
      "activeForm": "Creating S3 buckets with default SSE-KMS and bucket policies",
      "status": "in_progress"
    },
    {
      "content": "Upload placeholder etl.py to scripts bucket under CMK",
      "activeForm": "Uploading placeholder ETL script",
      "status": "pending"
    },
    {
      "content": "Attach inline policy to role scoped to CMK ARN and three bucket ARNs",
      "activeForm": "Attaching scoped inline policy to role",
      "status": "pending"
    },
    {
      "content": "Create Glue Security Configuration with all three encryption modes",
      "activeForm": "Creating Glue Security Configuration",
      "status": "pending"
    },
    {
      "content": "Configure data catalog encryption (best-effort) and mirror to SSM",
      "activeForm": "Configuring catalog encryption settings",
      "status": "pending"
    },
    {
      "content": "Create Glue Database, Table with column schema, and ETL Job",
      "activeForm": "Creating Glue Database, Table, and Job",
      "status": "pending"
    },
    {
      "content": "Publish full SSM manifest",
      "activeForm": "Publishing SSM manifest",
      "status": "pending"
    },
    {
      "content": "Verify the encryption surface aligns end-to-end",
      "activeForm": "Verifying alignment",
      "status": "pending"
    }
  ]
}

{
  "command": ". /app/build/vars.sh && for B in \"$SOURCE_BUCKET\" \"$TARGET_BUCKET\" \"$SCRIPTS_BUCKET\"; do aws s3api create-bucket --bucket \"$B\" --region \"$REGION\" >/dev/null && echo \"created $B\"; done",
  "description": "Create the three S3 buckets"
}

{
  "file_path": "/app/build/bucket-encryption.json",
  "content": "{\n  \"Rules\": [\n    {\n      \"ApplyServerSideEncryptionByDefault\": {\n        \"SSEAlgorithm\": \"aws:kms\",\n        \"KMSMasterKeyID\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n      },\n      \"BucketKeyEnabled\": true\n    }\n  ]\n}\n"
}

{
  "command": ". /app/build/vars.sh && for B in \"$SOURCE_BUCKET\" \"$TARGET_BUCKET\" \"$SCRIPTS_BUCKET\"; do aws s3api put-bucket-encryption --bucket \"$B\" --server-side-encryption-configuration file:///app/build/bucket-encryption.json && echo \"encrypted $B\"; done",
  "description": "Set bucket default encryption to CMK on all three buckets"
}

{
  "file_path": "/app/build/bucket-policy.tpl.json",
  "content": "{\n  \"Version\": \"2012-10-17\",\n  \"Id\": \"harbor-glue-bucket-hardening\",\n  \"Statement\": [\n    {\n      \"Sid\": \"DenyNonTLS\",\n      \"Effect\": \"Deny\",\n      \"Principal\": \"*\",\n      \"Action\": \"s3:*\",\n      \"Resource\": [\n        \"arn:aws:s3:::__BUCKET__\",\n        \"arn:aws:s3:::__BUCKET__/*\"\n      ],\n      \"Condition\": {\n        \"Bool\": { \"aws:SecureTransport\": \"false\" }\n      }\n    },\n    {\n      \"Sid\": \"DenyMissingSseHeader\",\n      \"Effect\": \"Deny\",\n      \"Principal\": \"*\",\n      \"Action\": \"s3:PutObject\",\n      \"Resource\": \"arn:aws:s3:::__BUCKET__/*\",\n      \"Condition\": {\n        \"Null\": { \"s3:x-amz-server-side-encryption\": \"true\" }\n      }\n    },\n    {\n      \"Sid\": \"DenyWrongSseAlgorithm\",\n      \"Effect\": \"Deny\",\n      \"Principal\": \"*\",\n      \"Action\": \"s3:PutObject\",\n      \"Resource\": \"arn:aws:s3:::__BUCKET__/*\",\n      \"Condition\": {\n        \"StringNotEquals\": {\n          \"s3:x-amz-server-side-encryption\": \"aws:kms\"\n        }\n      }\n    },\n    {\n      \"Sid\": \"DenyMissingKmsKeyId\",\n      \"Effect\": \"Deny\",\n      \"Principal\": \"*\",\n      \"Action\": \"s3:PutObject\",\n      \"Resource\": \"arn:aws:s3:::__BUCKET__/*\",\n      \"Condition\": {\n        \"Null\": { \"s3:x-amz-server-side-encryption-aws-kms-key-id\": \"true\" }\n      }\n    },\n    {\n      \"Sid\": \"DenyWrongKmsKeyId\",\n      \"Effect\": \"Deny\",\n      \"Principal\": \"*\",\n      \"Action\": \"s3:PutObject\",\n      \"Resource\": \"arn:aws:s3:::__BUCKET__/*\",\n      \"Condition\": {\n        \"StringNotEquals\": {\n          \"s3:x-amz-server-side-encryption-aws-kms-key-id\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n        }\n      }\n    }\n  ]\n}\n"
}

{
  "command": ". /app/build/vars.sh && for B in \"$SOURCE_BUCKET\" \"$TARGET_BUCKET\" \"$SCRIPTS_BUCKET\"; do sed \"s/__BUCKET__/$B/g\" /app/build/bucket-policy.tpl.json > /app/build/policy-$B.json && aws s3api put-bucket-policy --bucket \"$B\" --policy file:///app/build/policy-$B.json && echo \"policy applied to $B\"; done",
  "description": "Apply hardened bucket policies to all three buckets"
}

{
  "file_path": "/app/build/etl.py",
  "content": "import sys\nfrom awsglue.transforms import *\nfrom awsglue.utils import getResolvedOptions\nfrom pyspark.context import SparkContext\nfrom awsglue.context import GlueContext\nfrom awsglue.job import Job\n\nargs = getResolvedOptions(sys.argv, [\"JOB_NAME\"])\nsc = SparkContext()\nglueContext = GlueContext(sc)\nspark = glueContext.spark_session\njob = Job(glueContext)\njob.init(args[\"JOB_NAME\"], args)\n\norders = glueContext.create_dynamic_frame.from_catalog(\n    database=\"harbor_etl_db\",\n    table_name=\"orders_raw\",\n    transformation_ctx=\"orders_src\",\n)\n\njob.commit()\n"
}

{
  "command": ". /app/build/vars.sh && aws s3api put-object --bucket \"$SCRIPTS_BUCKET\" --key etl.py --body /app/build/etl.py --server-side-encryption aws:kms --ssekms-key-id \"$CMK_ARN\" --query '{ETag:ETag,SSE:ServerSideEncryption,KmsKey:SSEKMSKeyId}' --output json",
  "description": "Upload placeholder etl.py with explicit CMK SSE-KMS"
}

{
  "todos": [
    {
      "content": "Generate suffix and create IAM service role with trust policy",
      "activeForm": "Setting up suffix and IAM role",
      "status": "completed"
    },
    {
      "content": "Create CMK with hardened key policy and alias",
      "activeForm": "Creating CMK and alias",
      "status": "completed"
    },
    {
      "content": "Create three S3 buckets, default-encrypt with CMK, apply hardened bucket policies",
      "activeForm": "Creating S3 buckets with default SSE-KMS and bucket policies",
      "status": "completed"
    },
    {
      "content": "Upload placeholder etl.py to scripts bucket under CMK",
      "activeForm": "Uploading placeholder ETL script",
      "status": "completed"
    },
    {
      "content": "Attach inline policy to role scoped to CMK ARN and three bucket ARNs",
      "activeForm": "Attaching scoped inline policy to role",
      "status": "in_progress"
    },
    {
      "content": "Create Glue Security Configuration with all three encryption modes",
      "activeForm": "Creating Glue Security Configuration",
      "status": "pending"
    },
    {
      "content": "Configure data catalog encryption (best-effort) and mirror to SSM",
      "activeForm": "Configuring catalog encryption settings",
      "status": "pending"
    },
    {
      "content": "Create Glue Database, Table with column schema, and ETL Job",
      "activeForm": "Creating Glue Database, Table, and Job",
      "status": "pending"
    },
    {
      "content": "Publish full SSM manifest",
      "activeForm": "Publishing SSM manifest",
      "status": "pending"
    },
    {
      "content": "Verify the encryption surface aligns end-to-end",
      "activeForm": "Verifying alignment",
      "status": "pending"
    }
  ]
}

{
  "file_path": "/app/build/role-inline.json",
  "content": "{\n  \"Version\": \"2012-10-17\",\n  \"Statement\": [\n    {\n      \"Sid\": \"KmsEnvelopeOnHarborCmk\",\n      \"Effect\": \"Allow\",\n      \"Action\": [\n        \"kms:Encrypt\",\n        \"kms:Decrypt\",\n        \"kms:ReEncrypt*\",\n        \"kms:GenerateDataKey*\",\n        \"kms:DescribeKey\"\n      ],\n      \"Resource\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n    },\n    {\n      \"Sid\": \"S3ObjectRwOnHarborGlueBuckets\",\n      \"Effect\": \"Allow\",\n      \"Action\": [\n        \"s3:GetObject\",\n        \"s3:PutObject\",\n        \"s3:DeleteObject\"\n      ],\n      \"Resource\": [\n        \"arn:aws:s3:::harbor-glue-source-e8c5f0/*\",\n        \"arn:aws:s3:::harbor-glue-target-e8c5f0/*\",\n        \"arn:aws:s3:::harbor-glue-scripts-e8c5f0/*\"\n      ]\n    },\n    {\n      \"Sid\": \"S3ListOnHarborGlueBuckets\",\n      \"Effect\": \"Allow\",\n      \"Action\": [\n        \"s3:ListBucket\",\n        \"s3:GetBucketLocation\"\n      ],\n      \"Resource\": [\n        \"arn:aws:s3:::harbor-glue-source-e8c5f0\",\n        \"arn:aws:s3:::harbor-glue-target-e8c5f0\",\n        \"arn:aws:s3:::harbor-glue-scripts-e8c5f0\"\n      ]\n    }\n  ]\n}\n"
}

{
  "command": ". /app/build/vars.sh && aws iam put-role-policy --role-name \"$ROLE_NAME\" --policy-name harbor-glue-cmk-and-buckets --policy-document file:///app/build/role-inline.json && echo OK",
  "description": "Attach scoped inline policy to role"
}

{
  "todos": [
    {
      "content": "Generate suffix and create IAM service role with trust policy",
      "activeForm": "Setting up suffix and IAM role",
      "status": "completed"
    },
    {
      "content": "Create CMK with hardened key policy and alias",
      "activeForm": "Creating CMK and alias",
      "status": "completed"
    },
    {
      "content": "Create three S3 buckets, default-encrypt with CMK, apply hardened bucket policies",
      "activeForm": "Creating S3 buckets with default SSE-KMS and bucket policies",
      "status": "completed"
    },
    {
      "content": "Upload placeholder etl.py to scripts bucket under CMK",
      "activeForm": "Uploading placeholder ETL script",
      "status": "completed"
    },
    {
      "content": "Attach inline policy to role scoped to CMK ARN and three bucket ARNs",
      "activeForm": "Attaching scoped inline policy to role",
      "status": "completed"
    },
    {
      "content": "Create Glue Security Configuration with all three encryption modes",
      "activeForm": "Creating Glue Security Configuration",
      "status": "in_progress"
    },
    {
      "content": "Configure data catalog encryption (best-effort) and mirror to SSM",
      "activeForm": "Configuring catalog encryption settings",
      "status": "pending"
    },
    {
      "content": "Create Glue Database, Table with column schema, and ETL Job",
      "activeForm": "Creating Glue Database, Table, and Job",
      "status": "pending"
    },
    {
      "content": "Publish full SSM manifest",
      "activeForm": "Publishing SSM manifest",
      "status": "pending"
    },
    {
      "content": "Verify the encryption surface aligns end-to-end",
      "activeForm": "Verifying alignment",
      "status": "pending"
    }
  ]
}

{
  "file_path": "/app/build/sec-config.json",
  "content": "{\n  \"S3Encryption\": [\n    {\n      \"S3EncryptionMode\": \"SSE-KMS\",\n      \"KmsKeyArn\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n    }\n  ],\n  \"CloudWatchEncryption\": {\n    \"CloudWatchEncryptionMode\": \"SSE-KMS\",\n    \"KmsKeyArn\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n  },\n  \"JobBookmarksEncryption\": {\n    \"JobBookmarksEncryptionMode\": \"CSE-KMS\",\n    \"KmsKeyArn\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n  }\n}\n"
}

{
  "command": ". /app/build/vars.sh && aws glue create-security-configuration --name \"$SEC_CONFIG\" --encryption-configuration file:///app/build/sec-config.json",
  "description": "Create Glue Security Configuration"
}

{
  "file_path": "/app/build/catalog-enc.json",
  "content": "{\n  \"EncryptionAtRest\": {\n    \"CatalogEncryptionMode\": \"SSE-KMS\",\n    \"SseAwsKmsKeyId\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n  },\n  \"ConnectionPasswordEncryption\": {\n    \"ReturnConnectionPasswordEncrypted\": true,\n    \"AwsKmsKeyId\": \"arn:aws:kms:us-east-1:000000000000:key/5141225c-b357-48dc-b2e8-8f5bc3102829\"\n  }\n}\n"
}

{
  "command": ". /app/build/vars.sh && aws glue put-data-catalog-encryption-settings --data-catalog-encryption-settings file:///app/build/catalog-enc.json 2>&1; echo \"---\"; aws glue get-data-catalog-encryption-settings 2>&1",
  "description": "Best-effort PutDataCatalogEncryptionSettings"
}

{
  "todos": [
    {
      "content": "Generate suffix and create IAM service role with trust policy",
      "activeForm": "Setting up suffix and IAM role",
      "status": "completed"
    },
    {
      "content": "Create CMK with hardened key policy and alias",
      "activeForm": "Creating CMK and alias",
      "status": "completed"
    },
    {
      "content": "Create three S3 buckets, default-encrypt with CMK, apply hardened bucket policies",
      "activeForm": "Creating S3 buckets with default SSE-KMS and bucket policies",
      "status": "completed"
    },
    {
      "content": "Upload placeholder etl.py to scripts bucket under CMK",
      "activeForm": "Uploading placeholder ETL script",
      "status": "completed"
    },
    {
      "content": "Attach inline policy to role scoped to CMK ARN and three bucket ARNs",
      "activeForm": "Attaching scoped inline policy to role",
      "status": "completed"
    },
    {
      "content": "Create Glue Security Configuration with all three encryption modes",
      "activeForm": "Creating Glue Security Configuration",
      "status": "completed"
    },
    {
      "content": "Configure data catalog encryption (best-effort) and mirror to SSM",
      "activeForm": "Configuring catalog encryption settings",
      "status": "completed"
    },
    {
      "content": "Create Glue Database, Table with column schema, and ETL Job",
      "activeForm": "Creating Glue Database, Table, and Job",
      "status": "in_progress"
    },
    {
      "content": "Publish full SSM manifest",
      "activeForm": "Publishing SSM manifest",
      "status": "pending"
    },
    {
      "content": "Verify the encryption surface aligns end-to-end",
      "activeForm": "Verifying alignment",
      "status": "pending"
    }
  ]
}

{
  "file_path": "/app/build/db.json",
  "content": "{\n  \"Name\": \"harbor_etl_db\",\n  \"Description\": \"Harbor data-platform Glue catalog DB\"\n}\n"
}

{
  "command": ". /app/build/vars.sh && aws glue create-database --database-input file:///app/build/db.json && echo OK",
  "description": "Create Glue database"
}

{
  "file_path": "/app/build/table.json",
  "content": "{\n  \"Name\": \"orders_raw\",\n  \"Description\": \"Raw orders ingest, partitioned by ingest date\",\n  \"TableType\": \"EXTERNAL_TABLE\",\n  \"Parameters\": {\n    \"classification\": \"parquet\",\n    \"EXTERNAL\": \"TRUE\"\n  },\n  \"StorageDescriptor\": {\n    \"Columns\": [\n      { \"Name\": \"order_id\",   \"Type\": \"string\" },\n      { \"Name\": \"customer_id\", \"Type\": \"string\" },\n      { \"Name\": \"amount\",     \"Type\": \"decimal(18,2)\" },\n      { \"Name\": \"ts\",         \"Type\": \"timestamp\" }\n    ],\n    \"Location\": \"s3://harbor-glue-source-e8c5f0/orders_raw/\",\n    \"InputFormat\": \"org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat\",\n    \"OutputFormat\": \"org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat\",\n    \"SerdeInfo\": {\n      \"SerializationLibrary\": \"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe\",\n      \"Parameters\": { \"serialization.format\": \"1\" }\n    },\n    \"Compressed\": false,\n    \"StoredAsSubDirectories\": false\n  }\n}\n"
}

{
  "command": ". /app/build/vars.sh && aws glue create-table --database-name \"$DB_NAME\" --table-input file:///app/build/table.json && echo OK",
  "description": "Create Glue table"
}

{
  "file_path": "/app/build/job.json",
  "content": "{\n  \"Name\": \"harbor-orders-etl\",\n  \"Description\": \"Harbor orders raw → curated ETL, gated by harbor-glue-cmk\",\n  \"Role\": \"arn:aws:iam::000000000000:role/harbor-glue-svc-role\",\n  \"ExecutionProperty\": { \"MaxConcurrentRuns\": 1 },\n  \"Command\": {\n    \"Name\": \"glueetl\",\n    \"ScriptLocation\": \"s3://harbor-glue-scripts-e8c5f0/etl.py\",\n    \"PythonVersion\": \"3\"\n  },\n  \"DefaultArguments\": {\n    \"--job-language\": \"python\",\n    \"--job-bookmark-option\": \"job-bookmark-enable\",\n    \"--enable-job-insights\": \"true\",\n    \"--enable-glue-datacatalog\": \"true\",\n    \"--enable-metrics\": \"true\",\n    \"--enable-continuous-cloudwatch-log\": \"true\",\n    \"--TempDir\": \"s3://harbor-glue-target-e8c5f0/_tmp/\",\n    \"--SOURCE_BUCKET\": \"harbor-glue-source-e8c5f0\",\n    \"--TARGET_BUCKET\": \"harbor-glue-target-e8c5f0\",\n    \"--SOURCE_DATABASE\": \"harbor_etl_db\",\n    \"--SOURCE_TABLE\": \"orders_raw\"\n  },\n  \"MaxRetries\": 1,\n  \"Timeout\": 60,\n  \"GlueVersion\": \"4.0\",\n  \"WorkerType\": \"G.1X\",\n  \"NumberOfWorkers\": 2,\n  \"SecurityConfiguration\": \"harbor-glue-sec-config\"\n}\n"
}

{
  "command": ". /app/build/vars.sh && aws glue create-job --cli-input-json file:///app/build/job.json",
  "description": "Create Glue job"
}

{
  "todos": [
    {
      "content": "Generate suffix and create IAM service role with trust policy",
      "activeForm": "Setting up suffix and IAM role",
      "status": "completed"
    },
    {
      "content": "Create CMK with hardened key policy and alias",
      "activeForm": "Creating CMK and alias",
      "status": "completed"
    },
    {
      "content": "Create three S3 buckets, default-encrypt with CMK, apply hardened bucket policies",
      "activeForm": "Creating S3 buckets with default SSE-KMS and bucket policies",
      "status": "completed"
    },
    {
      "content": "Upload placeholder etl.py to scripts bucket under CMK",
      "activeForm": "Uploading placeholder ETL script",
      "status": "completed"
    },
    {
      "content": "Attach inline policy to role scoped to CMK ARN and three bucket ARNs",
      "activeForm": "Attaching scoped inline policy to role",
      "status": "completed"
    },
    {
      "content": "Create Glue Security Configuration with all three encryption modes",
      "activeForm": "Creating Glue Security Configuration",
      "status": "completed"
    },
    {
      "content": "Configure data catalog encryption (best-effort) and mirror to SSM",
      "activeForm": "Configuring catalog encryption settings",
      "status": "completed"
    },
    {
      "content": "Create Glue Database, Table with column schema, and ETL Job",
      "activeForm": "Creating Glue Database, Table, and Job",
      "status": "completed"
    },
    {
      "content": "Publish full SSM manifest",
      "activeForm": "Publishing SSM manifest",
      "status": "in_progress"
    },
    {
      "content": "Verify the encryption surface aligns end-to-end",
      "activeForm": "Verifying alignment",
      "status": "pending"
    }
  ]
}

{
  "command": ". /app/build/vars.sh\nset -e\nput() { aws ssm put-parameter --name \"$1\" --type \"$2\" --value \"$3\" --overwrite >/dev/null && echo \"  $1 = $3\"; }\nput /harbor/glue/cmk-arn                       String \"$CMK_ARN\"\nput /harbor/glue/sec-config                    String \"$SEC_CONFIG\"\nput /harbor/glue/database                      String \"$DB_NAME\"\nput /harbor/glue/table                         String \"$TABLE_NAME\"\nput /harbor/glue/job-name                      String \"$JOB_NAME\"\nput /harbor/glue/svc-role-arn                  String \"$ROLE_ARN\"\nput /harbor/glue/source-bucket                 String \"$SOURCE_BUCKET\"\nput /harbor/glue/target-bucket                 String \"$TARGET_BUCKET\"\nput /harbor/glue/scripts-bucket                String \"$SCRIPTS_BUCKET\"\nput /harbor/glue/catalog-encryption-mode       String \"SSE-KMS\"\nput /harbor/glue/catalog-cmk-id                String \"$CMK_ARN\"\nput /harbor/glue/connection-password-encrypted String \"true\"\necho \"ssm manifest published\"",
  "description": "Publish SSM manifest"
}

{
  "todos": [
    {
      "content": "Generate suffix and create IAM service role with trust policy",
      "activeForm": "Setting up suffix and IAM role",
      "status": "completed"
    },
    {
      "content": "Create CMK with hardened key policy and alias",
      "activeForm": "Creating CMK and alias",
      "status": "completed"
    },
    {
      "content": "Create three S3 buckets, default-encrypt with CMK, apply hardened bucket policies",
      "activeForm": "Creating S3 buckets with default SSE-KMS and bucket policies",
      "status": "completed"
    },
    {
      "content": "Upload placeholder etl.py to scripts bucket under CMK",
      "activeForm": "Uploading placeholder ETL script",
      "status": "completed"
    },
    {
      "content": "Attach inline policy to role scoped to CMK ARN and three bucket ARNs",
      "activeForm": "Attaching scoped inline policy to role",
      "status": "completed"
    },
    {
      "content": "Create Glue Security Configuration with all three encryption modes",
      "activeForm": "Creating Glue Security Configuration",
      "status": "completed"
    },
    {
      "content": "Configure data catalog encryption (best-effort) and mirror to SSM",
      "activeForm": "Configuring catalog encryption settings",
      "status": "completed"
    },
    {
      "content": "Create Glue Database, Table with column schema, and ETL Job",
      "activeForm": "Creating Glue Database, Table, and Job",
      "status": "completed"
    },
    {
      "content": "Publish full SSM manifest",
      "activeForm": "Publishing SSM manifest",
      "status": "completed"
    },
    {
      "content": "Verify the encryption surface aligns end-to-end",
      "activeForm": "Verifying alignment",
      "status": "in_progress"
    }
  ]
}

{
  "command": ". /app/build/vars.sh\necho \"=== alias resolves to CMK ===\" && aws kms describe-key --key-id alias/harbor-glue-cmk --query 'KeyMetadata.{KeyId:KeyId,Arn:Arn,Enabled:Enabled,Manager:KeyManager}' --output table\necho \"=== key policy summary ===\" && aws kms get-key-policy --key-id \"$CMK_ID\" --policy-name default --output text | python3 -c 'import json,sys;p=json.loads(sys.stdin.read());print(\"statements:\");[print(\" -\", s[\"Sid\"], \"->\", s.get(\"Principal\")) for s in p[\"Statement\"]]'",
  "description": "Verify CMK and policy"
}

{
  "command": ". /app/build/vars.sh\necho \"=== bucket default encryption ===\"\nfor B in \"$SOURCE_BUCKET\" \"$TARGET_BUCKET\" \"$SCRIPTS_BUCKET\"; do\n  KEY=$(aws s3api get-bucket-encryption --bucket \"$B\" --query 'ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault.KMSMasterKeyID' --output text)\n  ALG=$(aws s3api get-bucket-encryption --bucket \"$B\" --query 'ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault.SSEAlgorithm' --output text)\n  printf \"  %-32s %-7s %s\\n\" \"$B\" \"$ALG\" \"$KEY\"\ndone\necho \"=== sec config encryption modes ===\"\naws glue get-security-configuration --name \"$SEC_CONFIG\" --query 'SecurityConfiguration.EncryptionConfiguration' --output json\necho \"=== job binding ===\"\naws glue get-job --job-name \"$JOB_NAME\" --query 'Job.{Role:Role,SecConfig:SecurityConfiguration,Glue:GlueVersion,Worker:WorkerType,Workers:NumberOfWorkers,Timeout:Timeout,MaxRetries:MaxRetries,Bookmark:DefaultArguments.\"--job-bookmark-option\",Catalog:DefaultArguments.\"--enable-glue-datacatalog\"}' --output table",
  "description": "Verify alignment of S3, sec config, and job"
}

{
  "command": ". /app/build/vars.sh\necho \"=== role inline policy resources ===\"\naws iam get-role-policy --role-name \"$ROLE_NAME\" --policy-name harbor-glue-cmk-and-buckets --query 'PolicyDocument.Statement[*].{Sid:Sid,Action:Action,Resource:Resource}' --output json\necho \"=== attached managed policies ===\"\naws iam list-attached-role-policies --role-name \"$ROLE_NAME\" --output table\necho \"=== glue table columns ===\"\naws glue get-table --database-name \"$DB_NAME\" --name \"$TABLE_NAME\" --query 'Table.{TableType:TableType,Loc:StorageDescriptor.Location,Cols:StorageDescriptor.Columns[*].{n:Name,t:Type}}' --output json",
  "description": "Verify role and table"
}

{
  "command": ". /app/build/vars.sh\necho \"=== bucket policy on source , confirm Deny statements present ===\"\naws s3api get-bucket-policy --bucket \"$SOURCE_BUCKET\" --query Policy --output text | python3 -c 'import json,sys;p=json.loads(sys.stdin.read());[print(\" -\", s[\"Sid\"], s[\"Effect\"], \"/\", s[\"Action\"]) for s in p[\"Statement\"]]'\necho \"=== negative test: PutObject without SSE header should be denied ===\"\necho hi > /tmp/no-sse.txt\naws s3api put-object --bucket \"$SOURCE_BUCKET\" --key probe-no-sse.txt --body /tmp/no-sse.txt 2>&1 | head -5\necho \"=== negative test: PutObject with SSE-S3 (wrong algo) should be denied ===\"\naws s3api put-object --bucket \"$SOURCE_BUCKET\" --key probe-sse-s3.txt --body /tmp/no-sse.txt --server-side-encryption AES256 2>&1 | head -5\necho \"=== positive test: PutObject with correct CMK should succeed ===\"\naws s3api put-object --bucket \"$SOURCE_BUCKET\" --key probe-ok.txt --body /tmp/no-sse.txt --server-side-encryption aws:kms --ssekms-key-id \"$CMK_ARN\" --query '{SSE:ServerSideEncryption,Key:SSEKMSKeyId}' --output json\naws s3api delete-object --bucket \"$SOURCE_BUCKET\" --key probe-ok.txt >/dev/null && echo \"cleanup OK\"",
  "description": "Negative/positive test of bucket policy"
}

{
  "command": ". /app/build/vars.sh\naws s3api delete-object --bucket \"$SOURCE_BUCKET\" --key probe-no-sse.txt >/dev/null\naws s3api delete-object --bucket \"$SOURCE_BUCKET\" --key probe-sse-s3.txt >/dev/null\necho \"probes cleaned\"\necho \"=== final bucket listing ===\"\nfor B in \"$SOURCE_BUCKET\" \"$TARGET_BUCKET\" \"$SCRIPTS_BUCKET\"; do\n  echo \"-- $B --\"\n  aws s3api list-objects-v2 --bucket \"$B\" --query 'Contents[*].Key' --output text 2>/dev/null\ndone",
  "description": "Clean up test probes and list final buckets"
}

{
  "command": ". /app/build/vars.sh\necho \"=== full SSM manifest ===\"\naws ssm get-parameters-by-path --path /harbor/glue --query 'Parameters[*].{Name:Name,Value:Value}' --output table",
  "description": "Final SSM manifest dump"
}

{
  "todos": [
    {
      "content": "Generate suffix and create IAM service role with trust policy",
      "activeForm": "Setting up suffix and IAM role",
      "status": "completed"
    },
    {
      "content": "Create CMK with hardened key policy and alias",
      "activeForm": "Creating CMK and alias",
      "status": "completed"
    },
    {
      "content": "Create three S3 buckets, default-encrypt with CMK, apply hardened bucket policies",
      "activeForm": "Creating S3 buckets with default SSE-KMS and bucket policies",
      "status": "completed"
    },
    {
      "content": "Upload placeholder etl.py to scripts bucket under CMK",
      "activeForm": "Uploading placeholder ETL script",
      "status": "completed"
    },
    {
      "content": "Attach inline policy to role scoped to CMK ARN and three bucket ARNs",
      "activeForm": "Attaching scoped inline policy to role",
      "status": "completed"
    },
    {
      "content": "Create Glue Security Configuration with all three encryption modes",
      "activeForm": "Creating Glue Security Configuration",
      "status": "completed"
    },
    {
      "content": "Configure data catalog encryption (best-effort) and mirror to SSM",
      "activeForm": "Configuring catalog encryption settings",
      "status": "completed"
    },
    {
      "content": "Create Glue Database, Table with column schema, and ETL Job",
      "activeForm": "Creating Glue Database, Table, and Job",
      "status": "completed"
    },
    {
      "content": "Publish full SSM manifest",
      "activeForm": "Publishing SSM manifest",
      "status": "completed"
    },
    {
      "content": "Verify the encryption surface aligns end-to-end",
      "activeForm": "Verifying alignment",
      "status": "completed"
    }
  ]
}