Amazon S3

# Virtual-hosted style (khuyến nghị — mặc định từ 2023)
https://bucket-name.s3.region.amazonaws.com/key
https://my-app-prod.s3.ap-southeast-1.amazonaws.com/images/logo.png

# Path-style (deprecated cho bucket mới từ 09/2023)
https://s3.region.amazonaws.com/bucket-name/key

# S3 URI (dùng trong AWS CLI & SDK)
s3://my-app-prod/images/logo.png

# ARN (dùng trong IAM Policy)
arn:aws:s3:::my-app-prod/images/logo.png

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "PublicReadGetObject",
      "Effect": "Allow",
      "Principal": "*",
      "Action": "s3:GetObject",
      "Resource": "arn:aws:s3:::my-website-bucket/*"
    }
  ]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "DenyNonVPCAccess",
      "Effect": "Deny",
      "Principal": "*",
      "Action": "s3:*",
      "Resource": [
        "arn:aws:s3:::my-private-bucket",
        "arn:aws:s3:::my-private-bucket/*"
      ],
      "Condition": {
        "StringNotEquals": {
          "aws:sourceVpce": "vpce-1a2b3c4d"
        }
      }
    }
  ]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "DenyHTTP",
      "Effect": "Deny",
      "Principal": "*",
      "Action": "s3:*",
      "Resource": "arn:aws:s3:::my-bucket/*",
      "Condition": {
        "Bool": {
          "aws:SecureTransport": "false"
        }
      }
    }
  ]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "CrossAccountAccess",
      "Effect": "Allow",
      "Principal": {
        "AWS": "arn:aws:iam::123456789012:root"
      },
      "Action": [
        "s3:GetObject",
        "s3:ListBucket"
      ],
      "Resource": [
        "arn:aws:s3:::my-shared-bucket",
        "arn:aws:s3:::my-shared-bucket/*"
      ]
    }
  ]
}

# Enable versioning
aws s3api put-bucket-versioning \
  --bucket my-bucket \
  --versioning-configuration Status=Enabled

# Kiểm tra trạng thái versioning
aws s3api get-bucket-versioning --bucket my-bucket

# List tất cả versions của object
aws s3api list-object-versions \
  --bucket my-bucket \
  --prefix photos/beach.jpg

# GET version cụ thể
aws s3api get-object \
  --bucket my-bucket \
  --key photos/beach.jpg \
  --version-id "abc123" \
  output.jpg

# Xóa vĩnh viễn 1 version
aws s3api delete-object \
  --bucket my-bucket \
  --key photos/beach.jpg \
  --version-id "abc123"

# Enable MFA Delete (chỉ root account)
aws s3api put-bucket-versioning \
  --bucket my-bucket \
  --versioning-configuration Status=Enabled,MFADelete=Enabled \
  --mfa "arn:aws:iam::123456789012:mfa/root-mfa 123456"

# Xem default encryption hiện tại
aws s3api get-bucket-encryption --bucket my-bucket

# Set default encryption = SSE-KMS với customer managed key
aws s3api put-bucket-encryption \
  --bucket my-bucket \
  --server-side-encryption-configuration '{
    "Rules": [{
      "ApplyServerSideEncryptionByDefault": {
        "SSEAlgorithm": "aws:kms",
        "KMSMasterKeyID": "arn:aws:kms:ap-southeast-1:123456789012:key/abcd-1234"
      },
      "BucketKeyEnabled": true
    }]
  }'

# Enforce encryption bằng bucket policy (deny nếu không có encryption header)
# Lưu ý: từ 01/2023 S3 tự động encrypt SSE-S3, nên policy này
# chủ yếu dùng để enforce SSE-KMS cụ thể

{
  "Rules": [
    {
      "ID": "MoveToIAThenGlacier",
      "Status": "Enabled",
      "Filter": { "Prefix": "logs/" },
      "Transitions": [
        {
          "Days": 30,
          "StorageClass": "STANDARD_IA"
        },
        {
          "Days": 90,
          "StorageClass": "GLACIER"
        },
        {
          "Days": 365,
          "StorageClass": "DEEP_ARCHIVE"
        }
      ],
      "Expiration": {
        "Days": 2555
      }
    },
    {
      "ID": "DeleteOldVersions",
      "Status": "Enabled",
      "Filter": {},
      "NoncurrentVersionTransitions": [
        {
          "NoncurrentDays": 30,
          "StorageClass": "STANDARD_IA"
        }
      ],
      "NoncurrentVersionExpiration": {
        "NoncurrentDays": 90
      }
    },
    {
      "ID": "CleanupIncompleteUploads",
      "Status": "Enabled",
      "Filter": {},
      "AbortIncompleteMultipartUpload": {
        "DaysAfterInitiation": 7
      }
    }
  ]
}

# Áp dụng lifecycle configuration
aws s3api put-bucket-lifecycle-configuration \
  --bucket my-bucket \
  --lifecycle-configuration file://lifecycle-config.json

# Xem lifecycle rules hiện tại
aws s3api get-bucket-lifecycle-configuration --bucket my-bucket

{
  "Role": "arn:aws:iam::123456789012:role/S3ReplicationRole",
  "Rules": [
    {
      "ID": "ReplicateAll",
      "Status": "Enabled",
      "Priority": 1,
      "Filter": { "Prefix": "" },
      "Destination": {
        "Bucket": "arn:aws:s3:::my-app-dr-usw2",
        "StorageClass": "STANDARD_IA",
        "ReplicationTime": {
          "Status": "Enabled",
          "Time": { "Minutes": 15 }
        },
        "Metrics": {
          "Status": "Enabled",
          "EventThreshold": { "Minutes": 15 }
        }
      },
      "DeleteMarkerReplication": { "Status": "Enabled" }
    }
  ]
}

# Áp dụng replication configuration
aws s3api put-bucket-replication \
  --bucket my-app-prod-apse1 \
  --replication-configuration file://replication-config.json

# Replicate objects đã tồn tại (Batch Replication)
aws s3control create-job \
  --account-id 123456789012 \
  --operation '{"S3ReplicateObject":{}}' \
  --manifest-generator '...' \
  --priority 10 \
  --role-arn arn:aws:iam::123456789012:role/BatchReplicationRole

# Tạo bucket với Object Lock enabled
aws s3api create-bucket \
  --bucket my-compliance-bucket \
  --region ap-southeast-1 \
  --create-bucket-configuration LocationConstraint=ap-southeast-1 \
  --object-lock-enabled-for-bucket

# Set default retention cho bucket (Compliance mode, 365 ngày)
aws s3api put-object-lock-configuration \
  --bucket my-compliance-bucket \
  --object-lock-configuration '{
    "ObjectLockEnabled": "Enabled",
    "Rule": {
      "DefaultRetention": {
        "Mode": "COMPLIANCE",
        "Days": 365
      }
    }
  }'

# Set retention cho object cụ thể
aws s3api put-object-retention \
  --bucket my-compliance-bucket \
  --key financial-report-2024.pdf \
  --retention '{"Mode":"GOVERNANCE","RetainUntilDate":"2027-12-31T00:00:00Z"}'

# Bật Legal Hold
aws s3api put-object-legal-hold \
  --bucket my-compliance-bucket \
  --key evidence-doc.pdf \
  --legal-hold '{"Status":"ON"}'

# Tắt Legal Hold
aws s3api put-object-legal-hold \
  --bucket my-compliance-bucket \
  --key evidence-doc.pdf \
  --legal-hold '{"Status":"OFF"}'

# AWS CLI tự động dùng multipart cho file lớn
aws s3 cp large-file.zip s3://my-bucket/backups/ \
  --storage-class STANDARD_IA

# Cấu hình multipart threshold và chunk size
aws configure set default.s3.multipart_threshold 100MB
aws configure set default.s3.multipart_chunksize 50MB

# Manual multipart upload (low-level)
# 1. Initiate
aws s3api create-multipart-upload \
  --bucket my-bucket --key large-file.zip

# 2. Upload parts (song song)
aws s3api upload-part \
  --bucket my-bucket --key large-file.zip \
  --part-number 1 --upload-id "xxx" \
  --body part1.bin

# 3. Complete
aws s3api complete-multipart-upload \
  --bucket my-bucket --key large-file.zip \
  --upload-id "xxx" \
  --multipart-upload file://parts.json

# Liệt kê incomplete multipart uploads (tránh phí ẩn!)
aws s3api list-multipart-uploads --bucket my-bucket

# Enable Transfer Acceleration
aws s3api put-bucket-accelerate-configuration \
  --bucket my-bucket \
  --accelerate-configuration Status=Enabled

# Upload qua accelerated endpoint
aws s3 cp large-file.zip s3://my-bucket/ --endpoint-url https://s3-accelerate.amazonaws.com

# Test tốc độ: https://s3-accelerate-speedtest.s3-accelerate.amazonaws.com/en/accelerate-speed-comparsion.html

# Download 1MB đầu tiên
aws s3api get-object \
  --bucket my-bucket \
  --key large-video.mp4 \
  --range "bytes=0-1048575" \
  first-1mb.bin

# Download từ byte 1MB đến 2MB
aws s3api get-object \
  --bucket my-bucket \
  --key large-video.mp4 \
  --range "bytes=1048576-2097151" \
  second-1mb.bin

# Cấu hình notification
aws s3api put-bucket-notification-configuration \
  --bucket my-images-bucket \
  --notification-configuration '{
    "LambdaFunctionConfigurations": [
      {
        "Id": "ResizeImageOnUpload",
        "LambdaFunctionArn": "arn:aws:lambda:ap-southeast-1:123456789012:function:resize-image",
        "Events": ["s3:ObjectCreated:*"],
        "Filter": {
          "Key": {
            "FilterRules": [
              {"Name": "prefix", "Value": "uploads/"},
              {"Name": "suffix", "Value": ".jpg"}
            ]
          }
        }
      }
    ]
  }'

# Lưu ý: Lambda function cần resource-based policy cho phép S3 invoke
aws lambda add-permission \
  --function-name resize-image \
  --statement-id s3-trigger \
  --action lambda:InvokeFunction \
  --principal s3.amazonaws.com \
  --source-arn arn:aws:s3:::my-images-bucket

# 1. Enable static website hosting
aws s3 website s3://my-website-bucket/ \
  --index-document index.html \
  --error-document error.html

# 2. Tắt Block Public Access
aws s3api put-public-access-block \
  --bucket my-website-bucket \
  --public-access-block-configuration \
    BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false

# 3. Thêm bucket policy cho public read
aws s3api put-bucket-policy --bucket my-website-bucket --policy '{
  "Version": "2012-10-17",
  "Statement": [{
    "Sid": "PublicRead",
    "Effect": "Allow",
    "Principal": "*",
    "Action": "s3:GetObject",
    "Resource": "arn:aws:s3:::my-website-bucket/*"
  }]
}'

# 4. Upload website files
aws s3 sync ./dist/ s3://my-website-bucket/ \
  --cache-control "max-age=86400"

# Website URL format:
# http://my-website-bucket.s3-website-ap-southeast-1.amazonaws.com
# hoặc http://my-website-bucket.s3-website.ap-southeast-1.amazonaws.com

aws s3api put-bucket-cors --bucket my-assets-bucket --cors-configuration '{
  "CORSRules": [
    {
      "AllowedOrigins": ["https://www.example.com"],
      "AllowedMethods": ["GET", "HEAD"],
      "AllowedHeaders": ["*"],
      "ExposeHeaders": ["ETag"],
      "MaxAgeSeconds": 3600
    }
  ]
}'

# Tạo access point (Internet access)
aws s3control create-access-point \
  --account-id 123456789012 \
  --name analytics-ap \
  --bucket my-datalake-bucket

# Tạo access point (VPC only)
aws s3control create-access-point \
  --account-id 123456789012 \
  --name internal-ap \
  --bucket my-datalake-bucket \
  --vpc-configuration VpcId=vpc-1a2b3c4d

# Set access point policy
aws s3control put-access-point-policy \
  --account-id 123456789012 \
  --name analytics-ap \
  --policy '{
    "Version": "2012-10-17",
    "Statement": [{
      "Effect": "Allow",
      "Principal": {"AWS": "arn:aws:iam::123456789012:role/AnalyticsRole"},
      "Action": ["s3:GetObject", "s3:ListBucket"],
      "Resource": [
        "arn:aws:s3:ap-southeast-1:123456789012:accesspoint/analytics-ap",
        "arn:aws:s3:ap-southeast-1:123456789012:accesspoint/analytics-ap/object/*"
      ]
    }]
  }'

# Truy cập qua access point
aws s3 ls s3://arn:aws:s3:ap-southeast-1:123456789012:accesspoint/analytics-ap/

# Tạo VPC Gateway Endpoint cho S3
aws ec2 create-vpc-endpoint \
  --vpc-id vpc-1a2b3c4d \
  --service-name com.amazonaws.ap-southeast-1.s3 \
  --route-table-ids rtb-11aa22bb

# Kiểm tra endpoint
aws ec2 describe-vpc-endpoints \
  --filters "Name=service-name,Values=com.amazonaws.ap-southeast-1.s3"

# 1. Kiểm tra IAM user/role hiện tại
aws sts get-caller-identity

# 2. Kiểm tra bucket policy
aws s3api get-bucket-policy --bucket my-bucket

# 3. Kiểm tra Block Public Access
aws s3api get-public-access-block --bucket my-bucket

# 4. Kiểm tra object ACL
aws s3api get-object-acl --bucket my-bucket --key my-file.txt

# 5. Kiểm tra bucket ACL
aws s3api get-bucket-acl --bucket my-bucket

# 6. Kiểm tra encryption (SSE-KMS cần kms:Decrypt permission)
aws s3api head-object --bucket my-bucket --key my-file.txt

# 7. Simulate IAM policy
aws iam simulate-principal-policy \
  --policy-source-arn arn:aws:iam::123456789012:user/myuser \
  --action-names s3:GetObject \
  --resource-arns arn:aws:s3:::my-bucket/my-file.txt

# 8. Kiểm tra VPC Endpoint policy (nếu truy cập từ VPC)
aws ec2 describe-vpc-endpoints --vpc-endpoint-ids vpce-xxx

# Pattern: {org}-{env}-{purpose}-{region-short}
acme-prod-assets-apse1          # Production assets, Singapore
acme-dev-logs-use1              # Dev logs, N. Virginia
acme-prod-backup-usw2           # Production backup, Oregon
acme-staging-datalake-apse1     # Staging data lake

# Object key convention
# Pattern: {category}/{year}/{month}/{day}/{filename}
logs/2024/03/15/access-log-001.gz
backups/database/2024/03/15/mydb-full.sql.gz
uploads/users/u-12345/avatar.jpg
raw-data/events/2024/03/15/events-001.parquet

# Tạo presigned URL cho download (hết hạn sau 1 giờ)
aws s3 presign s3://my-bucket/reports/q1-2024.pdf \
  --expires-in 3600

# Tạo presigned URL cho upload
aws s3 presign s3://my-bucket/uploads/user-file.pdf \
  --expires-in 3600

# Output: https://my-bucket.s3.amazonaws.com/reports/q1-2024.pdf?X-Amz-Algorithm=...
# Ai có URL này đều có thể download trong 1 giờ

# Tạo bucket
aws s3 mb s3://my-new-bucket --region ap-southeast-1

# Tạo bucket (s3api — chi tiết hơn)
aws s3api create-bucket \
  --bucket my-new-bucket \
  --region ap-southeast-1 \
  --create-bucket-configuration LocationConstraint=ap-southeast-1

# Liệt kê tất cả buckets
aws s3 ls

# Liệt kê objects trong bucket
aws s3 ls s3://my-bucket/
aws s3 ls s3://my-bucket/photos/ --recursive --human-readable --summarize

# Xóa bucket (phải rỗng)
aws s3 rb s3://my-bucket

# Xóa bucket + tất cả objects (NGUY HIỂM!)
aws s3 rb s3://my-bucket --force

# Xem thông tin bucket
aws s3api get-bucket-location --bucket my-bucket
aws s3api get-bucket-versioning --bucket my-bucket
aws s3api get-bucket-encryption --bucket my-bucket

# Upload file
aws s3 cp myfile.txt s3://my-bucket/
aws s3 cp myfile.txt s3://my-bucket/folder/renamed.txt

# Upload với storage class
aws s3 cp myfile.txt s3://my-bucket/ --storage-class STANDARD_IA

# Upload với metadata
aws s3 cp myfile.txt s3://my-bucket/ \
  --metadata '{"project":"alpha","team":"backend"}' \
  --content-type "text/plain"

# Upload thư mục (recursive)
aws s3 cp ./local-folder/ s3://my-bucket/remote-folder/ --recursive

# Sync (chỉ upload files thay đổi)
aws s3 sync ./local-folder/ s3://my-bucket/remote-folder/
aws s3 sync ./dist/ s3://my-website/ --delete  # --delete xóa files không còn ở source

# Download
aws s3 cp s3://my-bucket/myfile.txt ./
aws s3 cp s3://my-bucket/folder/ ./local/ --recursive

# Copy giữa buckets
aws s3 cp s3://source-bucket/file.txt s3://dest-bucket/file.txt

# Move (copy + delete source)
aws s3 mv s3://my-bucket/old-key.txt s3://my-bucket/new-key.txt

# Xóa object
aws s3 rm s3://my-bucket/myfile.txt
aws s3 rm s3://my-bucket/folder/ --recursive  # Xóa cả folder

# Xem metadata object
aws s3api head-object --bucket my-bucket --key myfile.txt

# Kiểm tra Block Public Access
aws s3api get-public-access-block --bucket my-bucket

# Bật Block Public Access
aws s3api put-public-access-block --bucket my-bucket \
  --public-access-block-configuration \
    BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true

# Xem/Set bucket policy
aws s3api get-bucket-policy --bucket my-bucket
aws s3api put-bucket-policy --bucket my-bucket --policy file://policy.json
aws s3api delete-bucket-policy --bucket my-bucket

# Xem/Set encryption
aws s3api get-bucket-encryption --bucket my-bucket
aws s3api put-bucket-encryption --bucket my-bucket \
  --server-side-encryption-configuration '{
    "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}]
  }'

# Upload với encryption cụ thể
aws s3 cp myfile.txt s3://my-bucket/ --sse aws:kms --sse-kms-key-id alias/my-key

# Tạo presigned URL
aws s3 presign s3://my-bucket/file.pdf --expires-in 3600

# Enable versioning
aws s3api put-bucket-versioning --bucket my-bucket \
  --versioning-configuration Status=Enabled

# List versions
aws s3api list-object-versions --bucket my-bucket --prefix myfile.txt

# Download version cụ thể
aws s3api get-object --bucket my-bucket --key myfile.txt \
  --version-id "abc123" output.txt

# Xóa version cụ thể
aws s3api delete-object --bucket my-bucket --key myfile.txt \
  --version-id "abc123"

# Set lifecycle rules
aws s3api put-bucket-lifecycle-configuration --bucket my-bucket \
  --lifecycle-configuration file://lifecycle.json

# Xem lifecycle rules
aws s3api get-bucket-lifecycle-configuration --bucket my-bucket

# Set replication
aws s3api put-bucket-replication --bucket source-bucket \
  --replication-configuration file://replication.json

# Enable Transfer Acceleration
aws s3api put-bucket-accelerate-configuration --bucket my-bucket \
  --accelerate-configuration Status=Enabled

# Enable static website hosting
aws s3 website s3://my-bucket/ --index-document index.html --error-document error.html

# Set CORS
aws s3api put-bucket-cors --bucket my-bucket --cors-configuration file://cors.json

# Enable access logging
aws s3api put-bucket-logging --bucket my-bucket --bucket-logging-status '{
  "LoggingEnabled": {
    "TargetBucket": "my-logs-bucket",
    "TargetPrefix": "s3-access-logs/my-bucket/"
  }
}'

# Restore object từ Glacier
aws s3api restore-object --bucket my-bucket --key archive/old-data.gz \
  --restore-request '{"Days":7,"GlacierJobParameters":{"Tier":"Standard"}}'

# Kiểm tra restore status
aws s3api head-object --bucket my-bucket --key archive/old-data.gz

# S3 Select (query CSV trực tiếp)
aws s3api select-object-content \
  --bucket my-bucket \
  --key data/sales.csv \
  --expression "SELECT s.product, s.revenue FROM s3object s WHERE s.revenue > '1000'" \
  --expression-type SQL \
  --input-serialization '{"CSV":{"FileHeaderInfo":"USE"}}' \
  --output-serialization '{"CSV":{}}' \
  output.csv

# Tăng concurrent requests (mặc định: 10)
aws configure set default.s3.max_concurrent_requests 20

# Set multipart threshold (mặc định: 8MB)
aws configure set default.s3.multipart_threshold 100MB

# Set multipart chunk size (mặc định: 8MB)
aws configure set default.s3.multipart_chunksize 50MB

# Set max queue size
aws configure set default.s3.max_queue_size 10000

# Xem cấu hình hiện tại
aws configure get default.s3.max_concurrent_requests

# Dùng profile cụ thể
aws s3 ls --profile production

# Dry run (xem sẽ làm gì mà không thực hiện)
aws s3 sync ./local/ s3://my-bucket/ --dryrun