Amazon CloudWatch

# Đẩy custom metric
aws cloudwatch put-metric-data \
  --namespace "MyApp/Production" \
  --metric-name "ActiveUsers" \
  --value 142 \
  --unit Count \
  --dimensions Environment=Production,Service=WebApp

# High-resolution metric (storage-resolution = 1 giây)
aws cloudwatch put-metric-data \
  --namespace "MyApp/Production" \
  --metric-name "OrdersPerSecond" \
  --value 85 \
  --unit Count \
  --storage-resolution 1

# Error Rate (%) = Errors / Total Requests * 100
METRICS("m1", "AWS/ApplicationELB", "HTTPCode_Target_5XX_Count")
METRICS("m2", "AWS/ApplicationELB", "RequestCount")
Expression: (m1 / m2) * 100

# Anomaly Detection Band
ANOMALY_DETECTION_BAND(m1, 2)  # 2 standard deviations

# SEARCH expression — tìm metrics động
SEARCH('{AWS/EC2,InstanceId} MetricName="CPUUtilization"', 'Average', 300)

# Alarm khi CPU > 80% trong 2 evaluation periods liên tiếp (mỗi period 5 phút)
aws cloudwatch put-metric-alarm \
  --alarm-name "HighCPU-WebServer" \
  --metric-name CPUUtilization \
  --namespace AWS/EC2 \
  --statistic Average \
  --period 300 \
  --threshold 80 \
  --comparison-operator GreaterThanThreshold \
  --evaluation-periods 2 \
  --datapoints-to-alarm 2 \
  --alarm-actions arn:aws:sns:ap-southeast-1:123456789:AlertTeam \
  --dimensions Name=InstanceId,Value=i-0abcd1234efgh5678 \
  --treat-missing-data notBreaching

# Chỉ trigger khi CẢ HAI alarm đều ở trạng thái ALARM
aws cloudwatch put-composite-alarm \
  --alarm-name "Critical-WebApp" \
  --alarm-rule 'ALARM("HighCPU-WebServer") AND ALARM("High5xxErrors")' \
  --alarm-actions arn:aws:sns:ap-southeast-1:123456789:CriticalAlerts \
  --insufficient-data-actions []

# Tìm tất cả dòng chứa "ERROR"
ERROR

# Tìm dòng chứa cả "ERROR" và "timeout"
ERROR timeout

# Tìm dòng KHÔNG chứa "INFO"
-INFO

# JSON filter — tìm log có statusCode = 500
{ $.statusCode = 500 }

# JSON filter — tìm log có latency > 1000ms
{ $.latency > 1000 }

# JSON filter — kết hợp điều kiện
{ $.statusCode = 5* && $.latency > 2000 }

# Space-delimited filter
[ip, user, timestamp, request, status_code = 5*, bytes]

# Tạo metric filter đếm số lỗi 5xx
aws logs put-metric-filter \
  --log-group-name "/aws/lambda/my-api" \
  --filter-name "5xxErrors" \
  --filter-pattern '{ $.statusCode = 5* }' \
  --metric-transformations \
    metricName=Http5xxCount,metricNamespace=MyApp,metricValue=1,defaultValue=0

# 1. Tìm 25 log events gần nhất
fields @timestamp, @message
| sort @timestamp desc
| limit 25

# 2. Đếm số lỗi theo mỗi 5 phút
filter @message like /ERROR/
| stats count(*) as errorCount by bin(5m)

# 3. Top 10 IP có nhiều request nhất (Apache/Nginx logs)
parse @message "* - - [*] \"* * *\" * *" as ip, time, method, url, protocol, status, bytes
| stats count(*) as requestCount by ip
| sort requestCount desc
| limit 10

# 4. Phân tích latency percentiles (JSON logs)
filter ispresent(latency)
| stats avg(latency) as avgLatency,
        min(latency) as minLatency,
        max(latency) as maxLatency,
        pct(latency, 95) as p95,
        pct(latency, 99) as p99
  by bin(15m)

# 5. Tìm Lambda cold starts
filter @type = "REPORT"
| parse @message "Init Duration: * ms" as initDuration
| filter ispresent(initDuration)
| stats count(*) as coldStarts, avg(initDuration) as avgInitMs by bin(1h)

# 6. Tìm các exception phổ biến nhất
filter @message like /Exception/
| parse @message "*Exception*" as exType
| stats count(*) as cnt by exType
| sort cnt desc
| limit 10

# Cách 1: Dùng SSM (khuyến nghị)
aws ssm send-command \
  --document-name "AWS-ConfigureAWSPackage" \
  --parameters '{"action":["Install"],"name":["AmazonCloudWatchAgent"]}' \
  --targets "Key=instanceids,Values=i-0abc123"

# Cách 2: Cài thủ công
sudo yum install -y amazon-cloudwatch-agent

# Cách 3: Download trực tiếp
wget https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm
sudo rpm -U ./amazon-cloudwatch-agent.rpm

{
  "agent": {
    "metrics_collection_interval": 60,
    "run_as_user": "cwagent"
  },
  "metrics": {
    "namespace": "MyApp/EC2",
    "metrics_collected": {
      "mem": {
        "measurement": ["mem_used_percent", "mem_available_percent"],
        "metrics_collection_interval": 30
      },
      "disk": {
        "measurement": ["disk_used_percent", "disk_free"],
        "resources": ["/", "/data"],
        "metrics_collection_interval": 60
      },
      "swap": {
        "measurement": ["swap_used_percent"]
      },
      "cpu": {
        "measurement": ["cpu_usage_idle", "cpu_usage_user", "cpu_usage_system"],
        "totalcpu": true,
        "metrics_collection_interval": 30
      },
      "netstat": {
        "measurement": ["tcp_established", "tcp_time_wait"],
        "metrics_collection_interval": 60
      }
    },
    "append_dimensions": {
      "InstanceId": "${aws:InstanceId}",
      "AutoScalingGroupName": "${aws:AutoScalingGroupName}"
    }
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/messages",
            "log_group_name": "/ec2/system/messages",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 30
          },
          {
            "file_path": "/var/log/myapp/*.log",
            "log_group_name": "/ec2/myapp",
            "log_stream_name": "{instance_id}/{file_name}",
            "retention_in_days": 14,
            "multi_line_start_pattern": "^\\d{4}-\\d{2}-\\d{2}"
          }
        ]
      }
    }
  }
}

# Dùng wizard tạo config (interactive)
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-config-wizard

# Khởi động agent với config file
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a fetch-config \
  -m ec2 \
  -s \
  -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json

# Kiểm tra trạng thái
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a status

# Dừng agent
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a stop

# Lưu config vào SSM
aws ssm put-parameter \
  --name "AmazonCloudWatch-linux-config" \
  --type String \
  --value file://amazon-cloudwatch-agent.json

# Khởi động agent từ SSM config
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a fetch-config \
  -m ec2 \
  -s \
  -c ssm:AmazonCloudWatch-linux-config

# Tạo dashboard bằng JSON definition
aws cloudwatch put-dashboard \
  --dashboard-name "Production-Overview" \
  --dashboard-body '{
    "widgets": [
      {
        "type": "metric",
        "x": 0, "y": 0, "width": 12, "height": 6,
        "properties": {
          "metrics": [
            ["AWS/EC2", "CPUUtilization", "InstanceId", "i-0abc123"]
          ],
          "period": 300,
          "stat": "Average",
          "region": "ap-southeast-1",
          "title": "EC2 CPU Utilization"
        }
      },
      {
        "type": "text",
        "x": 0, "y": 6, "width": 12, "height": 2,
        "properties": {
          "markdown": "## 🟢 Production Dashboard\nCập nhật real-time"
        }
      }
    ]
  }'

{
  "source": ["aws.ec2"],
  "detail-type": ["EC2 Instance State-change Notification"],
  "detail": {
    "state": ["terminated", "stopped"]
  }
}

# Rule: Gửi SNS khi EC2 instance bị terminated
aws events put-rule \
  --name "EC2TerminationAlert" \
  --event-pattern '{
    "source": ["aws.ec2"],
    "detail-type": ["EC2 Instance State-change Notification"],
    "detail": {"state": ["terminated"]}
  }' \
  --state ENABLED

# Thêm target (SNS topic)
aws events put-targets \
  --rule "EC2TerminationAlert" \
  --targets "Id"="1","Arn"="arn:aws:sns:ap-southeast-1:123456789:Alerts"

# Schedule rule: Chạy Lambda mỗi 5 phút
aws events put-rule \
  --name "HealthCheck-Every5Min" \
  --schedule-expression "rate(5 minutes)" \
  --state ENABLED

const synthetics = require('Synthetics');
const log = require('SyntheticsLogger');

const pageLoadBlueprint = async function () {
  const page = await synthetics.getPage();

  // Navigate đến URL
  const response = await page.goto('https://myapp.example.com', {
    waitUntil: 'domcontentloaded',
    timeout: 30000
  });

  // Kiểm tra status code
  if (response.status() !== 200) {
    throw new Error(`Expected 200, got ${response.status()}`);
  }

  // Chụp screenshot
  await synthetics.takeScreenshot('homepage', 'loaded');

  // Kiểm tra element tồn tại
  await page.waitForSelector('#main-content', { timeout: 5000 });

  log.info('Page loaded successfully');
};

exports.handler = async () => {
  return await pageLoadBlueprint();
};

# Tạo canary chạy mỗi 5 phút
aws synthetics create-canary \
  --name "api-health-check" \
  --artifact-s3-location "s3://my-canary-artifacts/api-health/" \
  --execution-role-arn "arn:aws:iam::123456789:role/CanaryRole" \
  --schedule '{"Expression":"rate(5 minutes)"}' \
  --runtime-version "syn-nodejs-puppeteer-6.2" \
  --code '{"Handler":"index.handler","ZipFile":"..."}'

# Bắt đầu canary
aws synthetics start-canary --name "api-health-check"

# ECS — bật khi tạo cluster
aws ecs create-cluster \
  --cluster-name my-cluster \
  --settings name=containerInsights,value=enabled

# ECS — bật cho cluster hiện có
aws ecs update-cluster-settings \
  --cluster my-cluster \
  --settings name=containerInsights,value=enabled

# EKS — cài CloudWatch Agent (DaemonSet)
# Dùng AWS Distro for OpenTelemetry (ADOT) hoặc Fluent Bit
kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonSet/container-insights-monitoring/quickstart/cwagent-fluent-bit-quickstart.yaml

# Top 10 pods sử dụng CPU nhiều nhất
stats avg(CpuUtilized) as avgCPU by PodName
| sort avgCPU desc
| limit 10

# Tìm pods bị OOMKilled
filter @message like /OOMKilled/
| fields @timestamp, PodName, @message
| sort @timestamp desc

# Tạo application monitoring
aws application-insights create-application \
  --resource-group-name "MyWebApp-Resources" \
  --ops-center-enabled \
  --auto-config-enabled

# Liệt kê problems được phát hiện
aws application-insights list-problems \
  --resource-group-name "MyWebApp-Resources" \
  --start-time 2024-01-01T00:00:00Z \
  --end-time 2024-01-31T23:59:59Z

# Xem chi tiết problem
aws application-insights describe-problem \
  --problem-id "p-abc123def456"

# Liệt kê tất cả metrics trong namespace
aws cloudwatch list-metrics --namespace "AWS/EC2"

# Liệt kê metrics theo dimension
aws cloudwatch list-metrics \
  --namespace "AWS/EC2" \
  --dimensions Name=InstanceId,Value=i-0abc123

# Lấy metric statistics
aws cloudwatch get-metric-statistics \
  --namespace "AWS/EC2" \
  --metric-name CPUUtilization \
  --dimensions Name=InstanceId,Value=i-0abc123 \
  --start-time 2024-01-15T00:00:00Z \
  --end-time 2024-01-15T23:59:59Z \
  --period 3600 \
  --statistics Average Maximum

# Đẩy custom metric
aws cloudwatch put-metric-data \
  --namespace "MyApp" \
  --metric-name "ActiveSessions" \
  --value 42 \
  --unit Count \
  --dimensions App=WebServer,Env=Prod

# Lấy metric data (mới hơn, hỗ trợ Metric Math)
aws cloudwatch get-metric-data \
  --metric-data-queries '[
    {
      "Id": "cpu",
      "MetricStat": {
        "Metric": {
          "Namespace": "AWS/EC2",
          "MetricName": "CPUUtilization",
          "Dimensions": [{"Name":"InstanceId","Value":"i-0abc123"}]
        },
        "Period": 300,
        "Stat": "Average"
      }
    }
  ]' \
  --start-time 2024-01-15T00:00:00Z \
  --end-time 2024-01-15T12:00:00Z

# Liệt kê tất cả alarms
aws cloudwatch describe-alarms

# Liệt kê alarms đang ở trạng thái ALARM
aws cloudwatch describe-alarms --state-value ALARM

# Tạo alarm
aws cloudwatch put-metric-alarm \
  --alarm-name "HighCPU" \
  --namespace "AWS/EC2" \
  --metric-name CPUUtilization \
  --statistic Average \
  --period 300 \
  --evaluation-periods 3 \
  --datapoints-to-alarm 2 \
  --threshold 85 \
  --comparison-operator GreaterThanThreshold \
  --alarm-actions arn:aws:sns:ap-southeast-1:123456789:Alerts \
  --ok-actions arn:aws:sns:ap-southeast-1:123456789:Alerts \
  --dimensions Name=InstanceId,Value=i-0abc123 \
  --treat-missing-data notBreaching

# Test alarm (set state thủ công)
aws cloudwatch set-alarm-state \
  --alarm-name "HighCPU" \
  --state-value ALARM \
  --state-reason "Testing alarm notification"

# Xóa alarm
aws cloudwatch delete-alarms --alarm-names "HighCPU"

# Tắt alarm actions (không xóa alarm)
aws cloudwatch disable-alarm-actions --alarm-names "HighCPU"

# Bật lại alarm actions
aws cloudwatch enable-alarm-actions --alarm-names "HighCPU"

# Liệt kê log groups
aws logs describe-log-groups

# Liệt kê log streams trong group
aws logs describe-log-streams \
  --log-group-name "/aws/lambda/my-function" \
  --order-by LastEventTime \
  --descending

# Lấy log events
aws logs get-log-events \
  --log-group-name "/aws/lambda/my-function" \
  --log-stream-name "2024/01/15/[$LATEST]abc123"

# Tìm kiếm logs (filter)
aws logs filter-log-events \
  --log-group-name "/aws/lambda/my-function" \
  --filter-pattern "ERROR" \
  --start-time 1705276800000 \
  --end-time 1705363200000

# Tạo log group với retention
aws logs create-log-group --log-group-name "/myapp/production"
aws logs put-retention-policy \
  --log-group-name "/myapp/production" \
  --retention-in-days 30

# Xóa log group
aws logs delete-log-group --log-group-name "/myapp/old-logs"

# Tạo metric filter
aws logs put-metric-filter \
  --log-group-name "/myapp/production" \
  --filter-name "ErrorCount" \
  --filter-pattern "ERROR" \
  --metric-transformations \
    metricName=AppErrors,metricNamespace=MyApp,metricValue=1

# Export logs sang S3
aws logs create-export-task \
  --log-group-name "/myapp/production" \
  --from 1705276800000 \
  --to 1705363200000 \
  --destination "my-log-archive-bucket" \
  --destination-prefix "exports/myapp"

# Tail logs real-time (cần aws cli v2)
aws logs tail "/aws/lambda/my-function" --follow --since 1h

# Liệt kê dashboards
aws cloudwatch list-dashboards

# Lấy dashboard definition
aws cloudwatch get-dashboard --dashboard-name "Production"

# Tạo/cập nhật dashboard
aws cloudwatch put-dashboard \
  --dashboard-name "Production" \
  --dashboard-body file://dashboard.json

# Xóa dashboard
aws cloudwatch delete-dashboards --dashboard-names "OldDashboard"

# Liệt kê rules
aws events list-rules

# Tạo rule với event pattern
aws events put-rule \
  --name "S3UploadAlert" \
  --event-pattern '{
    "source": ["aws.s3"],
    "detail-type": ["Object Created"],
    "detail": {
      "bucket": {"name": ["my-important-bucket"]}
    }
  }'

# Tạo scheduled rule
aws events put-rule \
  --name "DailyCleanup" \
  --schedule-expression "cron(0 2 * * ? *)" \
  --description "Chạy cleanup lúc 2h sáng mỗi ngày"

# Thêm target cho rule
aws events put-targets \
  --rule "S3UploadAlert" \
  --targets '[{
    "Id": "1",
    "Arn": "arn:aws:lambda:ap-southeast-1:123456789:function:ProcessUpload"
  }]'

# Test event pattern
aws events test-event-pattern \
  --event-pattern '{"source":["aws.ec2"]}' \
  --event '{"source":"aws.ec2","detail-type":"EC2 Instance State-change Notification"}'

# Gửi custom event
aws events put-events \
  --entries '[{
    "Source": "myapp.orders",
    "DetailType": "Order Created",
    "Detail": "{\"orderId\":\"12345\",\"amount\":99.99}",
    "EventBusName": "default"
  }]'

# Xóa rule (phải xóa targets trước)
aws events remove-targets --rule "OldRule" --ids "1"
aws events delete-rule --name "OldRule"

# Liệt kê canaries
aws synthetics describe-canaries

# Lấy kết quả chạy gần nhất
aws synthetics get-canary-runs --name "my-canary" --max-results 5

# Bắt đầu / dừng canary
aws synthetics start-canary --name "my-canary"
aws synthetics stop-canary --name "my-canary"

# Xóa canary
aws synthetics delete-canary --name "old-canary"

# Chạy query
aws logs start-query \
  --log-group-name "/aws/lambda/my-function" \
  --start-time 1705276800 \
  --end-time 1705363200 \
  --query-string 'fields @timestamp, @message | filter @message like /ERROR/ | sort @timestamp desc | limit 20'

# Lấy kết quả query (dùng queryId từ start-query)
aws logs get-query-results --query-id "abc123-def456-ghi789"

# Liệt kê saved queries
aws logs describe-query-definitions