diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/README.md b/cloudwatch-log-alarm-lambda-remediation-cdk/README.md new file mode 100644 index 000000000..65116c2bf --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/README.md @@ -0,0 +1,119 @@ +# Amazon CloudWatch Log Alarm with AWS Lambda Auto-Remediation + +This pattern deploys a self-healing architecture using the new Amazon CloudWatch Log Alarm resource (`AWS::CloudWatch::LogAlarm`) to monitor application logs with CloudWatch Logs Insights queries and automatically trigger remediation through AWS Lambda and AWS Systems Manager when error thresholds are breached. + +Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/cloudwatch-log-alarm-lambda-remediation-cdk + +## Architecture + +``` +┌─────────────────────┐ ┌──────────────────────────┐ ┌─────────────┐ ┌──────────────┐ ┌──────────────────┐ +│ Amazon CloudWatch │────▶│ Amazon CloudWatch │────▶│ Amazon SNS │────▶│ AWS Lambda │────▶│ AWS Systems │ +│ Logs │ │ Log Alarm │ │ │ │ (Remediation)│ │ Manager │ +│ (Application Logs) │ │ (Logs Insights Query) │ │ │ │ │ │ (RunCommand) │ +└─────────────────────┘ └──────────────────────────┘ └─────────────┘ └──────────────┘ └──────────────────┘ +``` + +**How it works:** + +1. Application logs are written to Amazon CloudWatch Logs +2. The Amazon CloudWatch Log Alarm runs a CloudWatch Logs Insights query every 5 minutes to count ERROR messages +3. When error count ≥ 5 in a single evaluation window, the alarm transitions to ALARM state +4. The alarm publishes to Amazon SNS, which invokes the AWS Lambda remediation function +5. AWS Lambda sends a command via AWS Systems Manager to restart the application service on tagged Amazon EC2 instances + +**Key innovation:** The `AWS::CloudWatch::LogAlarm` resource (launched July 2026) eliminates the need for metric filters. Previously, monitoring log content required creating a CloudWatch Metric Filter, waiting for metric data points, then creating a standard alarm on that metric. Log Alarms run Logs Insights queries directly on schedule and evaluate results against thresholds — a single resource replaces a three-resource workaround. + +## Requirements + +- [AWS CDK v2](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html) installed and configured +- [Node.js 20+](https://nodejs.org/) with npm +- AWS account [bootstrapped for CDK](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping.html) +- Python 3.12 (for AWS Lambda functions) +- (Optional) Amazon EC2 instances tagged with `AutoRemediate=true` for SSM remediation + +## Deployment + +### Step 1: Install dependencies and synthesize + +```bash +cd cloudwatch-log-alarm-lambda-remediation-cdk/cdk +npm install +npx cdk synth +``` + +### Step 2: Deploy the stack + +```bash +npx cdk deploy +``` + +## Testing + +### 1. Generate error logs to trigger the alarm + +```bash +LOG_GROUP="/app/monitored-service" + +# Write ERROR messages to trigger the alarm (threshold = 5) +for i in $(seq 1 6); do + aws logs put-log-events \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "test-stream" \ + --log-events "[{\"timestamp\":$(date +%s000),\"message\":\"ERROR: Connection timeout to database (attempt $i)\"}]" \ + --sequence-token "$(aws logs describe-log-streams --log-group-name $LOG_GROUP --log-stream-name-prefix test --query 'logStreams[0].uploadSequenceToken' --output text 2>/dev/null)" + sleep 1 +done + +echo "Wrote 6 ERROR messages. Alarm will evaluate in ~5 minutes." +``` + +### 2. Check alarm state (after 5 minutes) + +```bash +aws cloudwatch describe-alarms \ + --alarm-names "app-error-rate-alarm" \ + --alarm-types "LogAlarm" \ + --query 'LogAlarms[0].{State:StateValue,Reason:StateReason}' +``` + +### 3. Verify AWS Lambda was invoked + +```bash +aws logs filter-log-events \ + --log-group-name "/aws/lambda/CloudwatchLogAlarmLambdaRem-RemediationFn*" \ + --filter-pattern "Executing remediation" \ + --query 'events[].message' +``` + +### 4. Check AWS Systems Manager command history + +```bash +aws ssm list-commands \ + --filters "key=DocumentName,value=AWS-RunShellScript" \ + --max-results 5 \ + --query 'Commands[].{Id:CommandId,Status:Status,Comment:Comment}' +``` + +## Cleanup + +> **Warning:** This will delete the log group and all log data. The Amazon CloudWatch Log Alarm and all associated resources will be removed. + +```bash +cd cloudwatch-log-alarm-lambda-remediation-cdk/cdk +npx cdk destroy +``` + +## Services Used + +| Service | Role | +|---------|------| +| Amazon CloudWatch Logs | Application log storage and query target | +| Amazon CloudWatch Log Alarm | Scheduled Logs Insights query with threshold evaluation | +| Amazon SNS | Alarm notification delivery to subscribers | +| AWS Lambda | Auto-remediation logic (classify alarm, invoke SSM) | +| AWS Systems Manager | Execute commands on tagged Amazon EC2 instances | + +---- +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: MIT-0 diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/.gitignore b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/.gitignore new file mode 100644 index 000000000..1f0ea0352 --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/.gitignore @@ -0,0 +1,6 @@ +node_modules +cdk.out +cdk.context.json +build +*.js +*.d.ts diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/bin/app.ts b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/bin/app.ts new file mode 100644 index 000000000..6a9edceb0 --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/bin/app.ts @@ -0,0 +1,12 @@ +#!/usr/bin/env node +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 (2026) + +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { CloudwatchLogAlarmLambdaRemediationStack } from '../lib/cloudwatch-log-alarm-lambda-remediation-stack'; + +const app = new cdk.App(); +new CloudwatchLogAlarmLambdaRemediationStack(app, 'CloudwatchLogAlarmLambdaRemediationStack', { + description: 'Amazon CloudWatch Log Alarm with AWS Lambda auto-remediation (uksb-1tupboc57)', +}); diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/cdk.json b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/cdk.json new file mode 100644 index 000000000..a6700a2ff --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/cdk.json @@ -0,0 +1,3 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/app.ts" +} diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/lib/cloudwatch-log-alarm-lambda-remediation-stack.ts b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/lib/cloudwatch-log-alarm-lambda-remediation-stack.ts new file mode 100644 index 000000000..a886559e0 --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/lib/cloudwatch-log-alarm-lambda-remediation-stack.ts @@ -0,0 +1,164 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 (2026) + +import * as cdk from 'aws-cdk-lib'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as sns from 'aws-cdk-lib/aws-sns'; +import * as snsSubscriptions from 'aws-cdk-lib/aws-sns-subscriptions'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import { Construct } from 'constructs'; +import * as path from 'path'; + +export class CloudwatchLogAlarmLambdaRemediationStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + const region = cdk.Stack.of(this).region; + const account = cdk.Stack.of(this).account; + + // ========================================================= + // 1. Amazon CloudWatch Log Group (monitored application logs) + // ========================================================= + const appLogGroup = new logs.LogGroup(this, 'AppLogGroup', { + logGroupName: '/app/monitored-service', + retention: logs.RetentionDays.ONE_WEEK, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + // ========================================================= + // 2. Amazon SNS Topic (alarm notification fan-out) + // ========================================================= + const alarmTopic = new sns.Topic(this, 'AlarmTopic', { + topicName: 'log-alarm-notifications', + displayName: 'Amazon CloudWatch Log Alarm Notifications', + }); + + // ========================================================= + // 3. AWS Lambda: Remediation Handler + // ========================================================= + const remediationFn = new lambda.Function(this, 'RemediationFn', { + runtime: lambda.Runtime.PYTHON_3_12, + handler: 'handler.lambda_handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../../lambdas/remediation')), + timeout: cdk.Duration.seconds(60), + memorySize: 256, + description: 'Auto-remediation: restarts service via AWS Systems Manager when error threshold breached', + environment: { + ALARM_TOPIC_ARN: alarmTopic.topicArn, + }, + }); + + // SSM permissions for remediation (send commands to EC2 instances) + remediationFn.addToRolePolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'ssm:SendCommand', + 'ssm:GetCommandInvocation', + ], + resources: [ + `arn:aws:ssm:${region}::document/AWS-RunShellScript`, + `arn:aws:ec2:${region}:${account}:instance/*`, + ], + conditions: { + StringEquals: { + 'aws:ResourceTag/AutoRemediate': 'true', + }, + }, + })); + + // CloudWatch Logs permissions for context retrieval + remediationFn.addToRolePolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'logs:GetLogEvents', + 'logs:FilterLogEvents', + ], + resources: [appLogGroup.logGroupArn, `${appLogGroup.logGroupArn}:*`], + })); + + // Subscribe Lambda to SNS topic + alarmTopic.addSubscription( + new snsSubscriptions.LambdaSubscription(remediationFn) + ); + + // ========================================================= + // 4. IAM Role for Scheduled Query execution + // ========================================================= + const scheduledQueryRole = new iam.Role(this, 'ScheduledQueryRole', { + assumedBy: new iam.ServicePrincipal('logs.amazonaws.com'), + description: 'Allows Amazon CloudWatch Logs to execute scheduled queries', + inlinePolicies: { + LogsQuery: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'logs:StartQuery', + 'logs:GetQueryResults', + 'logs:StopQuery', + ], + resources: [appLogGroup.logGroupArn, `${appLogGroup.logGroupArn}:*`], + }), + ], + }), + }, + }); + + // ========================================================= + // 5. AWS::CloudWatch::LogAlarm (new CFN resource type) + // Monitors error rate in application logs using Logs Insights + // ========================================================= + const logAlarm = new cdk.CfnResource(this, 'ErrorRateLogAlarm', { + type: 'AWS::CloudWatch::LogAlarm', + properties: { + AlarmName: 'app-error-rate-alarm', + AlarmDescription: 'Triggers when error count exceeds threshold in application logs (monitored via Amazon CloudWatch Logs Insights query)', + ComparisonOperator: 'GreaterThanOrEqualToThreshold', + Threshold: 5, + QueryResultsToAlarm: 1, + QueryResultsToEvaluate: 1, + TreatMissingData: 'notBreaching', + ActionsEnabled: true, + AlarmActions: [alarmTopic.topicArn], + OKActions: [alarmTopic.topicArn], + ScheduledQueryConfiguration: { + QueryString: 'fields @timestamp, @message | filter @message like /ERROR/ | stats count(*) as error_count by bin(5m)', + LogGroupIdentifiers: [appLogGroup.logGroupName], + AggregationExpression: 'count(*)', + ScheduleConfiguration: { + ScheduleExpression: 'rate(5 minutes)', + StartTimeOffset: 300, + }, + ScheduledQueryRoleARN: scheduledQueryRole.roleArn, + }, + }, + }); + + // Ensure role is created before the alarm + logAlarm.addDependency(scheduledQueryRole.node.defaultChild as cdk.CfnResource); + + // ========================================================= + // Outputs + // ========================================================= + new cdk.CfnOutput(this, 'LogGroupName', { + value: appLogGroup.logGroupName, + description: 'Amazon CloudWatch Logs group being monitored', + }); + + new cdk.CfnOutput(this, 'AlarmTopicArn', { + value: alarmTopic.topicArn, + description: 'Amazon SNS topic for alarm notifications', + }); + + new cdk.CfnOutput(this, 'RemediationFunctionArn', { + value: remediationFn.functionArn, + description: 'AWS Lambda remediation function ARN', + }); + + new cdk.CfnOutput(this, 'LogAlarmName', { + value: 'app-error-rate-alarm', + description: 'Amazon CloudWatch Log Alarm name', + }); + } +} diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/package.json b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/package.json new file mode 100644 index 000000000..4ee75f315 --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/package.json @@ -0,0 +1,20 @@ +{ + "name": "cloudwatch-log-alarm-lambda-remediation-cdk", + "version": "1.0.0", + "bin": { "app": "bin/app.ts" }, + "scripts": { + "build": "tsc", + "synth": "cdk synth", + "deploy": "cdk deploy", + "destroy": "cdk destroy" + }, + "dependencies": { + "aws-cdk-lib": "2.185.0", + "constructs": "^10.0.0", + "source-map-support": "^0.5.21" + }, + "devDependencies": { + "typescript": "~5.4.0", + "ts-node": "^10.9.0" + } +} diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/tsconfig.json b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/tsconfig.json new file mode 100644 index 000000000..7ddcfe705 --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/cdk/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["es2020"], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "outDir": "./build", + "rootDir": "." + }, + "exclude": ["node_modules", "build"] +} diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/example-pattern.json b/cloudwatch-log-alarm-lambda-remediation-cdk/example-pattern.json new file mode 100644 index 000000000..b1833be85 --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/example-pattern.json @@ -0,0 +1,101 @@ +{ + "title": "Amazon CloudWatch Log Alarm with AWS Lambda Auto-Remediation", + "description": "Monitor application logs with CloudWatch Logs Insights queries and auto-remediate via AWS Lambda and AWS Systems Manager when error thresholds are breached", + "language": "TypeScript", + "level": "300", + "framework": "AWS CDK", + "introBox": { + "headline": "How it works", + "text": [ + "Amazon CloudWatch Log Alarm runs a CloudWatch Logs Insights query on a schedule to count ERROR messages in application logs.", + "When the error count exceeds the threshold (5 errors in 5 minutes), the alarm transitions to ALARM state and publishes to Amazon SNS.", + "Amazon SNS triggers an AWS Lambda function that sends a remediation command via AWS Systems Manager RunCommand to restart the application service on tagged Amazon EC2 instances." + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/cloudwatch-log-alarm-lambda-remediation-cdk", + "templateURL": "serverless-patterns/cloudwatch-log-alarm-lambda-remediation-cdk", + "projectFolder": "cloudwatch-log-alarm-lambda-remediation-cdk", + "templateFile": "cdk/lib/cloudwatch-log-alarm-lambda-remediation-stack.ts" + } + }, + "resources": { + "bullets": [ + { + "text": "Amazon CloudWatch Log Alarm documentation", + "link": "https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/alarm-log.html" + }, + { + "text": "AWS::CloudWatch::LogAlarm CloudFormation reference", + "link": "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudwatch-logalarm.html" + }, + { + "text": "AWS Systems Manager RunCommand", + "link": "https://docs.aws.amazon.com/systems-manager/latest/userguide/run-command.html" + } + ] + }, + "deploy": { + "text": [ + "cd cloudwatch-log-alarm-lambda-remediation-cdk/cdk", + "npm install", + "npx cdk deploy" + ] + }, + "testing": { + "text": [ + "Write ERROR log messages to the monitored log group to trigger the alarm.", + "Wait 5 minutes for the scheduled query to evaluate.", + "Check alarm state transitions to ALARM.", + "Verify AWS Lambda remediation function was invoked via Amazon CloudWatch Logs." + ] + }, + "cleanup": { + "text": [ + "npx cdk destroy" + ] + }, + "authors": [ + { + "name": "Nithin Chandran R", + "image": "", + "bio": "Technical Account Manager at AWS", + "linkedin": "nithin-chandran-r", + "twitter": "" + } + ], + "patternArch": { + "icon1": { + "x": 10, + "y": 50, + "service": "cloudwatch", + "label": "Amazon CloudWatch Logs" + }, + "icon2": { + "x": 30, + "y": 50, + "service": "cloudwatch", + "label": "Log Alarm" + }, + "icon3": { + "x": 50, + "y": 50, + "service": "sns", + "label": "Amazon SNS" + }, + "icon4": { + "x": 70, + "y": 50, + "service": "lambda", + "label": "AWS Lambda" + }, + "icon5": { + "x": 90, + "y": 50, + "service": "ssm", + "label": "AWS Systems Manager" + } + }, + "patternType": "Serverless" +} diff --git a/cloudwatch-log-alarm-lambda-remediation-cdk/lambdas/remediation/handler.py b/cloudwatch-log-alarm-lambda-remediation-cdk/lambdas/remediation/handler.py new file mode 100644 index 000000000..9b7e1ef45 --- /dev/null +++ b/cloudwatch-log-alarm-lambda-remediation-cdk/lambdas/remediation/handler.py @@ -0,0 +1,76 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 (2026) +"""Auto-remediation handler: Triggered by Amazon CloudWatch Log Alarm via +Amazon SNS. Restarts application service on tagged Amazon EC2 instances +using AWS Systems Manager RunCommand.""" + +import json +import os + +import boto3 + +SSM_CLIENT = boto3.client('ssm') +LOGS_CLIENT = boto3.client('logs') + + +def lambda_handler(event, context): + """Process Amazon SNS notification from Amazon CloudWatch Log Alarm.""" + for record in event.get('Records', []): + try: + sns_message = json.loads(record['Sns']['Message']) + alarm_name = sns_message.get('AlarmName', 'unknown') + new_state = sns_message.get('NewStateValue', 'UNKNOWN') + reason = sns_message.get('NewStateReason', '') + + print(f'Alarm: {alarm_name} | State: {new_state} | Reason: {reason}') + + if new_state == 'ALARM': + remediate(alarm_name, reason) + elif new_state == 'OK': + print(f'Alarm {alarm_name} recovered to OK — no action needed') + + except (json.JSONDecodeError, KeyError) as e: + print(f'Failed to parse SNS message: {e}') + continue + + return {'statusCode': 200, 'message': 'Processed'} + + +def remediate(alarm_name: str, reason: str) -> None: + """Execute remediation via AWS Systems Manager RunCommand.""" + print(f'Executing remediation for alarm: {alarm_name}') + + # Send command to instances tagged with AutoRemediate=true + try: + response = SSM_CLIENT.send_command( + Targets=[ + { + 'Key': 'tag:AutoRemediate', + 'Values': ['true'], + } + ], + DocumentName='AWS-RunShellScript', + Parameters={ + 'commands': [ + '#!/bin/bash', + 'echo "Auto-remediation triggered by CloudWatch Log Alarm"', + f'echo "Alarm: {alarm_name}"', + f'echo "Reason: {reason[:200]}"', + 'systemctl restart application.service || echo "Service restart failed"', + 'echo "Remediation complete at $(date)"', + ], + 'executionTimeout': ['60'], + }, + TimeoutSeconds=120, + Comment=f'Auto-remediation: {alarm_name}', + ) + + command_id = response['Command']['CommandId'] + instance_count = response['Command']['TargetCount'] + print(f'SSM command sent: {command_id} | Targets: {instance_count} instance(s)') + + except SSM_CLIENT.exceptions.InvalidInstanceId: + print('No instances found with tag AutoRemediate=true') + except Exception as e: + print(f'SSM SendCommand failed: {e}') + raise