From 7b5012191639d0769514da70c80a92a52485c48f Mon Sep 17 00:00:00 2001 From: Chris Kalafarski Date: Mon, 14 Oct 2024 09:34:48 -0400 Subject: [PATCH] Add basic report function --- src/alarm-slack-report/index.js | 200 +++++++++++++++++++++++++++++ src/alarm-slack-report/regions.mjs | 52 ++++++++ src/alarm-slack-report/urls.mjs | 25 ++++ template.yml | 82 ++++++++++++ 4 files changed, 359 insertions(+) create mode 100644 src/alarm-slack-report/index.js create mode 100644 src/alarm-slack-report/regions.mjs create mode 100644 src/alarm-slack-report/urls.mjs diff --git a/src/alarm-slack-report/index.js b/src/alarm-slack-report/index.js new file mode 100644 index 0000000..f613c8f --- /dev/null +++ b/src/alarm-slack-report/index.js @@ -0,0 +1,200 @@ +/** @typedef { import('@aws-sdk/client-cloudwatch').AlarmType } AlarmType */ +/** @typedef { import('@aws-sdk/client-cloudwatch').StateValue } StateValue */ + +import { STSClient, AssumeRoleCommand } from "@aws-sdk/client-sts"; +import { + CloudWatchClient, + DescribeAlarmsCommand, +} from "@aws-sdk/client-cloudwatch"; +import { + EventBridgeClient, + PutEventsCommand, +} from "@aws-sdk/client-eventbridge"; +import regions from "./regions.mjs"; +import { alarmConsole, ssoDeepLink } from "./urls.mjs"; + +const sts = new STSClient({ apiVersion: "2011-06-15" }); +const eventbridge = new EventBridgeClient({ apiVersion: "2015-10-07" }); + +async function cloudWatchClient(accountId, region) { + const roleName = process.env.CLOUDWATCH_CROSS_ACCOUNT_SHARING_ROLE_NAME; + + const role = await sts.send( + new AssumeRoleCommand({ + RoleArn: `arn:aws:iam::${accountId}:role/${roleName}`, + RoleSessionName: "reminders_lambda_reader", + }), + ); + + return new CloudWatchClient({ + apiVersion: "2010-08-01", + region, + credentials: { + accessKeyId: role.Credentials.AccessKeyId, + secretAccessKey: role.Credentials.SecretAccessKey, + sessionToken: role.Credentials.SessionToken, + }, + }); +} + +/** + * + * @param {CloudWatchClient} cwClient + * @param {string} nextToken + * @returns {Promise} + */ +async function describeAllAlarms(cwClient, nextToken) { + /** @type {AlarmType[]} */ + const alarmTypes = ["CompositeAlarm", "MetricAlarm"]; + + /** @type {StateValue} */ + const stateValue = "ALARM"; + + const params = { + StateValue: stateValue, + AlarmTypes: alarmTypes, + ...(nextToken && { NextToken: nextToken }), + }; + + const data = await cwClient.send(new DescribeAlarmsCommand(params)); + + const results = { + CompositeAlarms: [], + MetricAlarms: [], + }; + + if (data.CompositeAlarms) { + results.CompositeAlarms.push(...data.CompositeAlarms); + } + + if (data.MetricAlarms) { + results.MetricAlarms.push(...data.MetricAlarms); + } + + if (data.NextToken) { + const more = await describeAllAlarms(cwClient, data.NextToken); + + if (more) { + results.CompositeAlarms.push(...more.CompositeAlarms); + results.MetricAlarms.push(...more.MetricAlarms); + } + } + + return results; +} + +function cleanName(alarmName) { + return alarmName + .replace(/>/g, ">") + .replace(/ { + console.log(JSON.stringify(event)); + + const alarms = { + CompositeAlarms: [], + MetricAlarms: [], + }; + + // eslint-disable-next-line no-restricted-syntax + for (const accountId of process.env.SEARCH_ACCOUNTS.split(",")) { + // eslint-disable-next-line no-restricted-syntax + for (const region of process.env.SEARCH_REGIONS.split(",")) { + // eslint-disable-next-line no-await-in-loop + const cloudwatch = await cloudWatchClient(accountId, region); + + // eslint-disable-next-line no-await-in-loop + const data = await describeAllAlarms(cloudwatch, undefined); + + alarms.CompositeAlarms.push(...data.CompositeAlarms); + alarms.MetricAlarms.push(...data.MetricAlarms.filter(filterByName)); + } + } + + console.log(JSON.stringify(alarms)); + + const count = alarms.CompositeAlarms.length + alarms.MetricAlarms.length; + + const blocks = []; + + if (count === 0) { + return; + } + + blocks.push({ + type: "header", + text: { + type: "plain_text", + text: ":memo: 24-Hour Alarm Report", + emoji: true, + }, + }); + + // blocks.push( + // ...alarms.MetricAlarms.map((a) => { + // const accountId = a.AlarmArn.split(":")[4]; + // const url = alarmConsole(a); + // const ssoUrl = ssoDeepLink(accountId, url); + + // const lines = [`*<${ssoUrl}|${title(a)}>*`]; + + // if (a.StateReasonData) { + // const reasonData = JSON.parse(a.StateReasonData); + // lines.push(started(reasonData)); + // } + + // return { + // type: "section", + // text: { + // type: "mrkdwn", + // text: lines.join("\n"), + // }, + // }; + // }), + // ); + + await eventbridge.send( + new PutEventsCommand({ + Entries: [ + { + Source: "org.prx.cloudwatch-alarm-reminders", + DetailType: "Slack Message Relay Message Payload", + Detail: JSON.stringify({ + username: "Amazon CloudWatch Alarms", + icon_emoji: ":ops-cloudwatch-alarm:", + // channel: "G2QH6NMEH", // #ops-error + channel: "CHZTAGBM2", // #sandbox2 + attachments: [ + { + color: "#a30200", + fallback: `tktktk`, + blocks, + }, + ], + }), + }, + ], + }), + ); +}; diff --git a/src/alarm-slack-report/regions.mjs b/src/alarm-slack-report/regions.mjs new file mode 100644 index 0000000..b1975a4 --- /dev/null +++ b/src/alarm-slack-report/regions.mjs @@ -0,0 +1,52 @@ +/** + * @param {String} region - A region (e.g., us-east-1) + * @returns {String} A region descriptor (e.g., Ohio) + */ +export default function regions(region) { + switch (region) { + case "us-east-1": + return "N. Virginia"; + case "us-east-2": + return "Ohio"; + case "us-west-1": + return "N. California"; + case "us-west-2": + return "Oregon"; + case "af-south-1": + return "Cape Town"; + case "ap-east-1": + return "Hong Kong"; + case "ap-south-1": + return "Mumbai"; + case "ap-northeast-3": + return "Osaka"; + case "ap-northeast-2": + return "Seoul"; + case "ap-southeast-1": + return "Singapore"; + case "ap-southeast-2": + return "Sydney"; + case "ap-northeast-1": + return "Tokyo"; + case "ca-central-1": + return "C. Canada"; + case "eu-central-1": + return "Frankfurt"; + case "eu-west-1": + return "Ireland"; + case "eu-west-2": + return "London"; + case "eu-south-1": + return "Milan"; + case "eu-west-3": + return "Paris"; + case "eu-north-1": + return "Stockholm"; + case "me-south-1": + return "Bahrain"; + case "sa-east-1": + return "São Paulo"; + default: + return region; + } +} diff --git a/src/alarm-slack-report/urls.mjs b/src/alarm-slack-report/urls.mjs new file mode 100644 index 0000000..b9a8bac --- /dev/null +++ b/src/alarm-slack-report/urls.mjs @@ -0,0 +1,25 @@ +/** + * Creates a deep link to an AWS Console URL in a specific account, using an + * IAM Identity Center access role + * @param {String} accountId + * @param {String} url + * @returns + */ +export function ssoDeepLink(accountId, url) { + const deepLinkRoleName = "AdministratorAccess"; + const urlEncodedUrl = encodeURIComponent(url); + return `https://d-906713e952.awsapps.com/start/#/console?account_id=${accountId}&role_name=${deepLinkRoleName}&destination=${urlEncodedUrl}`; +} + +/** + * Returns a URL to CloudWatch Alarms console for the alarm that triggered + * the event. + * @param {*} alarmDetail + * @returns {String} + */ +export function alarmConsole(alarmDetail) { + const name = alarmDetail.AlarmName; + const region = alarmDetail.AlarmArn.split(":")[3]; + const encoded = encodeURI(name.replace(/ /g, "+")).replace(/%/g, "$"); + return `https://console.aws.amazon.com/cloudwatch/home?region=${region}#alarmsV2:alarm/${encoded}`; +} diff --git a/template.yml b/template.yml index 3a29dff..6a3536b 100644 --- a/template.yml +++ b/template.yml @@ -301,3 +301,85 @@ Resources: - { Key: prx:dev:application, Value: CloudWatch Toolkit } Threshold: 0 TreatMissingData: notBreaching + + # Scans certain accounts and regions for all alarms, and sends a report + # message to Slack with information about any alarms that were active recently + AlarmSlackReportFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: src/alarm-slack-report + Description: >- + Scans a set of accounts and regions for active alarms, + and sends a summary to Slack + Environment: + Variables: + AWS_NODEJS_CONNECTION_REUSE_ENABLED: "1" + CLOUDWATCH_CROSS_ACCOUNT_SHARING_ROLE_NAME: !Ref CloudWatchCrossAccountSharingRoleName + SEARCH_REGIONS: !Join [",", !Ref AlarmReminderSearchRegions] + SEARCH_ACCOUNTS: !Join [",", !Ref AlarmReminderSearchAccountIds] + Events: + WeekdayPoller: + Properties: + Description: >- + Invokes the CloudWatch Alarm report function + Enabled: true + Schedule: cron(0 16 ? * * *) + Type: Schedule + Handler: index.handler + MemorySize: 192 + Policies: + - Statement: + - Action: events:PutEvents + Effect: Allow + Resource: !Sub arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:event-bus/default + Version: "2012-10-17" + - Statement: + - Action: sts:AssumeRole + Effect: Allow + Resource: !Sub arn:aws:iam::*:role/${CloudWatchCrossAccountSharingRoleName} + Version: "2012-10-17" + Runtime: nodejs20.x + Tags: + prx:meta:tagging-version: "2021-04-07" + prx:cloudformation:stack-name: !Ref AWS::StackName + prx:cloudformation:stack-id: !Ref AWS::StackId + prx:ops:environment: Production + prx:dev:application: CloudWatch Toolkit + Timeout: 60 + AlarmSlackReportLogGroup: + Type: AWS::Logs::LogGroup + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + LogGroupName: !Sub /aws/lambda/${AlarmSlackReportFunction} + RetentionInDays: 14 + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:ops:environment, Value: Production } + - { Key: prx:dev:application, Value: CloudWatch Toolkit } + AlarmSlackReportErrorAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: "MINOR [CloudWatch] Alarm Slack Report EXPERIENCING ERRORS" + AlarmDescription: >- + Errors are occurring on the CloudWatch Alarm report Lambda function, + so those reports may not be reaching Slack. + ComparisonOperator: GreaterThanThreshold + Dimensions: + - Name: FunctionName + Value: !Ref AlarmSlackReportFunction + EvaluationPeriods: 1 + MetricName: Errors + Namespace: AWS/Lambda + Period: 60 + Statistic: Sum + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:ops:environment, Value: Production } + - { Key: prx:dev:application, Value: CloudWatch Toolkit } + Threshold: 0 + TreatMissingData: notBreaching