Add basic report function

src/alarm-slack-report/index.js

@@ -0,0 +1,200 @@
+/** @typedef { import('@aws-sdk/client-cloudwatch').AlarmType } AlarmType */
+/** @typedef { import('@aws-sdk/client-cloudwatch').StateValue } StateValue */
+
+import { STSClient, AssumeRoleCommand } from "@aws-sdk/client-sts";
+import {
+  CloudWatchClient,
+  DescribeAlarmsCommand,
+} from "@aws-sdk/client-cloudwatch";
+import {
+  EventBridgeClient,
+  PutEventsCommand,
+} from "@aws-sdk/client-eventbridge";
+import regions from "./regions.mjs";
+import { alarmConsole, ssoDeepLink } from "./urls.mjs";
+
+const sts = new STSClient({ apiVersion: "2011-06-15" });
+const eventbridge = new EventBridgeClient({ apiVersion: "2015-10-07" });
+
+async function cloudWatchClient(accountId, region) {
+  const roleName = process.env.CLOUDWATCH_CROSS_ACCOUNT_SHARING_ROLE_NAME;
+
+  const role = await sts.send(
+    new AssumeRoleCommand({
+      RoleArn: `arn:aws:iam::${accountId}:role/${roleName}`,
+      RoleSessionName: "reminders_lambda_reader",
+    }),
+  );
+
+  return new CloudWatchClient({
+    apiVersion: "2010-08-01",
+    region,
+    credentials: {
+      accessKeyId: role.Credentials.AccessKeyId,
+      secretAccessKey: role.Credentials.SecretAccessKey,
+      sessionToken: role.Credentials.SessionToken,
+    },
+  });
+}
+
+/**
+ *
+ * @param {CloudWatchClient} cwClient
+ * @param {string} nextToken
+ * @returns {Promise<Object>}
+ */
+async function describeAllAlarms(cwClient, nextToken) {
+  /** @type {AlarmType[]} */
+  const alarmTypes = ["CompositeAlarm", "MetricAlarm"];
+
+  /** @type {StateValue} */
+  const stateValue = "ALARM";
+
+  const params = {
+    StateValue: stateValue,
+    AlarmTypes: alarmTypes,
+    ...(nextToken && { NextToken: nextToken }),
+  };
+
+  const data = await cwClient.send(new DescribeAlarmsCommand(params));
+
+  const results = {
+    CompositeAlarms: [],
+    MetricAlarms: [],
+  };
+
+  if (data.CompositeAlarms) {
+    results.CompositeAlarms.push(...data.CompositeAlarms);
+  }
+
+  if (data.MetricAlarms) {
+    results.MetricAlarms.push(...data.MetricAlarms);
+  }
+
+  if (data.NextToken) {
+    const more = await describeAllAlarms(cwClient, data.NextToken);
+
+    if (more) {
+      results.CompositeAlarms.push(...more.CompositeAlarms);
+      results.MetricAlarms.push(...more.MetricAlarms);
+    }
+  }
+
+  return results;
+}
+
+function cleanName(alarmName) {
+  return alarmName
+    .replace(/>/g, "&gt;")
+    .replace(/</g, "&lt;")
+    .replace(/\([A-Za-z0-9_-]+\)$/, "")
+    .replace(/^(FATAL|ERROR|WARN|INFO|CRITICAL|MAJOR|MINOR)/, "")
+    .trim();
+}
+
+function title(alarmDetail) {
+  const name = alarmDetail.AlarmName;
+  const region = regions(alarmDetail.AlarmArn.split(":")[3]);
+  return `${alarmDetail.StateValue} | ${region} » ${cleanName(name)}`;
+}
+
+function filterByName(alarm) {
+  return !(
+    alarm.AlarmName.includes("AS:In") ||
+    alarm.AlarmName.includes("AS:Out") ||
+    alarm.AlarmName.includes("TargetTracking") ||
+    alarm.AlarmName.includes("ScaleInAlarm") ||
+    alarm.AlarmName.includes("ScaleOutAlarm") ||
+    alarm.AlarmName.includes("Production Pollers Low CPU Usage")
+  );
+}
+
+export const handler = async (event) => {
+  console.log(JSON.stringify(event));
+
+  const alarms = {
+    CompositeAlarms: [],
+    MetricAlarms: [],
+  };
+
+  // eslint-disable-next-line no-restricted-syntax
+  for (const accountId of process.env.SEARCH_ACCOUNTS.split(",")) {
+    // eslint-disable-next-line no-restricted-syntax
+    for (const region of process.env.SEARCH_REGIONS.split(",")) {
+      // eslint-disable-next-line no-await-in-loop
+      const cloudwatch = await cloudWatchClient(accountId, region);
+
+      // eslint-disable-next-line no-await-in-loop
+      const data = await describeAllAlarms(cloudwatch, undefined);
+
+      alarms.CompositeAlarms.push(...data.CompositeAlarms);
+      alarms.MetricAlarms.push(...data.MetricAlarms.filter(filterByName));
+    }
+  }
+
+  console.log(JSON.stringify(alarms));
+
+  const count = alarms.CompositeAlarms.length + alarms.MetricAlarms.length;
+
+  const blocks = [];
+
+  if (count === 0) {
+    return;
+  }
+
+  blocks.push({
+    type: "header",
+    text: {
+      type: "plain_text",
+      text: ":memo: 24-Hour Alarm Report",
+      emoji: true,
+    },
+  });
+
+  // blocks.push(
+  //   ...alarms.MetricAlarms.map((a) => {
+  //     const accountId = a.AlarmArn.split(":")[4];
+  //     const url = alarmConsole(a);
+  //     const ssoUrl = ssoDeepLink(accountId, url);
+
+  //     const lines = [`*<${ssoUrl}|${title(a)}>*`];
+
+  //     if (a.StateReasonData) {
+  //       const reasonData = JSON.parse(a.StateReasonData);
+  //       lines.push(started(reasonData));
+  //     }
+
+  //     return {
+  //       type: "section",
+  //       text: {
+  //         type: "mrkdwn",
+  //         text: lines.join("\n"),
+  //       },
+  //     };
+  //   }),
+  // );
+
+  await eventbridge.send(
+    new PutEventsCommand({
+      Entries: [
+        {
+          Source: "org.prx.cloudwatch-alarm-reminders",
+          DetailType: "Slack Message Relay Message Payload",
+          Detail: JSON.stringify({
+            username: "Amazon CloudWatch Alarms",
+            icon_emoji: ":ops-cloudwatch-alarm:",
+            // channel: "G2QH6NMEH", // #ops-error
+            channel: "CHZTAGBM2", // #sandbox2
+            attachments: [
+              {
+                color: "#a30200",
+                fallback: `tktktk`,
+                blocks,
+              },
+            ],
+          }),
+        },
+      ],
+    }),
+  );
+};

src/alarm-slack-report/regions.mjs

@@ -0,0 +1,52 @@
+/**
+ * @param {String} region - A region (e.g., us-east-1)
+ * @returns {String} A region descriptor (e.g., Ohio)
+ */
+export default function regions(region) {
+  switch (region) {
+    case "us-east-1":
+      return "N. Virginia";
+    case "us-east-2":
+      return "Ohio";
+    case "us-west-1":
+      return "N. California";
+    case "us-west-2":
+      return "Oregon";
+    case "af-south-1":
+      return "Cape Town";
+    case "ap-east-1":
+      return "Hong Kong";
+    case "ap-south-1":
+      return "Mumbai";
+    case "ap-northeast-3":
+      return "Osaka";
+    case "ap-northeast-2":
+      return "Seoul";
+    case "ap-southeast-1":
+      return "Singapore";
+    case "ap-southeast-2":
+      return "Sydney";
+    case "ap-northeast-1":
+      return "Tokyo";
+    case "ca-central-1":
+      return "C. Canada";
+    case "eu-central-1":
+      return "Frankfurt";
+    case "eu-west-1":
+      return "Ireland";
+    case "eu-west-2":
+      return "London";
+    case "eu-south-1":
+      return "Milan";
+    case "eu-west-3":
+      return "Paris";
+    case "eu-north-1":
+      return "Stockholm";
+    case "me-south-1":
+      return "Bahrain";
+    case "sa-east-1":
+      return "São Paulo";
+    default:
+      return region;
+  }
+}

src/alarm-slack-report/urls.mjs

@@ -0,0 +1,25 @@
+/**
+ * Creates a deep link to an AWS Console URL in a specific account, using an
+ * IAM Identity Center access role
+ * @param {String} accountId
+ * @param {String} url
+ * @returns
+ */
+export function ssoDeepLink(accountId, url) {
+  const deepLinkRoleName = "AdministratorAccess";
+  const urlEncodedUrl = encodeURIComponent(url);
+  return `https://d-906713e952.awsapps.com/start/#/console?account_id=${accountId}&role_name=${deepLinkRoleName}&destination=${urlEncodedUrl}`;
+}
+
+/**
+ * Returns a URL to CloudWatch Alarms console for the alarm that triggered
+ * the event.
+ * @param {*} alarmDetail
+ * @returns {String}
+ */
+export function alarmConsole(alarmDetail) {
+  const name = alarmDetail.AlarmName;
+  const region = alarmDetail.AlarmArn.split(":")[3];
+  const encoded = encodeURI(name.replace(/ /g, "+")).replace(/%/g, "$");
+  return `https://console.aws.amazon.com/cloudwatch/home?region=${region}#alarmsV2:alarm/${encoded}`;
+}

template.yml

@@ -301,3 +301,85 @@ Resources:
         - { Key: prx:dev:application, Value: CloudWatch Toolkit }
       Threshold: 0
       TreatMissingData: notBreaching
+
+  # Scans certain accounts and regions for all alarms, and sends a report
+  # message to Slack with information about any alarms that were active recently
+  AlarmSlackReportFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: src/alarm-slack-report
+      Description: >-
+        Scans a set of accounts and regions for active alarms,
+        and sends a summary to Slack
+      Environment:
+        Variables:
+          AWS_NODEJS_CONNECTION_REUSE_ENABLED: "1"
+          CLOUDWATCH_CROSS_ACCOUNT_SHARING_ROLE_NAME: !Ref CloudWatchCrossAccountSharingRoleName
+          SEARCH_REGIONS: !Join [",", !Ref AlarmReminderSearchRegions]
+          SEARCH_ACCOUNTS: !Join [",", !Ref AlarmReminderSearchAccountIds]
+      Events:
+        WeekdayPoller:
+          Properties:
+            Description: >-
+              Invokes the CloudWatch Alarm report function
+            Enabled: true
+            Schedule: cron(0 16 ? * * *)
+          Type: Schedule
+      Handler: index.handler
+      MemorySize: 192
+      Policies:
+        - Statement:
+            - Action: events:PutEvents
+              Effect: Allow
+              Resource: !Sub arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:event-bus/default
+          Version: "2012-10-17"
+        - Statement:
+            - Action: sts:AssumeRole
+              Effect: Allow
+              Resource: !Sub arn:aws:iam::*:role/${CloudWatchCrossAccountSharingRoleName}
+          Version: "2012-10-17"
+      Runtime: nodejs20.x
+      Tags:
+        prx:meta:tagging-version: "2021-04-07"
+        prx:cloudformation:stack-name: !Ref AWS::StackName
+        prx:cloudformation:stack-id: !Ref AWS::StackId
+        prx:ops:environment: Production
+        prx:dev:application: CloudWatch Toolkit
+      Timeout: 60
+  AlarmSlackReportLogGroup:
+    Type: AWS::Logs::LogGroup
+    DeletionPolicy: Delete
+    UpdateReplacePolicy: Delete
+    Properties:
+      LogGroupName: !Sub /aws/lambda/${AlarmSlackReportFunction}
+      RetentionInDays: 14
+      Tags:
+        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
+        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
+        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
+        - { Key: prx:ops:environment, Value: Production }
+        - { Key: prx:dev:application, Value: CloudWatch Toolkit }
+  AlarmSlackReportErrorAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmName: "MINOR [CloudWatch] Alarm Slack Report <prod> EXPERIENCING ERRORS"
+      AlarmDescription: >-
+        Errors are occurring on the CloudWatch Alarm report Lambda function,
+        so those reports may not be reaching Slack.
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions:
+        - Name: FunctionName
+          Value: !Ref AlarmSlackReportFunction
+      EvaluationPeriods: 1
+      MetricName: Errors
+      Namespace: AWS/Lambda
+      Period: 60
+      Statistic: Sum
+      Tags:
+        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
+        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
+        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
+        - { Key: prx:ops:environment, Value: Production }
+        - { Key: prx:dev:application, Value: CloudWatch Toolkit }
+      Threshold: 0
+      TreatMissingData: notBreaching