diff --git a/package.json b/package.json index e8ec02e..8d5ba1b 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "@aws-sdk/client-cloudwatch": "*", "@aws-sdk/client-eventbridge": "*", "@aws-sdk/client-sns": "*", - "@aws-sdk/client-sts": "*" + "@aws-sdk/client-sts": "*", + "@aws-sdk/util-retry": "*" }, "devDependencies": { "@types/aws-lambda": "*", diff --git a/src/alarm-slack-report/index.js b/src/alarm-slack-report/index.mjs similarity index 61% rename from src/alarm-slack-report/index.js rename to src/alarm-slack-report/index.mjs index bb737a6..6918950 100644 --- a/src/alarm-slack-report/index.js +++ b/src/alarm-slack-report/index.mjs @@ -5,12 +5,14 @@ import { STSClient, AssumeRoleCommand } from "@aws-sdk/client-sts"; import { CloudWatchClient, DescribeAlarmsCommand, + paginateDescribeAlarmHistory, } from "@aws-sdk/client-cloudwatch"; import { EventBridgeClient, PutEventsCommand, } from "@aws-sdk/client-eventbridge"; -// import regions from "./regions.mjs"; +import { ConfiguredRetryStrategy } from "@aws-sdk/util-retry"; +import regions from "./regions.mjs"; // import { alarmConsole, ssoDeepLink } from "./urls.mjs"; const sts = new STSClient({ apiVersion: "2011-06-15" }); @@ -27,6 +29,10 @@ async function cloudWatchClient(accountId, region) { ); return new CloudWatchClient({ + retryStrategy: new ConfiguredRetryStrategy( + 10, + (attempt) => 100 + attempt * 1000, + ), apiVersion: "2010-08-01", region, credentials: { @@ -47,11 +53,7 @@ async function describeAllAlarms(cwClient, nextToken) { /** @type {AlarmType[]} */ const alarmTypes = ["CompositeAlarm", "MetricAlarm"]; - /** @type {StateValue} */ - const stateValue = "ALARM"; - const params = { - StateValue: stateValue, AlarmTypes: alarmTypes, ...(nextToken && { NextToken: nextToken }), }; @@ -83,20 +85,20 @@ async function describeAllAlarms(cwClient, nextToken) { return results; } -// function cleanName(alarmName) { -// return alarmName -// .replace(/>/g, ">") -// .replace(//g, ">") + .replace(/ { console.log(JSON.stringify(event)); - const alarms = { - CompositeAlarms: [], - MetricAlarms: [], - }; + const reports = []; + + const hoursAgo24 = new Date(); + hoursAgo24.setUTCHours(-24); // eslint-disable-next-line no-restricted-syntax for (const accountId of process.env.SEARCH_ACCOUNTS.split(",")) { @@ -127,18 +129,52 @@ export const handler = async (event) => { // eslint-disable-next-line no-await-in-loop const data = await describeAllAlarms(cloudwatch, undefined); - alarms.CompositeAlarms.push(...data.CompositeAlarms); - alarms.MetricAlarms.push(...data.MetricAlarms.filter(filterByName)); + // TODO Handle composite alarms + // eslint-disable-next-line no-restricted-syntax + for (const alarm of data.MetricAlarms.filter(filterByName)) { + const ts = Date.parse(alarm.StateTransitionedTimestamp); + + if (ts >= +hoursAgo24) { + const paginator = paginateDescribeAlarmHistory( + { + client: cloudwatch, + }, + { + AlarmName: alarm.alarmName, + HistoryItemType: "StateUpdate", + StartDate: hoursAgo24, + EndDate: new Date(), + }, + ); + + const alarmHistoryItems = []; + // eslint-disable-next-line no-restricted-syntax, no-await-in-loop + for await (const page of paginator) { + alarmHistoryItems.push(...page.AlarmHistoryItems); + } + + // const history = await cloudwatch.send( + // new DescribeAlarmHistoryCommand(), + // ); + + const toAlarmCount = alarmHistoryItems.filter((i) => + i.HistorySummary.includes("to ALARM"), + ).length; + + reports.push({ + Alarm: alarm, + Count: toAlarmCount, + }); + } + } } } - console.log(JSON.stringify(alarms)); - - const count = alarms.CompositeAlarms.length + alarms.MetricAlarms.length; + console.log(reports); const blocks = []; - if (count === 0) { + if (reports.length === 0) { return; } @@ -151,28 +187,22 @@ export const handler = async (event) => { }, }); - // blocks.push( - // ...alarms.MetricAlarms.map((a) => { - // const accountId = a.AlarmArn.split(":")[4]; - // const url = alarmConsole(a); - // const ssoUrl = ssoDeepLink(accountId, url); - - // const lines = [`*<${ssoUrl}|${title(a)}>*`]; - - // if (a.StateReasonData) { - // const reasonData = JSON.parse(a.StateReasonData); - // lines.push(started(reasonData)); - // } - - // return { - // type: "section", - // text: { - // type: "mrkdwn", - // text: lines.join("\n"), - // }, - // }; - // }), - // ); + const lines = reports.map((r) => { + // const accountId = r.Alarm.AlarmArn.split(":")[4]; + // const url = alarmConsole(r.Alarm); + // const ssoUrl = ssoDeepLink(accountId, url); + + return `*${title(r.Alarm)}*: \`${r.Count}\``; + // return `*<${ssoUrl}|${title(r.Alarm)}>*`; + }); + + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: lines.join("\n"), + }, + }); await eventbridge.send( new PutEventsCommand({ @@ -183,7 +213,7 @@ export const handler = async (event) => { Detail: JSON.stringify({ username: "Amazon CloudWatch Alarms", icon_emoji: ":ops-cloudwatch-alarm:", - // channel: "G2QH6NMEH", // #ops-error + // channel: "G2QHBL6UX", // #ops-info channel: "CHZTAGBM2", // #sandbox2 attachments: [ { diff --git a/template.yml b/template.yml index 6a3536b..7bc1500 100644 --- a/template.yml +++ b/template.yml @@ -345,7 +345,7 @@ Resources: prx:cloudformation:stack-id: !Ref AWS::StackId prx:ops:environment: Production prx:dev:application: CloudWatch Toolkit - Timeout: 60 + Timeout: 120 AlarmSlackReportLogGroup: Type: AWS::Logs::LogGroup DeletionPolicy: Delete diff --git a/yarn.lock b/yarn.lock index 37c31dd..952c51c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -802,6 +802,14 @@ dependencies: tslib "^2.6.2" +"@aws-sdk/util-retry@*": + version "3.374.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-retry/-/util-retry-3.374.0.tgz#7fd819d5857609b65a1bf06c39701fe5de5ad6cd" + integrity sha512-0p/trhYU+Ys8j3vMnWCvAkSOL6JRMooV9dVlQ+o7EHbQs9kDtnyucMUHU09ahHSIPTA/n/013hv7bzIt3MyKQg== + dependencies: + "@smithy/util-retry" "^1.0.3" + tslib "^2.5.0" + "@aws-sdk/util-user-agent-browser@3.535.0": version "3.535.0" resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.535.0.tgz#d67d72e8b933051620f18ddb1c2be225f79f588f" @@ -1120,6 +1128,11 @@ "@smithy/types" "^2.12.0" tslib "^2.6.2" +"@smithy/service-error-classification@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@smithy/service-error-classification/-/service-error-classification-1.1.0.tgz#264dd432ae513b3f2ad9fc6f461deda8c516173c" + integrity sha512-OCTEeJ1igatd5kFrS2VDlYbainNNpf7Lj1siFOxnRWqYOP9oNvC5HOJBd3t+Z8MbrmehBtuDJ2QqeBsfeiNkww== + "@smithy/service-error-classification@^2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@smithy/service-error-classification/-/service-error-classification-2.1.5.tgz#0568a977cc0db36299d8703a5d8609c1f600c005" @@ -1275,6 +1288,14 @@ "@smithy/types" "^2.12.0" tslib "^2.6.2" +"@smithy/util-retry@^1.0.3": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-1.1.0.tgz#f6e62ec7d7d30f1dd9608991730ba7a86e445047" + integrity sha512-ygQW5HBqYXpR3ua09UciS0sL7UGJzGiktrKkOuEJwARoUuzz40yaEGU6xd9Gs7KBmAaFC8gMfnghHtwZ2nyBCQ== + dependencies: + "@smithy/service-error-classification" "^1.1.0" + tslib "^2.5.0" + "@smithy/util-retry@^2.2.0": version "2.2.0" resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-2.2.0.tgz#e8e019537ab47ba6b2e87e723ec51ee223422d85" @@ -2991,6 +3012,11 @@ tslib@^2.3.1, tslib@^2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tslib@^2.5.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.7.0.tgz#d9b40c5c40ab59e8738f297df3087bf1a2690c01" + integrity sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"