From a5a0be75bd4e6934e1001d71b580285ff7a74a59 Mon Sep 17 00:00:00 2001 From: Alexander Lukin Date: Tue, 19 Mar 2024 10:19:45 +0400 Subject: [PATCH 1/2] feat: add minimum bound to alerts threshold Previously all validator alerts had a relative threshold of 1/3. Now alerts will be triggered if the number of affected validators is more than the specified threshold or more than 1000. --- .../alertmanager/alerts/CriticalMissedAttestations.ts | 6 ++++-- src/common/alertmanager/alerts/CriticalMissedProposes.ts | 6 ++++-- src/common/alertmanager/alerts/CriticalNegativeDelta.ts | 6 ++++-- src/common/consensus-provider/consensus-provider.service.ts | 4 +++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/common/alertmanager/alerts/CriticalMissedAttestations.ts b/src/common/alertmanager/alerts/CriticalMissedAttestations.ts index 5d061271..0700e378 100644 --- a/src/common/alertmanager/alerts/CriticalMissedAttestations.ts +++ b/src/common/alertmanager/alerts/CriticalMissedAttestations.ts @@ -8,7 +8,9 @@ import { RegistrySourceOperator } from 'validators-registry'; import { Alert, AlertRequestBody, AlertRuleResult } from './BasicAlert'; -const VALIDATORS_WITH_MISSED_ATTESTATION_COUNT_THRESHOLD = 1 / 3; +const validatorsWithMissedAttestationCountThreshold = (quantity: number) => { + return Math.min(quantity / 3, 1000); +}; export class CriticalMissedAttestations extends Alert { constructor(config: ConfigService, storage: ClickhouseService, operators: RegistrySourceOperator[]) { @@ -25,7 +27,7 @@ export class CriticalMissedAttestations extends Alert { (a) => a.val_nos_id != null && +a.val_nos_module_id == operator.module && +a.val_nos_id == operator.index, ); if (!missedAtt) continue; - if (missedAtt.amount > noStats.active_ongoing * VALIDATORS_WITH_MISSED_ATTESTATION_COUNT_THRESHOLD) { + if (missedAtt.amount > validatorsWithMissedAttestationCountThreshold(noStats.active_ongoing)) { result[operator.name] = { ongoing: noStats.active_ongoing, missedAtt: missedAtt.amount }; } } diff --git a/src/common/alertmanager/alerts/CriticalMissedProposes.ts b/src/common/alertmanager/alerts/CriticalMissedProposes.ts index e041bcef..4f8f6eee 100644 --- a/src/common/alertmanager/alerts/CriticalMissedProposes.ts +++ b/src/common/alertmanager/alerts/CriticalMissedProposes.ts @@ -8,7 +8,9 @@ import { RegistrySourceOperator } from 'validators-registry'; import { Alert, AlertRequestBody, AlertRuleResult } from './BasicAlert'; -const VALIDATORS_WITH_MISSED_PROPOSALS_COUNT_THRESHOLD = 1 / 3; +const validatorsWithMissedProposalsCountThreshold = (quantity: number) => { + return Math.min(quantity / 3, 1000); +}; export class CriticalMissedProposes extends Alert { constructor(config: ConfigService, storage: ClickhouseService, operators: RegistrySourceOperator[]) { @@ -25,7 +27,7 @@ export class CriticalMissedProposes extends Alert { (a) => a.val_nos_id != null && +a.val_nos_module_id == operator.module && +a.val_nos_id == operator.index, ); if (!proposeStats) continue; - if (proposeStats.missed > proposeStats.all * VALIDATORS_WITH_MISSED_PROPOSALS_COUNT_THRESHOLD) { + if (proposeStats.missed > validatorsWithMissedProposalsCountThreshold(proposeStats.all)) { result[operator.name] = { all: proposeStats.all, missed: proposeStats.missed }; } } diff --git a/src/common/alertmanager/alerts/CriticalNegativeDelta.ts b/src/common/alertmanager/alerts/CriticalNegativeDelta.ts index ad8647b4..e376d4b8 100644 --- a/src/common/alertmanager/alerts/CriticalNegativeDelta.ts +++ b/src/common/alertmanager/alerts/CriticalNegativeDelta.ts @@ -8,7 +8,9 @@ import { RegistrySourceOperator } from 'validators-registry'; import { Alert, AlertRequestBody, AlertRuleResult } from './BasicAlert'; -const VALIDATORS_WITH_NEGATIVE_DELTA_COUNT_THRESHOLD = 1 / 3; +const validatorsWithNegativeDeltaCountThreshold = (quantity: number) => { + return Math.min(quantity / 3, 1000); +}; export class CriticalNegativeDelta extends Alert { constructor(config: ConfigService, storage: ClickhouseService, operators: RegistrySourceOperator[]) { @@ -23,7 +25,7 @@ export class CriticalNegativeDelta extends Alert { const operator = this.operators.find((o) => +noStats.val_nos_module_id == o.module && +noStats.val_nos_id == o.index); const negDelta = negativeValidatorsCount.find((a) => +a.val_nos_module_id == operator.module && +a.val_nos_id == operator.index); if (!negDelta) continue; - if (negDelta.amount > noStats.active_ongoing * VALIDATORS_WITH_NEGATIVE_DELTA_COUNT_THRESHOLD) { + if (negDelta.amount > validatorsWithNegativeDeltaCountThreshold(noStats.active_ongoing)) { result[operator.name] = { ongoing: noStats.active_ongoing, negDelta: negDelta.amount }; } } diff --git a/src/common/consensus-provider/consensus-provider.service.ts b/src/common/consensus-provider/consensus-provider.service.ts index cc628064..ff095fb8 100644 --- a/src/common/consensus-provider/consensus-provider.service.ts +++ b/src/common/consensus-provider/consensus-provider.service.ts @@ -127,7 +127,9 @@ export class ConsensusProviderService { if (nodeLatestSlot < this.latestSlot.slot) { // we assume that the node must never return a slot less than the last saved slot - this.logger.error(`Received ${latestFrom} slot [${nodeLatestSlot}] is less than last [${this.latestSlot.slot}] slot received before, but shouldn't`); + this.logger.error( + `Received ${latestFrom} slot [${nodeLatestSlot}] is less than last [${this.latestSlot.slot}] slot received before, but shouldn't`, + ); return true; } if (nodeLatestSlot > this.latestSlot.slot) { From 90805015fd0aef7f9ee26d73450d4d9198f555eb Mon Sep 17 00:00:00 2001 From: Alexander Lukin Date: Mon, 25 Mar 2024 12:54:45 +0400 Subject: [PATCH 2/2] fix: return previous rule of missed proposes calculation As the number of missed proposes must never be greater than 1000, it doesn't make sense to have a minimal absolute threshold for this rule. --- src/common/alertmanager/alerts/CriticalMissedProposes.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/common/alertmanager/alerts/CriticalMissedProposes.ts b/src/common/alertmanager/alerts/CriticalMissedProposes.ts index 4f8f6eee..e041bcef 100644 --- a/src/common/alertmanager/alerts/CriticalMissedProposes.ts +++ b/src/common/alertmanager/alerts/CriticalMissedProposes.ts @@ -8,9 +8,7 @@ import { RegistrySourceOperator } from 'validators-registry'; import { Alert, AlertRequestBody, AlertRuleResult } from './BasicAlert'; -const validatorsWithMissedProposalsCountThreshold = (quantity: number) => { - return Math.min(quantity / 3, 1000); -}; +const VALIDATORS_WITH_MISSED_PROPOSALS_COUNT_THRESHOLD = 1 / 3; export class CriticalMissedProposes extends Alert { constructor(config: ConfigService, storage: ClickhouseService, operators: RegistrySourceOperator[]) { @@ -27,7 +25,7 @@ export class CriticalMissedProposes extends Alert { (a) => a.val_nos_id != null && +a.val_nos_module_id == operator.module && +a.val_nos_id == operator.index, ); if (!proposeStats) continue; - if (proposeStats.missed > validatorsWithMissedProposalsCountThreshold(proposeStats.all)) { + if (proposeStats.missed > proposeStats.all * VALIDATORS_WITH_MISSED_PROPOSALS_COUNT_THRESHOLD) { result[operator.name] = { all: proposeStats.all, missed: proposeStats.missed }; } }