Merge pull request lidofinance#251 from lidofinance/feature/absolute-…

…min-threshold-for-alerts feat: add minimum bound to alerts threshold
dsrvlabs · Mar 25, 2024 · 1be5b45 · 1be5b45
2 parents 162a673 + 9080501
commit 1be5b45
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 5 deletions.
diff --git a/src/common/alertmanager/alerts/CriticalMissedAttestations.ts b/src/common/alertmanager/alerts/CriticalMissedAttestations.ts
@@ -8,7 +8,9 @@ import { RegistrySourceOperator } from 'validators-registry';
 
 import { Alert, AlertRequestBody, AlertRuleResult } from './BasicAlert';
 
-const VALIDATORS_WITH_MISSED_ATTESTATION_COUNT_THRESHOLD = 1 / 3;
+const validatorsWithMissedAttestationCountThreshold = (quantity: number) => {
+  return Math.min(quantity / 3, 1000);
+};
 
 export class CriticalMissedAttestations extends Alert {
   constructor(config: ConfigService, storage: ClickhouseService, operators: RegistrySourceOperator[]) {
@@ -25,7 +27,7 @@ export class CriticalMissedAttestations extends Alert {
         (a) => a.val_nos_id != null && +a.val_nos_module_id == operator.module && +a.val_nos_id == operator.index,
       );
       if (!missedAtt) continue;
-      if (missedAtt.amount > noStats.active_ongoing * VALIDATORS_WITH_MISSED_ATTESTATION_COUNT_THRESHOLD) {
+      if (missedAtt.amount > validatorsWithMissedAttestationCountThreshold(noStats.active_ongoing)) {
         result[operator.name] = { ongoing: noStats.active_ongoing, missedAtt: missedAtt.amount };
       }
     }

diff --git a/src/common/alertmanager/alerts/CriticalNegativeDelta.ts b/src/common/alertmanager/alerts/CriticalNegativeDelta.ts
@@ -8,7 +8,9 @@ import { RegistrySourceOperator } from 'validators-registry';
 
 import { Alert, AlertRequestBody, AlertRuleResult } from './BasicAlert';
 
-const VALIDATORS_WITH_NEGATIVE_DELTA_COUNT_THRESHOLD = 1 / 3;
+const validatorsWithNegativeDeltaCountThreshold = (quantity: number) => {
+  return Math.min(quantity / 3, 1000);
+};
 
 export class CriticalNegativeDelta extends Alert {
   constructor(config: ConfigService, storage: ClickhouseService, operators: RegistrySourceOperator[]) {
@@ -23,7 +25,7 @@ export class CriticalNegativeDelta extends Alert {
       const operator = this.operators.find((o) => +noStats.val_nos_module_id == o.module && +noStats.val_nos_id == o.index);
       const negDelta = negativeValidatorsCount.find((a) => +a.val_nos_module_id == operator.module && +a.val_nos_id == operator.index);
       if (!negDelta) continue;
-      if (negDelta.amount > noStats.active_ongoing * VALIDATORS_WITH_NEGATIVE_DELTA_COUNT_THRESHOLD) {
+      if (negDelta.amount > validatorsWithNegativeDeltaCountThreshold(noStats.active_ongoing)) {
         result[operator.name] = { ongoing: noStats.active_ongoing, negDelta: negDelta.amount };
       }
     }

diff --git a/src/common/consensus-provider/consensus-provider.service.ts b/src/common/consensus-provider/consensus-provider.service.ts
@@ -127,7 +127,9 @@ export class ConsensusProviderService {
 
           if (nodeLatestSlot < this.latestSlot.slot) {
             // we assume that the node must never return a slot less than the last saved slot
-            this.logger.error(`Received ${latestFrom} slot [${nodeLatestSlot}] is less than last [${this.latestSlot.slot}] slot received before, but shouldn't`);
+            this.logger.error(
+              `Received ${latestFrom} slot [${nodeLatestSlot}] is less than last [${this.latestSlot.slot}] slot received before, but shouldn't`,
+            );
             return true;
           }
           if (nodeLatestSlot > this.latestSlot.slot) {