[pipelines] for contract v2 skipping old merkle trees

.buildkite/claim-settlements.yml

@@ -27,30 +27,38 @@ steps:
     - '. "$HOME/.cargo/env"'
     - 'buildkite-agent artifact download --include-retried-jobs target/release/list-claimable-epoch .'
     - 'chmod +x target/release/list-claimable-epoch'
-    - 'claimable_epochs_json=$(./target/release/list-claimable-epoch --rpc-url $$RPC_URL)'
-    - 'buildkite-agent meta-data set claimable_epochs_json "$$claimable_epochs_json"'
+    - 'possible_epochs_json=$(./target/release/list-claimable-epoch --rpc-url $$RPC_URL)'
+    - 'buildkite-agent meta-data set possible_epochs_json "$$possible_epochs_json"'
 
   - wait: ~
 
   - label: ":floppy_disk: :arrow_left: :cloud: Downloading all epochs merkle trees"
     env:
       gs_bucket: gs://marinade-validator-bonds-mainnet
+      # epoch when the contract v2 was deployed, using different structure of merkle tree than v1
+      starting_epoch_contract_v2: 640
     commands:
     - 'mkdir ./merkle-trees/'
-    - 'claimable_epochs_json=$(buildkite-agent meta-data get claimable_epochs_json)'
-    - 'claimable_epochs_num=$(echo "$$claimable_epochs_json" | jq "length")'
-    - 'echo "Claimable epochs [$$claimable_epochs_num]: $$claimable_epochs_json"'
-    - 'buildkite-agent meta-data set claimable_epochs_num "$$claimable_epochs_num"'
-    - |
-      if [ "$$claimable_epochs_num" -eq 0 ]; then
-        echo "No claimable epochs found"
-        exit 0
-      fi
+    - 'possible_epochs_json=$(buildkite-agent meta-data get possible_epochs_json)'
+    - 'possible_epochs_num=$(echo "$$possible_epochs_json" | jq "length")'
+    - 'echo "Possible claimable epochs [$$possible_epochs_num]: $$possible_epochs_json"'
+    - 'claimable_epochs_num=0'
+    - 'claimable_epochs_json=()'
     - |
-      for epoch in $(echo "$$claimable_epochs_json" | jq ".[]"); do
+      for epoch in $(echo "$$possible_epochs_json" | jq ".[]"); do
+        if [[ $$epoch -lt $$starting_epoch_contract_v2 ]]; then
+          echo "Skipping epoch $$epoch, because it's before the contract v2 deployment epoch $$starting_epoch_contract_v2"
+          continue
+        fi
+        claimable_epochs_num=$(($$claimable_epochs_num + 1))
+        claimable_epochs_json+=($$epoch)
         gcloud storage cp "$$gs_bucket/$$epoch/settlement-merkle-trees.json" "./merkle-trees/$${epoch}_settlement-merkle-trees.json"
         gcloud storage cp "$$gs_bucket/$$epoch/settlements.json" "./merkle-trees/$${epoch}_settlements.json"
       done
+      claimable_epochs_json=$(jq --compact-output --null-input '$ARGS.positional' --args -- "$${claimable_epochs_json[@]}")
+    - 'buildkite-agent meta-data set claimable_epochs_json "$$claimable_epochs_json"'
+    - 'buildkite-agent meta-data set claimable_epochs_num "$$claimable_epochs_num"'
+    - 'echo "Claimable epochs [$$claimable_epochs_num]: $$claimable_epochs_json"'
     artifact_paths:
       - "./merkle-trees/*"
 
@@ -71,9 +79,9 @@ steps:
     commands:
     - '. "$HOME/.cargo/env"'
     - |
-      claimable_epochs_num=$(buildkite-agent meta-data get claimable_epochs_num)
+      claimable_epochs_num=$(buildkite-agent meta-data get claimable_epochs_num || echo "0")
       if [[ $$claimable_epochs_num -eq 0 ]]; then
-        echo 'No settlement to claim from' | tee ./claiming-report.txt
+        echo 'No claimable settlement to claim from' | tee ./claiming-report.txt
         exit 0
       fi
     - 'prior_build_number=$(($$BUILDKITE_RETRY_COUNT - 1))'

.buildkite/close-settlements.yml

@@ -23,6 +23,8 @@ steps:
       gs_bucket: gs://marinade-validator-bonds-mainnet
       config_epochs_non_closable: 3 # configured onchain in config
       past_epochs_to_load: 5
+      # epoch when the contract v2 was deployed, using different structure of merkle tree than v1
+      starting_epoch_contract_v2: 640
     commands:
     - |
       set -x
@@ -38,6 +40,10 @@ steps:
     - |
       set -x
       for epoch in $(seq $$epochs_start_index $$epochs_end_index); do
+        if [[ $$epoch -lt $$starting_epoch_contract_v2 ]]; then
+          echo "Skipping epoch $$epoch, because it's before the contract v2 deployment epoch $$starting_epoch_contract_v2"
+          continue
+        fi
         gcloud storage cp "$$gs_bucket/$$epoch/settlement-merkle-trees.json" "./merkle-trees/$${epoch}_settlement-merkle-trees.json"
       done
     artifact_paths:

.buildkite/fund-settlements.yml

@@ -20,6 +20,8 @@ steps:
     env:
       gs_bucket: gs://marinade-validator-bonds-mainnet
       past_epochs_to_load: 4
+      # epoch when the contract v2 was deployed, using different structure of merkle tree than v1
+      starting_epoch_contract_v2: 640
     commands:
     - |
       set -x
@@ -36,6 +38,10 @@ steps:
     - |
       set -x
       for epoch in $(seq $$epochs_start_index $$current_epoch); do
+        if [[ $$epoch -lt $$starting_epoch_contract_v2 ]]; then
+          echo "Skipping epoch $$epoch, because it's before the contract v2 deployment epoch $$starting_epoch_contract_v2"
+          continue
+        fi
         gcloud storage cp "$$gs_bucket/$$epoch/settlement-merkle-trees.json" "./merkle-trees/$${epoch}_settlement-merkle-trees.json" || true
         gcloud storage cp "$$gs_bucket/$$epoch/settlements.json" "./merkle-trees/$${epoch}_settlements.json" || true
       done

settlement-pipelines/src/lib.rs

@@ -9,3 +9,5 @@ pub mod settlement_data;
 pub mod settlements;
 pub mod stake_accounts;
 pub mod stake_accounts_cache;
+
+pub const CONTRACT_V2_DEPLOYMENT_EPOCH: u64 = 640_u64;

settlement-pipelines/src/settlements.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use validator_bonds::state::config::{find_bonds_withdrawer_authority, Config};
 use validator_bonds::state::settlement::{find_settlement_staker_authority, Settlement};
 
+use crate::CONTRACT_V2_DEPLOYMENT_EPOCH;
 use validator_bonds_common::settlement_claims::SettlementClaimsBitmap;
 use validator_bonds_common::settlements::{
     get_bonds_for_settlements, get_settlement_claims_for_settlement_pubkeys, get_settlements,
@@ -84,15 +85,21 @@ pub async fn list_claimable_settlements(
     let claimable_settlement_claims: Vec<(Pubkey, Pubkey, SettlementClaimsBitmap)> =
         claimable_settlement_claims
             .into_iter()
-            .map(|(settlement_pubkey, claims_pubkey, claims)| {
+            .zip(claimable_settlements.iter())
+            .filter_map(|((settlement_pubkey, claims_pubkey, claims), (_, settlement))| {
                 if let Some(claims) = claims {
-                    Ok((settlement_pubkey, claims_pubkey, claims))
+                    Some(Ok((settlement_pubkey, claims_pubkey, claims)))
                 } else {
-                    Err(CliError::Processing(anyhow!(
-                        "CRITICAL [list_claimable]: No SettlementClaims account {} for an existing Settlement {}",
-                        claims_pubkey,
-                        settlement_pubkey
-                    )))
+                    let error_msg = format!("[list_claimable]: No SettlementClaims account {} for an existing Settlement {}",
+                    claims_pubkey,
+                    settlement_pubkey
+                    );
+                    if settlement.epoch_created_for < CONTRACT_V2_DEPLOYMENT_EPOCH {
+                        info!("{}", error_msg);
+                        None
+                    } else {
+                        Some(Err(CliError::Processing(anyhow!("CRITICAL {}", error_msg))))
+                    }
                 }
             })
             .collect::<Result<Vec<(Pubkey, Pubkey, SettlementClaimsBitmap)>, CliError>>()?;