-
Notifications
You must be signed in to change notification settings - Fork 3
/
init-settlements.yml
234 lines (207 loc) · 9.66 KB
/
init-settlements.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
agents:
queue: "snapshots"
env:
gs_bucket: gs://marinade-validator-bonds-mainnet
steps:
# epoch number provided in ENV and then waiting for confirmation to proceed
# or waiting to be prompted by the user after notification is sent
- input: "What is the epoch to init settlements for?"
fields:
- text: "Epoch"
key: "epoch"
format: "[0-9]+"
if: "build.env('EPOCH') == null"
- command: "echo 'Epoch is: $EPOCH'"
if: "build.env('EPOCH') != null"
- input: "What is the claim type to init?"
fields:
- select: "Claim type"
key: "claim_type"
options:
- label: "Protected Events"
value: "protected-events"
- label: "Bidding"
value: "bidding"
if: "build.env('CLAIM_TYPE') == null"
- wait: ~
- label: ":black_nib: Env variables setup: Init Settlements"
commands:
- |
discord_webhook="${DISCORD_WEBHOOK:-$$DISCORD_WEBHOOK_VALIDATOR_BONDS}"
buildkite-agent meta-data set discord_webhook "$$discord_webhook"
- |
epoch=${EPOCH:-$(buildkite-agent meta-data get epoch)}
buildkite-agent meta-data set epoch "$$epoch"
- |
claim_type=${CLAIM_TYPE:-$(buildkite-agent meta-data get claim_type)}
buildkite-agent meta-data set claim_type "$$claim_type"
- 'echo "Epoch: $$epoch", Claim Type: $$claim_type'
- wait: ~
- label: ":mega: Notification initializing settlements"
commands:
- epoch=$(buildkite-agent meta-data get epoch)
- claim_type=$(buildkite-agent meta-data get claim_type)
- |
curl "$(buildkite-agent meta-data get discord_webhook)" \
-H "Content-Type: application/json" -d '{
"embeds": [
{
"title": "Confirm Settlements Initialization for Validator Bonds ('"$$epoch"')/('"$$claim_type"').",
"description": "Please review the 'Unblock' if ready to proceed.",
"url": "'"$$BUILDKITE_BUILD_URL"'",
"color": "8388863"
}
]
}'
- wait: ~
- label: ":hammer_and_wrench: :rust: Build init-settlement"
commands:
- '. "$HOME/.cargo/env"'
- 'cargo build --release --bin init-settlement'
artifact_paths:
- target/release/init-settlement
- label: ":floppy_disk: :arrow_left: :cloud: Downloading merkle trees"
commands:
- epoch=$(buildkite-agent meta-data get epoch)
- claim_type=$(buildkite-agent meta-data get claim_type)
- |
gcloud storage cp "$gs_bucket/$$epoch/$${claim_type}-settlement-merkle-trees.json" "./settlement-merkle-trees.json"
gcloud storage cp "$gs_bucket/$$epoch/$${claim_type}-settlements.json" "./settlements.json"
key: 'download-json'
artifact_paths:
- "./settlement-merkle-trees.json"
- "./settlements.json"
- label: ":black_nib: Settlement JSON listing"
commands:
- buildkite-agent artifact download --include-retried-jobs settlement-merkle-trees.json .
- buildkite-agent artifact download --include-retried-jobs settlements.json .
- bash ./scripts/settlement-json-listing.sh --settlements settlements.json --merkle-trees settlement-merkle-trees.json
depends_on: "download-json"
- wait: ~
- block: "Confirm to get processed init settlements"
prompt: "Please review the 'Unblock' if ready to proceed."
blocked_state: running
- wait: ~
- label: ":closed_lock_with_key: Concurrency gate lock"
command: echo "--> Start of concurrency gate"
concurrency_group: 'validator-bonds/init-settlements${CLAIM_TYPE}'
concurrency: 1
- wait: ~
- label: ":campfire::arrow_right: Create settlements"
env:
RUST_LOG: info,solana_transaction_builder_executor=debug,solana_transaction_builder=debug,builder_executor=debug,solana_transaction_executor=debug,settlement_pipelines=debug,init_settlement=debug
# RUST_BACKTRACE: full
commands:
- . "$HOME/.cargo/env"
- epoch=$(buildkite-agent meta-data get epoch)
- prior_build_number=$(($$BUILDKITE_RETRY_COUNT - 1))
- buildkite-agent artifact download --include-retried-jobs init-execution-report.$$prior_build_number . || true
- |
cp "init-execution-report.$$prior_build_number" "init-execution-report.$$BUILDKITE_RETRY_COUNT" || true
rm -f "init-execution-report.$$prior_build_number"
echo "#ATTEMPT INIT SETTLEMENTS $$BUILDKITE_RETRY_COUNT" | tee -a "./init-execution-report.$$BUILDKITE_RETRY_COUNT"
- buildkite-agent artifact download --include-retried-jobs target/release/init-settlement .
- buildkite-agent artifact download --include-retried-jobs settlement-merkle-trees.json .
- buildkite-agent artifact download --include-retried-jobs settlements.json .
- chmod +x target/release/init-settlement
- |
set -o pipefail
./target/release/init-settlement \
--rpc-url $$RPC_URL \
--input-merkle-tree-collection "./settlement-merkle-trees.json" \
--input-settlement-collection "./settlements.json" \
--operator-authority "$$VALIDATOR_BONDS_OPERATOR_AUTHORITY" \
--fee-payer "$$PSR_TX_FEE_WALLET" \
--rent-payer "$$PSR_RENT_PAYER_WALLET" \
--epoch "$$epoch" | tee -a "./init-execution-report.$$BUILDKITE_RETRY_COUNT"
key: 'init-settlement'
artifact_paths:
- "./init-execution-report.*"
retry:
automatic:
- exit_status: 100
limit: 5
- label: ":memo: Notification setup: Init Settlements"
commands:
- 'epoch=$(buildkite-agent meta-data get epoch)'
- |
build_result=$(buildkite-agent step get "outcome" --step "init-settlement")
if [[ -z "$$build_result" || "$$build_result" =~ "failed" ]]; then
notification_result="FAILED"
notification_color="15158332"
else
notification_result="SUCCEEDED"
notification_color="52224"
fi
- 'buildkite-agent meta-data set notification_color "$$notification_color"'
- 'buildkite-agent meta-data set notification_result "$$notification_result"'
- 'echo "Notification result: $$notification_result, color: $$notification_color"'
- 'mkdir ./reports'
- 'buildkite-agent artifact download --include-retried-jobs "init-execution-report.*" ./reports/ || echo "No report ERROR" > ./reports/init-execution-report.error'
- 'report_path="./reports/$(ls -v1 reports/ | tail -n 1)"'
- 'cp "$$report_path" ./init-report.txt'
- 'buildkite-agent meta-data set attempts_count "$(grep -c ATTEMPT ./init-report.txt)"'
artifact_paths:
- "./init-report.txt"
key: 'notification-setup-init'
depends_on: "init-settlement"
allow_dependency_failure: true
- wait: ~
- label: ":floppy_disk: :arrow_right: :cloud: Upload artifacts Init Settlements"
commands:
- epoch=$(buildkite-agent meta-data get epoch)
- claim_type=$(buildkite-agent meta-data get claim_type)
- buildkite-agent artifact download --include-retried-jobs init-report.txt . || echo "UNKNOWN ERROR" > init-report.txt
- gcloud storage cp ./init-report.txt "$gs_bucket/$$epoch/buildkite/$${claim_type}-init-settlement-report.$(date +%s).txt"
depends_on: "notification-setup-init"
allow_dependency_failure: true
- label: ":mega: Notification :fast_forward: Monitoring Initialize Settlements"
commands:
- epoch=$(buildkite-agent meta-data get epoch)
- notification_result=$(buildkite-agent meta-data get notification_result || echo "UNKNOWN")
- notification_color=$(buildkite-agent meta-data get notification_color || echo "15158332")
- attempts_count=$(buildkite-agent meta-data get attempts_count || echo "UNKNOWN")
- buildkite-agent artifact download --include-retried-jobs init-report.txt . || echo "UNKNOWN ERROR" > init-report.txt
- |
curl "$(buildkite-agent meta-data get discord_webhook)" \
-F 'payload_json={
"embeds":[{
"title": "Init Settlements '"$notification_result"' for Validator Bonds ('"$$epoch"') after '"$$attempts_count"' attempts",
"url": "'"$$BUILDKITE_BUILD_URL"'",
"color": "'"$$notification_color"'"
}]
}' \
-F "file1=@./init-report.txt"
depends_on: "notification-setup-init"
allow_dependency_failure: true
- label: ":mega: Notification :fast_forward: PSR feed"
commands:
- claim_type=$(buildkite-agent meta-data get claim_type)
- epoch=$(buildkite-agent meta-data get epoch)
- gcloud storage cp "$gs_bucket/$$epoch/$${claim_type}-discord-public-report.txt" "./discord-public-report.txt"
- |
split_command=$(which split >&2 > /dev/null && echo "split" || echo "gsplit")
# discord max message length is 2000 characters
$$split_command -C 2000 "./discord-public-report.txt" './discord-public-report.txt.chunk-'
for report_chunk in './discord-public-report.txt.chunk-'*; do
report=$(cat "$$report_chunk")
curl "$$DISCORD_WEBHOOK_PSR_FEED" -H "Content-Type: application/json" -d "$(jq -n --arg report "$$report" '{"content": ("```\n"+$$report+"```"), "embeds": null, "attachments": []}')"
done
depends_on: "notification-setup-init"
allow_dependency_failure: true
if: "build.env('NOTIFY_PSR_FEED') != null"
- wait: ~
continue_on_failure: true
- label: ":gear: Upload and trigger fund-settlements pipeline"
commands:
- epoch=$(buildkite-agent meta-data get epoch)
- claim_type=$(buildkite-agent meta-data get claim_type)
- 'echo "Epoch: $$epoch, Claim type: $$claim_type"'
- buildkite-agent pipeline upload .buildkite/fund-settlements.yml
depends_on: "init-settlement"
allow_dependency_failure: true
- wait: ~
- label: ":unlock: Concurrency gate unlock"
command: echo "End of concurrency gate <--"
concurrency_group: 'validator-bonds/init-settlements${CLAIM_TYPE}'
concurrency: 1