S3 snapshots

playbooks/group_vars/opensearch.yml

@@ -116,14 +116,11 @@ iac_enable: false
 ### ipfs-search.com
 os_enable_security: false
 
-snapshot_repo_name: filebase
+snapshot_repo_name: s3
 snapshot_repo_settings:
-  client: filebase
-  region: us-east-1
-  bucket: ipfs-search-snapshots-v8
-  endpoint: s3.filebase.com
+  client: s3
+  region: eu-central-1
+  storage_class: standard_ia
+  bucket: ipfs-search-snapshots-v9
   chunk_size: 524288000B
   compress: true
-  read_timeout: "5m"
-  max_retries: 10
-  max_snapshot_bytes_per_sec: "60mb"

roles/vendor/opensearch_rolling_upgrade/tasks/init.yml

@@ -0,0 +1,2 @@
+---
+# Intentionally empty to allow loading handlers etc.

roles/vendor/snapshots/defaults/main.yml

@@ -2,4 +2,49 @@
 os_home: /usr/share/opensearch
 os_conf_dir: /usr/share/opensearch/config
 os_keystore_bin_path: "{{ os_home }}/bin/opensearch-keystore"
-snapshot_hour: "2"
+snapshot_hour: 0
+snapshot_delete_hour: 12
+os_notifications_channel_id: matrix
+os_notifications_slack_url: "{{ slack_url }}"
+os_notifications_channel_config:
+  name: "{{ os_notifications_channel_id }}"
+  description: "Slack notification channel."
+  config_type: slack
+  is_enabled: true
+  slack:
+    url: "{{ os_notifications_slack_url }}"
+snapshot_management_policy_name: ipfs-daily
+snapshot_management_policy:
+  enabled: true
+  schema_version: 16
+  creation:
+    schedule:
+      cron:
+        expression: "16 {{ snapshot_hour }} * * *"
+        timezone: Europe/Lisbon
+    time_limit: 11h
+  deletion:
+    schedule:
+      cron:
+        expression: "16 {{ snapshot_delete_hour }} * * *"
+        timezone: Europe/Lisbon
+    condition:
+      max_age: 30d
+      min_count: 7
+    time_limit: 11h
+  snapshot_config:
+    date_format: yyMMdd_HHmm # 221121_1454
+    date_format_timezone: Europe/Lisbon
+    indices: "ipfs_*"
+    repository: "{{ snapshot_repo_name }}"
+    include_global_state: true
+  #  ignore_unavailable: true
+  #  partial: true
+  notification:
+    channel:
+      id: "{{ os_notifications_channel_id }}"
+    conditions:
+      creation: true
+      deletion: true
+      failure: true
+      time_limit_exceeded: true

roles/vendor/snapshots/tasks/main.yml

@@ -1,9 +1,17 @@
 ---
+- name: Import handlers and vars from opensearch rolling upgrade
+  import_role:
+    name: "vendor/opensearch_rolling_upgrade"
+    tasks_from: "init.yml"
 - name: Configuring S3 snapshot repo
   import_tasks: s3.yml
   tags:
     - snapshots-s3
-- name: Configuring snapshot cron job
-  import_tasks: cron.yml
+- name: Configuring OS notifications
+  import_tasks: notifications.yml
   tags:
-    - snapshots-cron
+    - snapshots-notifications
+- name: Configuring snapshot management
+  import_tasks: management.yml
+  tags:
+    - snapshots-management

roles/vendor/snapshots/tasks/management.yml

@@ -0,0 +1,33 @@
+- name: Configure snapshot management
+  run_once: true
+  block:
+  - name: Get snapshot management policy
+    uri:
+      url: http://127.0.0.1:9200/_plugins/_sm/policies/{{ snapshot_management_policy_name }}
+      method: GET
+      status_code: 200, 404
+    register: snapshot_management_policy_get
+    changed_when: false
+  - name: Remove non idempotent fields from policy
+    set_fact:
+      current_policy: "{{ snapshot_management_policy_get.json.sm_policy | combine({'schedule': omit, 'enabled_time': omit, 'last_updated_time': omit, 'name': omit}) }}"
+    when: snapshot_management_policy_get.status == 200
+  - name: Create snapshot management policy
+    uri:
+      url: "http://127.0.0.1:9200/_plugins/_sm/policies/{{ snapshot_management_policy_name }}"
+      method: POST
+      status_code: 201
+      body_format: json
+      body: "{{ snapshot_management_policy }}"
+    changed_when: true
+    when: snapshot_management_policy_get.status == 404
+  - name: Update snapshot management policy
+    uri:
+      url: "http://127.0.0.1:9200/_plugins/_sm/policies/{{ snapshot_management_policy_name }}?if_seq_no={{ snapshot_management_policy_get.json._seq_no }}&if_primary_term={{ snapshot_management_policy_get.json._primary_term }}"
+      method: PUT
+      status_code: 200
+      body_format: json
+      body: "{{ snapshot_management_policy }}"
+    register: snapshot_management_policy_post
+    changed_when: true
+    when: snapshot_management_policy_get.status == 200 and current_policy != snapshot_management_policy

roles/vendor/snapshots/tasks/notifications.yml

@@ -0,0 +1,37 @@
+- name: Configure notifications
+  run_once: true
+  block:
+  - name: Get channel
+    uri:
+      url: http://127.0.0.1:9200/_plugins/_notifications/configs/{{ os_notifications_channel_id }}
+      method: GET
+      status_code: 200, 404
+    register: channel_get
+    changed_when: false
+  - name: Remove non idempotent fields from channel
+    set_fact:
+      current_config: "{{ channel_get.json.config_list[0].config }}"
+    when: channel_get.status == 200
+  - name: Create channel
+    uri:
+      url: "http://127.0.0.1:9200/_plugins/_notifications/configs/"
+      method: POST
+      status_code: 200
+      body_format: json
+      body:
+        config_id: "{{ os_notifications_channel_id }}"
+        config: "{{ os_notifications_channel_config }}"
+    changed_when: true
+    when: channel_get.status == 404
+  - name: Update channel
+    uri:
+      url: "http://127.0.0.1:9200/_plugins/_notifications/configs/{{ os_notifications_channel_id }}"
+      method: PUT
+      status_code: 200
+      body_format: json
+      body:
+        config_id: "{{ os_notifications_channel_id }}"
+        config: "{{ os_notifications_channel_config }}"
+    register: os_notifications_channel_config_post
+    changed_when: true
+    when: channel_get.status == 200 and current_config != os_notifications_channel_config

roles/vendor/snapshots/tasks/s3.yml

@@ -49,8 +49,3 @@
     url: "http://127.0.0.1:9200/_snapshot/{{ snapshot_repo_name }}/_verify"
     method: POST
   run_once: true
-- name: "Install snapshot script {{ snapshot_repo_name }}"
-  template:
-    src: ipfs-search-snapshot.sh
-    dest: "/usr/local/bin/ipfs-search-snapshot-{{ snapshot_repo_name }}.sh"
-    mode: 0755