feature: Add script to download ledger snapshot and options

node-runner-cli/commands/dockercommand.py

@@ -144,6 +144,16 @@ def dockercommand(dockercommand_args=[], parent=docker_parser):
             default="",
             choices=["true", "false"],
         ),
+        argument(
+            "-dcs",
+            "--downloadcommunitysnapshot",
+            help="Boolean to indicate if in case of empty ledger, download latest community snapshot"
+            "Set this to false to not download the latest community snapshot"
+            f"The default value is true if not provided",
+            default="true",
+            action="store",
+            choices=["true", "false"],
+        ),
     ]
 )
 def config(args):

node-runner-cli/config/CoreDockerConfig.py

@@ -19,6 +19,7 @@ def __init__(self, config_dict: dict):
         self.core_release: str = ""
         self.repo: str = os.getenv(CORE_DOCKER_REPO_OVERRIDE, "radixdlt/babylon-node")
         self.data_directory: str = f"{Helpers.get_home_dir()}/babylon-ledger"
+        self.download_community_snapshot: bool = True
         self.trusted_node: str = ""
         self.memory_limit: str = "14000m"
         self.validator_address: str = ""
@@ -53,6 +54,12 @@ def ask_data_directory(self):
         if self.data_directory:
             Path(self.data_directory).mkdir(parents=True, exist_ok=True)
 
+    def ask_download_community_snapshot(self):
+        if "DETAILED" in SetupMode.instance().mode:
+            self.download_community_snapshot = (
+                BaseSetup.get_download_community_snapshot()
+            )
+
     def set_trusted_node(self, trusted_node):
         if not trusted_node:
             trusted_node = Prompts.ask_trusted_node()
@@ -64,6 +71,7 @@ def ask_config(self, release, trustednode, ks_password, new_keystore, validator)
         self.ask_validator_address(validator)
         self.keydetails = BaseSetup.ask_keydetails(ks_password, new_keystore)
         self.ask_data_directory()
+        self.ask_download_community_snapshot()
         return self
 
     def set_validator_address(self, validator_address: str):

node-runner-cli/setup/BaseSetup.py

@@ -189,6 +189,25 @@ def get_data_dir(create_dir=True):
             run_shell_command(f"sudo mkdir -p {data_dir_path}", shell=True)
         return data_dir_path
 
+    @staticmethod
+    def get_download_community_snapshot():
+        Helpers.section_headline("DOWNLOAD COMMUNITY SNAPSHOT")
+        print(f"\n Latest snapshot version of the ledger can be downloaded")
+        download_community_snapshot_string = Helpers.input_guestion(
+            f"\nRadix node stores all the ledger data on a folder."
+            f"Downloading latest snapshot of the ledger will allow to sync faster when starting the node"
+            f"if the ledger folder is empty."
+            f"\n{bcolors.WARNING}Press Enter to accept default or choose to download latest snapshot of the ledger [true/false]: ",
+            QuestionKeys.input_ledger_snapshot,
+        )
+        if download_community_snapshot_string == "":
+            download_community_snapshot = True
+        elif download_community_snapshot_string == "false":
+            download_community_snapshot = False
+        else:
+            download_community_snapshot = True
+        return download_community_snapshot
+
     @staticmethod
     def load_all_config(configfile):
         yaml.add_representer(type(None), Helpers.represent_none)

node-runner-cli/templates/radix-fullnode-compose.yml.j2

@@ -6,13 +6,72 @@
 version: '3.8'
 services:
 {% if core_node is not none and core_node is defined %}
+  {% if core_node.download_community_snapshot == true %}
+  download-community-ledger-snapshot:
+    build:
+      context: .
+      dockerfile_inline: |
+            FROM ubuntu:20.04 as BUILD
+            # Install necessary dependencies
+            ENV DEBIAN_FRONTEND noninteractive
+            RUN apt-get update && apt-get install -y \
+                wget \
+                zstd \
+                bash \
+                build-essential \
+                curl \
+                bash \ 
+                aria2 \
+                zstd \
+                && rm -rf /var/lib/apt/lists/*
+            # Create the /usr/local/scripts directory
+            RUN mkdir -p /usr/local/scripts
+            RUN echo '#!/bin/bash\n\
+            DATA_DIR=$${1:-"/data"}\n\
+            if [ ! -d "$$DATA_DIR" ]; then\n\
+                echo "Creating $$DATA_DIR directory..."\n\
+                mkdir -p "\$$DATA_DIR"\n\
+            fi\n\
+
+            if [ -z "$(ls -A "$$DATA_DIR")" ]; then\n\
+                cd $$DATA_DIR\n\
+                echo "Directory $$DATA_DIR is empty. Downloading LedgerSnapshot..."\n\
+                echo "Fetching and executing the latest snapshot script from Radix..."\n\
+                wget https://snapshots.radix.live/latest-snapshot-INDEX.sh --no-check-certificate\n\
+                echo "Starting snapshot download"\n\
+                bash latest-snapshot-INDEX.sh\n\
+
+                if [ $? -eq 0 ]; then\n\
+                    echo "Snapshot download and execution completed successfully."\n\
+                else\n\
+                    echo "Snapshot download or execution failed."\n\
+                    exit 1\n\
+                fi\n\
+                tar --use-compress-program=zstdmt -xvf RADIXDB-INDEX.tar.zst --exclude=./address_book -C .\n\
+                rm -rf RADIXDB-INDEX.*\n\
+                rm -rf latest-snapshot-INDEX.sh*\n\
+                echo "Snapshot restored"\n\
+            else\n\
+                echo "Directory $$DATA_DIR is not empty. Downloading Ledger Snapshot aborted:"\n\
+                ls -l "$$DATA_DIR"\n\ 
+            fi' > /usr/local/scripts/downloadLedgerSnapshot.sh
+            RUN chmod +x /usr/local/scripts/downloadLedgerSnapshot.sh
+            ENTRYPOINT ["/usr/local/scripts/downloadLedgerSnapshot.sh"]
+    volumes:
+      - {{ core_node.data_directory }}:{{ core_node.data_directory }}
+    entrypoint: ["/usr/local/scripts/downloadLedgerSnapshot.sh", {{ core_node.data_directory }}]  # Pass the environment variable as an argument
+  {% endif %}   
   core:
     cap_add:
     - NET_ADMIN
     {% if core_node.advanced_user_envs is defined and core_node.advanced_user_envs is not none %}
     env_file:
       - {{core_node.advanced_user_envs}}
     {% endif %}
+    {% if core_node.download_community_snapshot == true %}
+    depends_on:
+      - download-community-ledger-snapshot
+    {% endif %}   
     environment:
       JAVA_OPTS: {{core_node.java_opts or '--enable-preview -server -Xms12g -Xmx12g  -XX:MaxDirectMemorySize=2048m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseCompressedOops -Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts -Djavax.net.ssl.trustStoreType=jks -Djava.security.egd=file:/dev/urandom -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector'}}
       RADIXDLT_CORE_API_PORT: {{core_node.core_api_port}}

node-runner-cli/templates/systemd-default.config.j2

@@ -5,7 +5,7 @@
 
 ntp=false
 ntp.pool=pool.ntp.org
-
+download_community_snapshot=true
 network.id={{common_config.network_id}}
 {% if common_config.genesis_bin_data_file is not none %}
 network.genesis_data_file={{common_config.genesis_bin_data_file}}

node-runner-cli/test-prompts/core-gateway-all-local.yml

@@ -7,6 +7,7 @@
 - input_path_keystore: "/home/runner/babylon-node"
 - enter_keystore_name: "node-keystore.ks"
 - input_ledger_path: "/tmp/babylon-ledger"
+- input_ledger_snapshot: "false"
 - core_nginx_setup: "true"
 - setup_gateway: N
 - input_nginx_release: "development-latest"

node-runner-cli/test-prompts/core.yml

@@ -6,6 +6,7 @@
 - input_path_keystore: "/home/runner/babylon-node"
 - enter_keystore_name: "node-keystore.ks"
 - input_ledger_path: "/tmp/babylon-ledger"
+- input_ledger_snapshot: "true"
 - core_nginx_setup: "true"
 - setup_gateway: N
 - input_nginx_release: "development-latest"

node-runner-cli/test-prompts/corenode-01.yml

@@ -7,6 +7,7 @@
 - input_path_keystore: "/home/runner/babylon-node"
 - enter_keystore_name: "node-keystore.ks"
 - input_ledger_path: "/tmp/babylon-ledger"
+- input_ledger_snapshot: "true"
 - core_nginx_setup: "true"
 - setup_gateway: "false"
 - input_nginx_release: "1.3.1"

node-runner-cli/test-prompts/corenode-02.yml

@@ -7,6 +7,7 @@
 - input_path_keystore: "/home/runner/babylon-node"
 - enter_keystore_name: "node-keystore.ks"
 - input_ledger_path: "/tmp/babylon-ledger"
+- input_ledger_snapshot: "true"
 - core_nginx_setup: "false"
 - setup_gateway: "false"
 - enable_engine_state_api: N

node-runner-cli/test-prompts/gateway-local-core-remote-postgress.yml

@@ -6,6 +6,7 @@
 - input_path_keystore: "/home/runner/babylon-node"
 - enter_keystore_name: "node-keystore.ks"
 - input_ledger_path: "/tmp/babylon-ledger"
+- input_ledger_snapshot: "true"
 - core_nginx_setup: "true"
 - setup_gateway: Y
 - input_core_api_address: "http://core:3333/core"

node-runner-cli/test-prompts/gateway.yml

@@ -6,6 +6,7 @@
 - input_path_keystore: "/home/runner/babylon-node"
 - enter_keystore_name: "node-keystore.ks"
 - input_ledger_path: "/tmp/babylon-ledger"
+- input_ledger_snapshot: "true"
 - core_nginx_setup: "true"
 - setup_gateway: Y
 - input_nginx_release: "development-latest"

node-runner-cli/test-prompts/individual-prompts/systemd_install_default_config.yml

@@ -1,4 +1,5 @@
 ---
 - select_network: S
 - input_ledger_path: "/tmp/babylon-ledger"
+- input_ledger_snapshot: "true"
 - continue_systemd_install: Y

node-runner-cli/tests/unit/test_prompt_feeder_local.py

@@ -21,6 +21,7 @@ def test_docker_config_gateway_external_postgres(self, mockout):
         PROMPT_KEYSTORE_LOCATION = ""
         PROMPT_KEYSTORE_FILE_NAME = ""
         PROMPT_LEDGER_LOCATION = ""
+        PROMPT_LEDGER_SNAPSHOT = ""
         PROMPT_NGINX_PROTECT_CORE_API = ""
         PROMPT_ENABLE_GATEWAY = "Y"
         PROMPT_GATEWAY_CORE_API_URL = "http://core:3333/core"
@@ -48,6 +49,7 @@ def test_docker_config_gateway_external_postgres(self, mockout):
                 PROMPT_KEYSTORE_LOCATION,
                 PROMPT_KEYSTORE_FILE_NAME,
                 PROMPT_LEDGER_LOCATION,
+                PROMPT_LEDGER_SNAPSHOT,
                 PROMPT_NGINX_PROTECT_CORE_API,
                 PROMPT_ENABLE_GATEWAY,
                 PROMPT_GATEWAY_CORE_API_URL,
@@ -97,6 +99,7 @@ def test_docker_config_all_local(self, mockout):
         PROMPT_KEYSTORE_LOCATION = "/tmp/babylon-node-config"
         PROMPT_KEYSTORE_FILE_NAME = "node-keystore.ks"
         PROMPT_LEDGER_LOCATION = "/tmp/data"
+        PROMPT_LEDGER_SNAPSHOT = "true"
         PROMPT_NGINX_PROTECT_CORE_API = "true"
         PROMPT_ENABLE_GATEWAY = "Y"
         PROMPT_GATEWAY_CORE_API_URL = "http://core:3333/core"
@@ -122,6 +125,7 @@ def test_docker_config_all_local(self, mockout):
                 PROMPT_KEYSTORE_LOCATION,
                 PROMPT_KEYSTORE_FILE_NAME,
                 PROMPT_LEDGER_LOCATION,
+                PROMPT_LEDGER_SNAPSHOT,
                 PROMPT_NGINX_PROTECT_CORE_API,
                 PROMPT_ENABLE_GATEWAY,
                 PROMPT_GATEWAY_CORE_API_URL,

node-runner-cli/tests/unit/test_systemd.py

@@ -182,7 +182,7 @@ def test_systemd_setup_default_config(self, mockout):
 
 ntp=false
 ntp.pool=pool.ntp.org
-
+download_community_snapshot=true
 network.id=1
 
 node.key.path=/home/radixdlt/babylon-node-config/node-keystore.ks
@@ -236,7 +236,7 @@ def test_systemd_setup_default_config_without_validator(self, mockout):
 
 ntp=false
 ntp.pool=pool.ntp.org
-
+download_community_snapshot=true
 network.id=1
 
 node.key.path=/home/radixdlt/babylon-node-config/node-keystore.ks
@@ -290,7 +290,7 @@ def test_systemd_setup_default_config_jinja(self, mockout):
 
 ntp=false
 ntp.pool=pool.ntp.org
-
+download_community_snapshot=true
 network.id=1
 
 node.key.path=/home/radixdlt/babylon-node-config/node-keystore.ks

node-runner-cli/utils/PromptFeeder.py

@@ -20,6 +20,7 @@ class QuestionKeys:
     input_path_keystore = "input_path_keystore"
     enter_keystore_name = "enter_keystore_name"
     input_ledger_path = "input_ledger_path"
+    input_ledger_snapshot = "input_ledger_snapshot"
     core_nginx_setup = "core_nginx_setup"
     gateway_nginx_setup = "gateway_nginx_setup"
     setup_gateway = "setup_gateway"