DO-1644 / Migration babylon

.github/workflows/ci.yml

@@ -67,16 +67,27 @@
           token: ${{ secrets.GITHUB_TOKEN }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build the binary for ubuntu
+      - name: Build the binary for ubuntu jammy
         run: |
           cd node-runner-cli
-          make output-ubuntu-focal
+          make output-ubuntu-jammy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: "Upload generated cli file"
         uses: actions/[email protected]
         with:
           name: ubuntu 22.04
+          path: "${{ github.workspace }}/node-runner-cli/out/ubuntu/jammy/radixnode"
+      - name: Build the binary for ubuntu focal
+        run: |
+          cd node-runner-cli
+          make output-ubuntu-focal
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Upload generated cli file"
+        uses: actions/[email protected]
+        with:
+          name: ubuntu 20.04
           path: "${{ github.workspace }}/node-runner-cli/out/ubuntu/focal/radixnode"
 
   upload-asset-store:
@@ -93,14 +104,14 @@
       - name: Checkout
         uses: actions/[email protected]
       - name: set branchname with commit
         run: |
           ls -l
           BRANCH_NAME_WITH_HYPENS=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g')
           COMMIT=$(git log -1 --format=%h )
           BRANCH_WITH_COMMIT=$BRANCH_NAME_WITH_HYPENS-$COMMIT
           echo "BRANCH_WITH_COMMIT=$BRANCH_WITH_COMMIT" >> $GITHUB_ENV
       - name: Configure AWS Region
         run: echo "AWS_DEFAULT_REGION=eu-west-1" >> $GITHUB_ENV
       - id: install-aws-cli
         uses: unfor19/install-aws-cli-action@v1
         with:
@@ -115,11 +126,11 @@
         with:
           name: ubuntu 22.04
       - name: Upload cli to asset store
         run: |
           ls */**
           aws s3 cp radixnode s3://${{secrets.ARTIFACT_AWS_BUCKET }}/radixnode/${{env.BRANCH_WITH_COMMIT}}/radixnode-ubuntu-22.04
 
-  upload-release:
+  upload-release-jammy:
     runs-on: ubuntu-22.04
     if: ${{ github.event_name == 'release' }}
     needs:
@@ -129,25 +140,40 @@
         uses: actions/download-artifact@v3
         with:
           name: ubuntu 22.04
-      - name: Get release
-        id: get_release
-        uses: bruceadams/[email protected]
+      - name: Upload radixcli ubuntu binary
+        uses: actions/upload-release-asset@v1
         env:
-          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: ./radixnode
+          asset_name: radixnode-ubuntu-22.04
+          asset_content_type: application/octet-stream
+        if: ${{ github.event_name == 'release' }}
 
+  upload-release-focal:
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name == 'release' }}
+    needs:
+      - package_ubuntu_cli
+    steps:
+      - name: Download packaged cli
+        uses: actions/download-artifact@v3
+        with:
+          name: ubuntu 20.04
       - name: Upload radixcli ubuntu binary
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: ${{ steps.get_release.outputs.upload_url }}
+          upload_url: ${{ github.event.release.upload_url }}
           asset_path: ./radixnode
-          asset_name: radixnode-ubuntu-22.04
+          asset_name: radixnode-ubuntu-20.04
           asset_content_type: application/octet-stream
         if: ${{ github.event_name == 'release' }}
 
   test-systemd:
     runs-on: [node-only]
     needs:
       - package_ubuntu_cli
     steps:
@@ -171,7 +197,7 @@
           # ./radixnode systemd dependencies
           echo "expecting the dependencies to be already installed"
       - name: Run systemd config
         run: |
           ls -a
           chmod +x ./radixnode
           echo "HOME=$HOME"
@@ -193,7 +219,7 @@
           echo "PATH=$PWD"
           ./radixnode systemd install
       - name: Get Logs and Status
         run: |
           sleep 15
           sudo tail /var/log/syslog  -n 100
           ./radixnode auth set-admin-password --setupmode SYSTEMD -p $NGINX_ADMIN_PASSWORD
@@ -240,7 +266,7 @@
           sudo apt-get install containerd runc
           ./radixnode docker dependencies
       - name: core-gateway-all-local
         run: |
           ls -a
           chmod +x ./radixnode
           mkdir -p $HOME/node-config
@@ -248,6 +274,7 @@
           echo "PATH=$PWD"
           export PROMPT_FEEDS="node-runner-cli/test-prompts/core-gateway-all-local.yml"
           ./radixnode docker config -m DETAILED \
+           -d $HOME/node-config \
            -k $KEYSTORE_PASSWORD -nk -a
         env:
           KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
@@ -255,11 +282,12 @@
           RADIXDLT_GATEWAY_VERSION_OVERRIDE: "0.0.1-rc1"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: corenode-01
         run: |
           ls -a
           chmod +x ./radixnode
           export PROMPT_FEEDS="node-runner-cli/test-prompts/corenode-01.yml"
           ./radixnode docker config -m DETAILED \
+           -d $HOME/node-config \
            -k $KEYSTORE_PASSWORD -nk -a
         env:
           KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
@@ -267,10 +295,11 @@
           RADIXDLT_GATEWAY_VERSION_OVERRIDE: "0.0.1-rc1"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: corenode-02
         run: |
           ls -a
           export PROMPT_FEEDS="node-runner-cli/test-prompts/corenode-02.yml"
           ./radixnode docker config -m DETAILED \
+           -d $HOME/node-config \
            -k $KEYSTORE_PASSWORD -nk -a
         env:
           KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
@@ -283,6 +312,7 @@
 #          ls -a
 #          export PROMPT_FEEDS="node-runner-cli/test-prompts/gateway-remote-core-local-postgress.yml"
 #          ./radixnode docker config -m DETAILED \
+#           -d $HOME/node-config \
 #           -k $KEYSTORE_PASSWORD -nk -a
 #        env:
 #          KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
@@ -293,6 +323,7 @@
 #          ls -a
 #          export PROMPT_FEEDS="node-runner-cli/test-prompts/gateway-remote-core-remote-postgress.yml"
 #          ./radixnode docker config -m DETAILED \
+#           -d $HOME/node-config \
 #           -k $KEYSTORE_PASSWORD -nk -a
 #        env:
 #          KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
@@ -323,7 +354,7 @@
           sudo apt-get install containerd runc
           ./radixnode docker dependencies
       - name: Setup config
         run: |
           chmod +x ./radixnode
           mkdir -p $HOME/node-config
           export DISABLE_VERSION_CHECK=true
@@ -331,6 +362,7 @@
           export DOCKER_COMPOSE_LOCATION="/usr/local/bin/docker-compose"
           export PROMPT_FEEDS="node-runner-cli/test-prompts/core-gateway-all-local.yml"
           ./radixnode docker config -m DETAILED \
+           -d $HOME/node-config \
            -k $KEYSTORE_PASSWORD -nk -a
         env:
           KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
@@ -358,7 +390,7 @@
           username: _json_key
           password: ${{ secrets.GCR_EU_DEV_JSON_KEY }}
       - name: Run CLI setup
         run: |
           export DISABLE_VERSION_CHECK=true
           export DOCKER_COMPOSE_LOCATION="/usr/local/bin/docker-compose"
           ./radixnode docker install -f $HOME/node-config/config.yaml -a
@@ -376,7 +408,7 @@
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run Monitoring setup
         run: |
           export DOCKER_COMPOSE_LOCATION="/usr/local/bin/docker-compose"
           ./radixnode monitoring config \
                         -m MONITOR_CORE  \

Readme.adoc

@@ -94,59 +94,3 @@ optional arguments:
   -h, --help            show this help message and exit
 
 ----
-
-
-== Testing with babylon releases
-
-The babylon release is still not public. Therefor testing with babylon will be difficult.
-Docker images, version references and binaries can not be fetched without authorization. Because of this, we added this
-documentation as a guideline on how to work with babylon.
-
-[source, bash]
-----
-# This command references https://raw.githubusercontent.com/radixdlt/babylon-nodecli
-# It needs access to github in order to load the ansible files. The babylon-nodecli repository is still private.
-# So a private access token (PAT) is required. This can be put into the environment variable GITHUB_TOKEN and will then
-# be used to fetch the ansible script.
-# See here for more information on PAT: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token
-export GITHUB_TOKEN="token <YOUR-PAT-HERE>"
-/tmp/radixnode docker dependencies
-
-
-# This command heavily uses latest_version. This function extracts the latest
-# version of a resource/repo from github releases. Since the repositories are
-# private, fetching the latest version also requires a PAT. See above.
-
-
-export GITHUB_TOKEN="token <YOUR-PAT-HERE>"
-/tmp/radixnode docker config -m CORE
-
-# There is however not a release yet for all resources. So fetching the latest
-# version might not always be possible. This is the case for the gateway for
-# example. This can be solved by manually overriding the version of these resources.
-# Use the below mentioned environment variables for that.
-export RADIXDLT_APP_VERSION_OVERRIDE=1.3.1
-export RADIXDLT_NGINX_VERSION_OVERRIDE=1.3.3
-export RADIXDLT_CLI_VERSION_OVERRIDE=1.3.2
-export RADIXDLT_GATEWAY_VERSION_OVERRIDE=1.5.0
-/tmp/radixnode docker config -m CORE GATEWAY
-
-
-# Will try to download ansible playbooks as a resource from a github release at:
-# https://raw.githubusercontent.com/radixdlt/babylon-nodecli/<NODE_CLI_VERSION>/node-runner-cli
-# Make sure your CLI version is set to a released version of the babylon-nodecli.
-# This is done by setting a tag at the current commit before building the binary.
-# It will also try to download docker images. As these are not public yet, it is
-# mandatory to download the image you are trying to test youself beforehand.
-# For this you need to log in to the google registry, download the image and
-# then retag it to the expected format. See the example
-
-
-docker login eu.gcr.io # keeping this secret
-docker pull eu.gcr.io/dev-container-repo/babylon-node:pr-247
-docker tag eu.gcr.io/dev-container-repo/babylon-node:pr-247 radixdlt/babylon-node:$RADIXDLT_APP_VERSION_OVERRIDE
-docker pull radixdlt/babylon-nginx:development-latest
-docker tag radixdlt/babylon-nginx:development-latest radixdlt/babylon-nginx:$RADIXDLT_NGINX_VERSION_OVERRIDE
-
-/tmp/radixnode docker install
-----

node-runner-cli/Dockerfile.ubuntufocal

@@ -1,8 +1,8 @@
-FROM ubuntu:22.04 as BUILD
+FROM ubuntu:20.04 as BUILD
 MAINTAINER radixdlt <[email protected]>
 
 ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHON_VERSION 3.10.6
+ENV PYTHON_VERSION 3.7.6
 
 CMD /bin/bash
 
@@ -22,11 +22,11 @@ RUN set -ex \
     && pyenv update
 
 
-RUN CONFIGURE_OPTS=--enable-shared pyenv install 3.10.6
+RUN CONFIGURE_OPTS=--enable-shared pyenv install 3.7.6
 
 
 
-RUN pyenv virtualenv 3.10.6 nodecli
+RUN pyenv virtualenv 3.7.6 nodecli
 RUN pyenv local nodecli
 RUN pip install pyinstaller==4.10
 

node-runner-cli/Dockerfile.ubuntujammy

@@ -0,0 +1,43 @@
+FROM ubuntu:22.04 as BUILD
+MAINTAINER radixdlt <[email protected]>
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHON_VERSION 3.10.6
+
+CMD /bin/bash
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends make build-essential libssl-dev zlib1g-dev \
+     libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+     libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev ca-certificates git  > /dev/null
+
+
+
+ENV PYENV_ROOT /root/.pyenv
+ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
+
+# Install pyenv
+RUN set -ex \
+    && curl https://pyenv.run | bash \
+    && pyenv update
+
+
+RUN CONFIGURE_OPTS=--enable-shared pyenv install 3.10.6
+
+
+
+RUN pyenv virtualenv 3.10.6 nodecli
+RUN pyenv local nodecli
+RUN pip install pyinstaller==4.10
+
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install -r requirements.txt
+
+COPY . /app
+RUN pyinstaller --onefile --windowed radixnode.spec
+
+RUN DISABLE_VERSION_CHECK=true /app/dist/radixnode version
+
+FROM scratch AS export-stage
+COPY --from=BUILD /app/dist /

node-runner-cli/Makefile

@@ -7,6 +7,9 @@ install:
 .PHONY: output
 output-ubuntu-focal: install
 	DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --output type=local,dest=out/ubuntu/focal --progress plain -f Dockerfile.ubuntufocal  .
+.PHONY: output
+output-ubuntu-jammy: install
+	DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --output type=local,dest=out/ubuntu/jammy --progress plain -f Dockerfile.ubuntujammy  .
 
 .PHONY: local
 local: install

node-runner-cli/commands/dockercommand.py

@@ -52,7 +52,14 @@ def dockercommand(dockercommand_args=[], parent=docker_parser):
                   \n\nGATEWAY: Use this value to setup GATEWAY using defaults.
                   \n\nDETAILED: Default value if not provided. This mode takes your through series of questions.
                   """,
-             choices=["CORE", "GATEWAY", "DETAILED"], action="store"),
+             choices=["CORE", "GATEWAY", "DETAILED", "MIGRATION"], action="store"),
+    argument("-miu", "--migration_url", help="The url of the olympia node to migrate the ledger from", action="store"),
+    argument("-miau", "--migration_auth_user", help="The user to authenticate to the olympia node for migration",
+             action="store"),
+    argument("-miap", "--migration_auth_password",
+             help="The password to authenticate to the olympia node for migration", action="store"),
+    argument("-miba", "--migration_bech_url", help="The bech url of the olympia node to migrate the ledger from",
+             action="store"),
     argument("-n", "--networkid",
              help="Network id of network you want to connect.For stokenet it is 2 and for mainnet it is 1."
                   "If not provided you will be prompted to enter a value ",
@@ -89,6 +96,11 @@ def config(args):
     new_keystore = args.newkeystore
     validator = args.validator
 
+    olympia_node_url = args.migration_url
+    olympia_node_bech32_address = args.migration_auth_user
+    olympia_node_auth_user = args.migration_auth_user
+    olympia_node_auth_password = args.migration_auth_password
+
     if "DETAILED" in setupmode.mode and len(setupmode.mode) > 1:
         print(f"{bcolors.FAIL}You cannot have DETAILED option with other options together."
               f"\nDETAILED option goes through asking each and every question that to customize setup. "
@@ -154,6 +166,11 @@ def config(args):
         # else:
         #     configuration.common_config.nginx_settings.protect_gateway = "false"
 
+    if "MIGRATION" in setupmode.mode:
+        configuration.migration.ask_migration_config(olympia_node_url, olympia_node_auth_user,
+                                                     olympia_node_auth_password,
+                                                     olympia_node_bech32_address)
+
     if configuration.common_config.check_nginx_required():
         configuration.common_config.ask_nginx_release()
         if configuration.core_node.enable_transaction == "true":
@@ -165,6 +182,8 @@ def config(args):
         configuration.common_config.nginx_settings = None
 
     config_to_dump["common_config"] = dict(configuration.common_config)
+    config_to_dump["migration"] = dict(configuration.migration)
+    config_to_dump["gateway_settings"] = dict(configuration.gateway_settings)
 
     yaml.add_representer(type(None), Helpers.represent_none)
     Helpers.section_headline("CONFIG is Generated as below")

node-runner-cli/commands/systemdcommand.py

@@ -50,7 +50,14 @@ def systemdcommand(systemdcommand_args=None, parent=systemd_parser):
               \n\nCORE: Use this value to setup CORE using defaults.
               \n\nDETAILED: Default value if not provided. This mode takes your through series of questions.
               """,
-             choices=["CORE", "DETAILED"], action="store"),
+             choices=["CORE", "DETAILED", "MIGRATION"], action="store"),
+    argument("-miu", "--migration_url", help="The url of the olympia node to migrate the ledger from", action="store"),
+    argument("-miau", "--migration_auth_user", help="The user to authenticate to the olympia node for migration",
+             action="store"),
+    argument("-miap", "--migration_auth_password",
+             help="The password to authenticate to the olympia node for migration", action="store"),
+    argument("-miba", "--migration_bech_url", help="The bech url of the olympia node to migrate the ledger from",
+             action="store"),
     argument("-n", "--networkid",
              help="Network id of network you want to connect.For stokenet it is 2 and for mainnet it is 1."
                   "If not provided you will be prompted to enter a value ",
@@ -95,6 +102,11 @@ def config(args):
     data_directory = args.data_directory
     new_keystore = args.newkeystore
 
+    olympia_node_url = args.migration_url
+    olympia_node_bech32_address = args.migration_auth_user
+    olympia_node_auth_user = args.migration_auth_user
+    olympia_node_auth_password = args.migration_auth_password
+
     Helpers.section_headline("CONFIG FILE")
     config_file = f"{args.configdir}/config.yaml"
     Path(f"{args.configdir}").mkdir(parents=True, exist_ok=True)
@@ -117,6 +129,12 @@ def config(args):
                                                                     trustednode,
                                                                     keystore_password, new_keystore)
     configuration.common_config.ask_enable_nginx_for_core(nginx_on_core)
+
+    if "MIGRATION" in setupmode.mode:
+        configuration.migration.ask_migration_config(olympia_node_url, olympia_node_auth_user,
+                                                     olympia_node_auth_password,
+                                                     olympia_node_bech32_address)
+
     config_to_dump["core_node"] = dict(configuration.core_node)
 
     if configuration.common_config.check_nginx_required():
@@ -126,6 +144,9 @@ def config(args):
 
     config_to_dump["common_config"] = dict(configuration.common_config)
 
+    config_to_dump["migration"] = dict(configuration.migration)
+    config_to_dump["gateway_settings"] = dict(configuration.gateway_settings)
+
     yaml.add_representer(type(None), Helpers.represent_none)
     Helpers.section_headline("CONFIG is Generated as below")
     print(f"\n{yaml.dump(config_to_dump)}")
@@ -253,14 +274,19 @@ def restart(args):
         sys.exit(1)
 
 
-@systemdcommand([])
+@systemdcommand([
+    argument("-s", "--skip", default="false",
+             help="Skip installation of base dependencies",
+             action="store_true")
+])
 def dependencies(args):
     """
     This commands installs all necessary software on the Virtual Machine(VM).
     Run this command on fresh VM or on an existing VM  as the command is tested to be idempotent
     """
 
-    Base.dependencies()
+    if not args.skip:
+        Base.dependencies()
     SystemD.install_java()
     SystemD.setup_user()
     SystemD.make_etc_directory()