From 3b0d8e2d67518e9b9e959e5ec835ef71a396648c Mon Sep 17 00:00:00 2001 From: Julien Adamek Date: Thu, 5 Oct 2023 15:41:22 +0200 Subject: [PATCH] ci: add airgap e2e test (#1034) --- .github/workflows/airgap-e2e.yaml | 154 ++++++++++++++++++++++++++++ tests/Makefile | 4 + tests/assets/net-default-airgap.xml | 15 +++ tests/e2e/airgap_test.go | 72 +++++++++++++ tests/e2e/install_test.go | 2 +- tests/e2e/suite_test.go | 54 +++++----- tests/scripts/build-airgap | 81 +++++++++++++++ 7 files changed, 355 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/airgap-e2e.yaml create mode 100644 tests/assets/net-default-airgap.xml create mode 100644 tests/e2e/airgap_test.go create mode 100755 tests/scripts/build-airgap diff --git a/.github/workflows/airgap-e2e.yaml b/.github/workflows/airgap-e2e.yaml new file mode 100644 index 000000000..959240f37 --- /dev/null +++ b/.github/workflows/airgap-e2e.yaml @@ -0,0 +1,154 @@ +name: Elemental Airgap E2E tests with Rancher Manager + +on: + pull_request: + #test + workflow_dispatch: + inputs: + qase_run_id: + description: Case run ID where the results will be reported + required: false + type: string + cert-manager_version: + description: Version of cert-manager to use + type: string + destroy_runner: + description: Destroy the auto-generated self-hosted runner + default: true + type: boolean + operator_repo: + description: Elemental operator repository to use + type: string + default: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher + os_to_test: + description: OS repository to test (dev/staging/stable) + type: string + default: dev + rancher_version: + description: Rancher Manager channel/version to use for installation + default: stable/latest + type: string + +jobs: + create-runner: + runs-on: ubuntu-latest + outputs: + uuid: ${{ steps.generator.outputs.uuid }} + runner: ${{ steps.generator.outputs.runner }} + public_dns: ${{ steps.dns.outputs.public_dns }} + steps: + # actions/checkout MUST come before auth + - name: Checkout + uses: actions/checkout@v3 + - name: Generate UUID and Runner hostname + id: generator + run: | + UUID=$(uuidgen) + echo "uuid=${UUID}" >> ${GITHUB_OUTPUT} + echo "runner=elemental-ci-${UUID}" >> ${GITHUB_OUTPUT} + - name: Authenticate to GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_CREDENTIALS }} + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@v1 + - name: Create runner + run: | + gcloud compute instances create ${{ steps.generator.outputs.runner }} \ + --source-instance-template elemental-e2e-ci-runner-spot-x86-64-template-n2-standard-16-v4 \ + --zone us-central1-a + - name: Create PAT token secret + run: | + echo -n ${{ secrets.SELF_HOSTED_RUNNER_PAT_TOKEN }} \ + | gcloud secrets create PAT_TOKEN_${{ steps.generator.outputs.uuid }} --data-file=- + - name: Get public dns name in GCP + id: dns + run: | + # Do a timed out loop here, as gcloud can sometimes fail + typeset -i i=0 + while true; do + # Get public IP + PUBLIC_IP=$(gcloud compute instances list 2> /dev/null \ + | awk '/${{ steps.generator.outputs.runner }}/ {print $6}') + # Exit if we reach the timeout or if IP is set + if (( ++i > 10 )) || [[ -n "${PUBLIC_IP}" ]]; then + break + fi + # Wait a little before retrying + sleep 2 + done + # Get the public DNS + PUBLIC_DNS=$(host -l ${PUBLIC_IP} 2> /dev/null \ + | awk '{sub(/\.$/, ""); print $5}') + echo "public_dns=${PUBLIC_DNS}" >> ${GITHUB_OUTPUT} + # Raise an error if either IP and/or DNS are empty + if [[ -z "${PUBLIC_IP}" || -z "${PUBLIC_DNS}" ]]; then + echo "PUBLIC_IP and/or PUBLIC_DNS are empty!" >&2 + false + fi + e2e: + needs: create-runner + runs-on: ${{ needs.create-runner.outputs.uuid }} + env: + ARCH: amd64 + CERT_MANAGER_VERSION: v1.12.2 + # Distribution to use to host Rancher Manager (K3s) + K8S_UPSTREAM_VERSION: 1.26.7 + # QASE variables + QASE_API_TOKEN: ${{ secrets.qase_api_token }} + QASE_RUN_ID: ${{ inputs.qase_run_id }} + # For Rancher Manager + RANCHER_VERSION: stable/2.7.6 + TIMEOUT_SCALE: 3 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version-file: tests/go.mod + - name: Prepare the archive file to send to air-gapped nodes + run: | + cd tests && make e2e-prepare-archive && make e2e-airgap-rancher + + +# - name: Deploy a node to join Rancher manager +# if: inputs.test_type == 'ui' +# env: +# ISO_BOOT: ${{ inputs.iso_boot }} +# VM_INDEX: 1 +# VM_MEM: 8192 +# HOST_MEMORY_RESERVED: 49152 +# run: | +# cd tests && ( +# # Removing 'downloads' is needed to avoid this error during 'make': +# # 'pattern all: open .../elemental/tests/cypress/downloads: permission denied' +# sudo rm -rf cypress/latest/downloads +# make e2e-ui-rancher +# ) +# +# - name: Configure Rancher & Libvirt +# if: inputs.test_type == 'cli' +# run: cd tests && make e2e-configure-rancher +# delete-runner: +# if: always() && needs.create-runner.result == 'success' && inputs.destroy_runner == true +# needs: [create-runner, e2e] +# runs-on: ubuntu-latest +# steps: +# # actions/checkout MUST come before auth +# - name: Checkout +# uses: actions/checkout@v3 +# - name: Authenticate to GCP +# uses: google-github-actions/auth@v1 +# with: +# credentials_json: ${{ secrets.credentials }} +# - name: Setup gcloud +# uses: google-github-actions/setup-gcloud@v1 +# - name: Delete PAT token secret +# run: | +# gcloud --quiet secrets delete PAT_TOKEN_${{ needs.create-runner.outputs.uuid }} +# - name: Delete runner +# run: | +# gcloud --quiet compute instances delete ${{ needs.create-runner.outputs.runner }} \ +# --delete-disks all \ +# --zone ${{ inputs.zone }} diff --git a/tests/Makefile b/tests/Makefile index 3a72f0011..166328640 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -42,6 +42,8 @@ generate-readme: @./scripts/generate-readme > README.md # E2E tests +e2e-airgap-rancher: deps + ginkgo --label-filter airgap-rancher -r -v ./e2e e2e-bootstrap-node: deps ginkgo --timeout $(GINKGO_TIMEOUT)s --label-filter bootstrap -r -v ./e2e @@ -72,6 +74,8 @@ e2e-install-backup-restore: deps e2e-iso-image: deps ginkgo --label-filter iso-image -r -v ./e2e +e2e-prepare-archive: deps + ginkgo --label-filter prepare-archive -r -v ./e2e e2e-ui-rancher: deps ginkgo --label-filter ui -r -v ./e2e diff --git a/tests/assets/net-default-airgap.xml b/tests/assets/net-default-airgap.xml new file mode 100644 index 000000000..cf43b4922 --- /dev/null +++ b/tests/assets/net-default-airgap.xml @@ -0,0 +1,15 @@ + + default + + + + rancher-manager.test + + + + + + + + + diff --git a/tests/e2e/airgap_test.go b/tests/e2e/airgap_test.go new file mode 100644 index 000000000..90d7f5311 --- /dev/null +++ b/tests/e2e/airgap_test.go @@ -0,0 +1,72 @@ +/* +Copyright © 2022 - 2023 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_test + +import ( + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("E2E - Build the airgap archive", Label("prepare-archive"), func() { + It("Execute the script to build the archive", func() { + err := exec.Command("sudo", airgapBuildScript, k8sUpstreamVersion, certManagerVersion, rancherVersion).Run() + Expect(err).To(Not(HaveOccurred())) + }) +}) + +var _ = Describe("E2E - Deploy K3S/Rancher in airgap environment", Label("airgap-rancher"), func() { + It("Create the rancher-manager machine", func() { + By("Updating the default network configuration", func() { + // Don't check return code, as the default network could be already removed + for _, c := range []string{"net-destroy", "net-undefine"} { + _ = exec.Command("sudo", "virsh", c, "default").Run() + } + + // Wait a bit between virsh commands + time.Sleep(1 * time.Minute) + err := exec.Command("sudo", "virsh", "net-create", netDefaultAirgapFileName).Run() + Expect(err).To(Not(HaveOccurred())) + }) + + By("Downloading the qcow2 image from GCP storage", func() { + err := exec.Command("/opt/google-cloud-sdk/bin/gcloud", "storage", "cp", "gs://elemental-airgap-image/rancher-image.qcow2", "/home/gh-runner/rancher-image.qcow2").Run() + Expect(err).To(Not(HaveOccurred())) + }) + + By("Creating the VM", func() { + err := exec.Command("sudo", "virt-install", + "--name", "rancher-manager", + "--memory", "4096", + "--vcpus", "2", + "--disk", "path=/home/gh-runner/rancher-image.qcow2,bus=sata", + "--import", + "--os-variant", "opensuse-unknown", + "--network=default,mac=52:54:00:00:00:01", + "--noautoconsole").Run() + Expect(err).To(Not(HaveOccurred())) + }) + }) + + It("Install K3S/Rancher in the rancher-manager machine", func() { + By("Add the default route", func() { + }) + + By("Copy and execute the deploy script", func() { + }) + }) +}) diff --git a/tests/e2e/install_test.go b/tests/e2e/install_test.go index f85bbb551..1253e8cf0 100644 --- a/tests/e2e/install_test.go +++ b/tests/e2e/install_test.go @@ -175,7 +175,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { } if clusterType == "hardened" { - flags = append(flags, "--version", CertManagerVersion) + flags = append(flags, "--version", certManagerVersion) } RunHelmCmdWithRetry(flags...) diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 59d17a64c..4acc10c76 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -30,37 +30,39 @@ import ( ) const ( - appYaml = "../assets/hello-world_app.yaml" - backupYaml = "../assets/backup.yaml" - clusterYaml = "../assets/cluster.yaml" - ciTokenYaml = "../assets/local-kubeconfig-token-skel.yaml" - configPrivateCAScript = "../scripts/config-private-ca" - dumbRegistrationYaml = "../assets/dumb_machineRegistration.yaml" - emulateTPMYaml = "../assets/emulateTPM.yaml" - getOSScript = "../scripts/get-name-from-managedosversion" - httpSrv = "http://192.168.122.1:8000" - installConfigYaml = "../../install-config.yaml" - installHardenedScript = "../scripts/config-hardened" - installVMScript = "../scripts/install-vm" - localKubeconfigYaml = "../assets/local-kubeconfig-skel.yaml" - netDefaultFileName = "../assets/net-default.xml" - numberOfNodesMax = 30 - registrationYaml = "../assets/machineRegistration.yaml" - resetMachineInv = "../assets/reset_machine_inventory.yaml" - restoreYaml = "../assets/restore.yaml" - seedImageYaml = "../assets/seedImage.yaml" - selectorYaml = "../assets/selector.yaml" - upgradeSkelYaml = "../assets/upgrade_skel.yaml" - userName = "root" - userPassword = "r0s@pwd1" - vmNameRoot = "node" + airgapBuildScript = "../scripts/build-airgap" + appYaml = "../assets/hello-world_app.yaml" + backupYaml = "../assets/backup.yaml" + clusterYaml = "../assets/cluster.yaml" + ciTokenYaml = "../assets/local-kubeconfig-token-skel.yaml" + configPrivateCAScript = "../scripts/config-private-ca" + dumbRegistrationYaml = "../assets/dumb_machineRegistration.yaml" + emulateTPMYaml = "../assets/emulateTPM.yaml" + getOSScript = "../scripts/get-name-from-managedosversion" + httpSrv = "http://192.168.122.1:8000" + installConfigYaml = "../../install-config.yaml" + installHardenedScript = "../scripts/config-hardened" + installVMScript = "../scripts/install-vm" + localKubeconfigYaml = "../assets/local-kubeconfig-skel.yaml" + netDefaultFileName = "../assets/net-default.xml" + netDefaultAirgapFileName = "../assets/net-default-airgap.xml" + numberOfNodesMax = 30 + registrationYaml = "../assets/machineRegistration.yaml" + resetMachineInv = "../assets/reset_machine_inventory.yaml" + restoreYaml = "../assets/restore.yaml" + seedImageYaml = "../assets/seedImage.yaml" + selectorYaml = "../assets/selector.yaml" + upgradeSkelYaml = "../assets/upgrade_skel.yaml" + userName = "root" + userPassword = "r0s@pwd1" + vmNameRoot = "node" ) var ( arch string backupRestoreVersion string caType string - CertManagerVersion string + certManagerVersion string clusterName string clusterNS string clusterType string @@ -199,7 +201,7 @@ var _ = BeforeSuite(func() { arch = os.Getenv("ARCH") backupRestoreVersion = os.Getenv("BACKUP_RESTORE_VERSION") caType = os.Getenv("CA_TYPE") - CertManagerVersion = os.Getenv("CERT_MANAGER_VERSION") + certManagerVersion = os.Getenv("CERT_MANAGER_VERSION") clusterName = os.Getenv("CLUSTER_NAME") clusterNS = os.Getenv("CLUSTER_NS") clusterType = os.Getenv("CLUSTER_TYPE") diff --git a/tests/scripts/build-airgap b/tests/scripts/build-airgap new file mode 100755 index 000000000..5b3619c6d --- /dev/null +++ b/tests/scripts/build-airgap @@ -0,0 +1,81 @@ +#!/bin/bash + +set -e -x +K3S_UPSTREAM_VERSION=$1 +CERT_MANAGER_VERSION=$2 +RANCHER_MANAGER_VERSION=$3 +# Variable(s) and default values + +export PATH=$PATH:/usr/local/bin + +mkdir -p /opt/rancher/{k3s_$K3S_UPSTREAM_VERSION,helm} /opt/rancher/images/{cert,rancher,registry,elemental} +cd /opt/rancher/k3s_$K3S_UPSTREAM_VERSION/ + +echo - Install packages +zypper --no-refresh -n in zstd skopeo + +echo - Download k3s and rancher +curl -#OL https://github.com/k3s-io/k3s/releases/download/v$K3S_UPSTREAM_VERSION%2Bk3s1/k3s-airgap-images-amd64.tar.zst +curl -#OL https://github.com/k3s-io/k3s/releases/download/v$K3S_UPSTREAM_VERSION%2Bk3s1/k3s + +echo - Get the install script +curl -sfL https://get.k3s.io/ -o install.sh + +echo - Get Helm Charts +cd /opt/rancher/helm/ + +echo - Add repos +helm repo add jetstack https://charts.jetstack.io > /dev/null 2>&1 +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest > /dev/null 2>&1 +helm repo update > /dev/null 2>&1 + +echo - Get charts +helm pull jetstack/cert-manager --version $CERT_MANAGER_VERSION > /dev/null 2>&1 +helm pull rancher-latest/rancher --version v$RANCHER_MANAGER_VERSION > /dev/null 2>&1 + +echo - Get Images - Rancher/Elemental +cd /opt/rancher/images/ + +echo - Rancher image list +curl -#L https://github.com/rancher/rancher/releases/download/v$RANCHER_MANAGER_VERSION/rancher-images.txt -o rancher/orig_rancher-images.txt + +echo - Shorten rancher list with a sort +# fix library tags +sed -i -e '0,/busybox/s/busybox/library\/busybox/' -e 's/registry/library\/registry/g' rancher/orig_rancher-images.txt + +# remove things that are not needed and overlapped +sed -i -E '/neuvector|minio|gke|aks|eks|sriov|harvester|mirrored|longhorn|thanos|tekton|istio|multus|hyper|jenkins|windows/d' rancher/orig_rancher-images.txt + +# get latest version +for i in $(cat rancher/orig_rancher-images.txt|awk -F: '{print $1}'); do + grep -w $i rancher/orig_rancher-images.txt | sort -Vr| head -1 >> rancher/version_unsorted.txt +done + +# final sort +cat rancher/version_unsorted.txt | sort -u > rancher/rancher-images.txt + +echo - Cert-manager image list +helm template --kube-version=1.22 /opt/rancher/helm/cert-manager-$CERT_MANAGER_VERSION.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g > cert/cert-manager-images.txt + +# get images +echo - Skopeo - cert-manager +for i in $(cat cert/cert-manager-images.txt); do + skopeo copy docker://$i docker-archive:cert/$(echo $i| awk -F/ '{print $3}'|sed 's/:/_/g').tar:$(echo $i| awk -F/ '{print $3}') > /dev/null 2>&1 +done + +echo - Skopeo - Rancher - be patient... +for i in $(cat rancher/rancher-images.txt); do + skopeo copy docker://$i docker-archive:rancher/$(echo $i| awk -F/ '{print $2}'|sed 's/:/_/g').tar:$(echo $i| awk -F/ '{print $2}') > /dev/null 2>&1 +done + +curl -#L https://github.com/clemenko/rke_airgap_install/raw/main/registry.tar -o registry/registry.tar > /dev/null 2>&1 + +cd /opt/rancher/ +echo - Compress all the things +tar -I zstd -vcf /opt/airgap_rancher.zst $(ls) > /dev/null 2>&1 + +echo "------------------------------------------------------------------" +echo " to uncompress : " +echo " mkdir /opt/rancher" +echo " tar -I zstd -vxf airgap_rancher.zst -C /opt/rancher" +echo "------------------------------------------------------------------"