Add Velero e2e tests

scripts/run-from-container.sh

@@ -65,7 +65,7 @@ fi
 declare -a args
 args=("--rm")
 
-if [[ -t 1 ]] && [[ -z "${CI:-}" ]]; then
+if [[ -t 0 ]] && [[ -t 1 ]] && [[ -z "${CI:-}" ]]; then
  args+=("-it")
 fi
 

tests/Dockerfile

@@ -102,5 +102,11 @@ RUN curl -LOs "https://github.com/kubernetes-sigs/kind/releases/download/v${KIND
  install -Tm 755 kind-linux-amd64 /usr/local/bin/kind && \
  rm kind-linux-amd64
 
+ARG VELERO_VERSION="1.13.0"
+RUN curl -LOs "https://github.com/vmware-tanzu/velero/releases/download/v${VELERO_VERSION}/velero-v${VELERO_VERSION}-linux-amd64.tar.gz" && \
+ tar -zxvf "velero-v${VELERO_VERSION}-linux-amd64.tar.gz" "velero-v${VELERO_VERSION}-linux-amd64" && \
+ install -Tm 755 "velero-v${VELERO_VERSION}-linux-amd64/velero" /usr/local/bin/velero && \
+ rm -r "velero-v${VELERO_VERSION}-linux-amd64.tar.gz" "velero-v${VELERO_VERSION}-linux-amd64"
+
 ENV DOCS_PATH="/usr/local/share/docs"
 RUN git clone --depth 1 https://github.com/elastisys/compliantkubernetes.git "${DOCS_PATH}"

tests/common/bats/ctr.bash

@@ -13,7 +13,6 @@ ctr() {
 ctr.insecure() {
  if docker version >/dev/null 2>&1 && [[ ! "$(docker version)" =~ Podman ]]; then
  docker --tlsverify=false "${@}"
- echo "${ctr_insecure+"--tlsverify=false"}"
  else
  podman "${1}" --tls-verify=false "${@:2}"
  fi

tests/common/bats/harbor.bash

@@ -29,10 +29,6 @@ harbor.load_env() {
  harbor_secure="$(yq.get sc '.global.verifyTls')"
  export harbor_secure
 
- if [[ "${harbor_secure}" != "true" ]]; then
- export ctr_insecure="true"
- fi
-
  harbor_endpoint="$(yq.get sc '.harbor.subdomain + "." + .global.baseDomain')"
  export harbor_endpoint
 
@@ -57,7 +53,11 @@ harbor.setup_user_demo_image() {
  export user_demo_image="${harbor_endpoint}/${harbor_project}/user-demo:test"
 
  ctr build "${user_demo}" -t "${user_demo_image}"
- ctr push "$(ctr.insecure)" "${user_demo_image}"
+ if [[ "${harbor_secure}" != "true" ]]; then
+ ctr.insecure push "${user_demo_image}"
+ else
+ ctr push "${user_demo_image}"
+ fi
 }
 
 # Expects variables to be set with harbor.load_env
@@ -70,13 +70,21 @@ harbor.setup_project() {
  jq -r .id <<< "${output}" > "${harbor_robot_id_path}"
  jq -r .secret <<< "${output}" > "${harbor_robot_secret_path}"
 
- ctr login "$(ctr.insecure)" --username "${harbor_robot_fullname}" --password-stdin "${harbor_endpoint}" < "${harbor_robot_secret_path}"
+ if [[ "${harbor_secure}" != "true" ]]; then
+ ctr.insecure login --username "${harbor_robot_fullname}" --password-stdin "${harbor_endpoint}" < "${harbor_robot_secret_path}"
+ else
+ ctr login --username "${harbor_robot_fullname}" --password-stdin "${harbor_endpoint}" < "${harbor_robot_secret_path}"
+ fi
 }
 
 # Expects variables to be set with harbor.load_env
 harbor.teardown_project() {
  # Allow failure
- ctr logout "${harbor_endpoint}" || true
+ if [[ "${harbor_secure}" != "true" ]]; then
+ ctr.insecure logout "${harbor_endpoint}" || true
+ else
+ ctr logout "${harbor_endpoint}" || true
+ fi
 
  readarray -t robots < <(harbor.get_robots "${harbor_project}" | jq -r '.[].id')
  if [[ -n "${robots[*]}" ]]; then
@@ -250,10 +258,10 @@ harbor.create_pull_secret() {
  with_kubeconfig "${1}"
  with_namespace "${2}"
 
- kubectl -n "${NAMESPACE}" create secret ctr-registry pull-secret \
- "--ctr-server=${harbor_endpoint}" \
- "--ctr-username=${harbor_robot_fullname}" \
- "--ctr-password=$(<"${harbor_robot_secret_path}")"
+ kubectl -n "${NAMESPACE}" create secret docker-registry pull-secret \
+ "--docker-server=${harbor_endpoint}" \
+ "--docker-username=${harbor_robot_fullname}" \
+ "--docker-password=$(<"${harbor_robot_secret_path}")"
 
  kubectl -n "${NAMESPACE}" patch serviceaccount default -p '{"imagePullSecrets": [{"name": "pull-secret"}]}'
 }

tests/end-to-end/velero/backup-restore-sc.gen.bats

@@ -0,0 +1,34 @@
+#!/usr/bin/env bats
+
+# Generated from tests/end-to-end/velero/backup-restore.bats.gotmpl
+
+setup() {
+ load "../../bats.lib.bash"
+ load_assert
+ load "./common.bash"
+}
+
+@test "velero backup spec sc" {
+ run velero_backups_spec sc
+ assert_success
+ assert_output "$(cat "${BATS_TEST_DIRNAME}/resources/backup-spec-sc.yaml")"
+}
+
+@test "velero backup and restore sc" {
+ backup_name="test-backup-$(date +%s)"
+ restore_name="test-restore-$(date +%s)"
+
+ run velero_backup_create sc "${backup_name}"
+ assert_success
+
+ run velero_backup_get_phase sc "${backup_name}"
+ assert_success
+ assert_output Completed
+
+ run velero_restore_create sc "${restore_name}" "${backup_name}"
+ assert_success
+
+ run velero_restore_get_phase sc "${restore_name}"
+ assert_success
+ assert_output Completed
+}

tests/end-to-end/velero/backup-restore-wc.gen.bats

@@ -0,0 +1,34 @@
+#!/usr/bin/env bats
+
+# Generated from tests/end-to-end/velero/backup-restore.bats.gotmpl
+
+setup() {
+ load "../../bats.lib.bash"
+ load_assert
+ load "./common.bash"
+}
+
+@test "velero backup spec wc" {
+ run velero_backups_spec wc
+ assert_success
+ assert_output "$(cat "${BATS_TEST_DIRNAME}/resources/backup-spec-wc.yaml")"
+}
+
+@test "velero backup and restore wc" {
+ backup_name="test-backup-$(date +%s)"
+ restore_name="test-restore-$(date +%s)"
+
+ run velero_backup_create wc "${backup_name}"
+ assert_success
+
+ run velero_backup_get_phase wc "${backup_name}"
+ assert_success
+ assert_output Completed
+
+ run velero_restore_create wc "${restore_name}" "${backup_name}"
+ assert_success
+
+ run velero_restore_get_phase wc "${restore_name}"
+ assert_success
+ assert_output Completed
+}

tests/end-to-end/velero/backup-restore.bats.gotmpl

@@ -0,0 +1,45 @@
+#!/usr/bin/env bats
+
+{{- define "template" -}}
+#!/usr/bin/env bats
+
+# Generated from {{ tmpl.Path }}
+
+setup() {
+ load "../../bats.lib.bash"
+ load_assert
+ load "./common.bash"
+}
+
+@test "velero backup spec {{ .cluster }}" {
+ run velero_backups_spec {{ .cluster }}
+ assert_success
+ assert_output "$(cat "${BATS_TEST_DIRNAME}/resources/backup-spec-{{ .cluster }}.yaml")"
+}
+
+@test "velero backup and restore {{ .cluster }}" {
+ backup_name="test-backup-$(date +%s)"
+ restore_name="test-restore-$(date +%s)"
+
+ run velero_backup_create {{ .cluster }} "${backup_name}"
+ assert_success
+
+ run velero_backup_get_phase {{ .cluster }} "${backup_name}"
+ assert_success
+ assert_output Completed
+
+ run velero_restore_create {{ .cluster }} "${restore_name}" "${backup_name}"
+ assert_success
+
+ run velero_restore_get_phase {{ .cluster }} "${restore_name}"
+ assert_success
+ assert_output Completed
+}
+{{ end }}
+
+# These tests are generated into these files:
+{{- range $cluster := coll.Slice "sc" "wc" }}
+{{- $file := path.Join (tmpl.Path | path.Dir) (printf "./backup-restore-%s.gen.bats" $cluster) }}
+# - {{ $file }}
+{{- coll.Dict "cluster" $cluster | tmpl.Exec "template" | file.Write $file }}
+{{- end }}

tests/end-to-end/velero/backup-restore.gen.bats

@@ -0,0 +1,5 @@
+#!/usr/bin/env bats
+
+# These tests are generated into these files:
+# - tests/end-to-end/velero/backup-restore-sc.gen.bats
+# - tests/end-to-end/velero/backup-restore-wc.gen.bats

tests/end-to-end/velero/common.bash

@@ -0,0 +1,19 @@
+velero_backups_spec() {
+ ck8s ops velero "${1}" backup create --from-schedule velero-daily-backup -o yaml 2>/dev/null | yq4 .spec
+}
+
+velero_backup_create() {
+ ck8s ops velero "${1}" backup create "${2}" --from-schedule velero-daily-backup --wait
+}
+
+velero_backup_get_phase() {
+ ck8s ops velero "${1}" backup get "${2}" -o json 2>/dev/null | jq -r .status.phase
+}
+
+velero_restore_create() {
+ ck8s ops velero "${1}" restore create "${2}" --from-backup "${3}" --wait
+}
+
+velero_restore_get_phase() {
+ ck8s ops velero "${1}" restore get "${2}" -o json 2>/dev/null | jq -r .status.phase
+}

tests/end-to-end/velero/resources/backup-spec-sc.yaml

@@ -0,0 +1,21 @@
+csiSnapshotTimeout: 0s
+excludedResources:
+ - clustercompliancereports.aquasecurity.github.io
+ - clusterconfigauditreports.aquasecurity.github.io
+ - clusterinfraassessmentreports.aquasecurity.github.io
+ - clusterrbacassessmentreports.aquasecurity.github.io
+ - clustersbomreports.aquasecurity.github.io
+ - configauditreports.aquasecurity.github.io
+ - exposedsecretreports.aquasecurity.github.io
+ - infraassessmentreports.aquasecurity.github.io
+ - rbacassessmentreports.aquasecurity.github.io
+ - sbomreports.aquasecurity.github.io
+ - vulnerabilityreports.aquasecurity.github.io
+hooks: {}
+itemOperationTimeout: 0s
+labelSelector:
+ matchLabels:
+ velero: backup
+metadata: {}
+storageLocation: default
+ttl: 720h0m0s

tests/end-to-end/velero/resources/backup-spec-wc.yaml

@@ -0,0 +1,40 @@
+csiSnapshotTimeout: 0s
+excludedNamespaces:
+ - cert-manager
+ - falco
+ - fluentd
+ - hnc-system
+ - ingress-nginx
+ - kube-node-lease
+ - kube-public
+ - kube-system
+ - kured
+ - monitoring
+ - rook-ceph
+ - velero
+ - gatekeeper-system
+ - jaeger-system
+ - postgres-system
+ - rabbitmq-system
+ - redis-system
+excludedResources:
+ - clustercompliancereports.aquasecurity.github.io
+ - clusterconfigauditreports.aquasecurity.github.io
+ - clusterinfraassessmentreports.aquasecurity.github.io
+ - clusterrbacassessmentreports.aquasecurity.github.io
+ - clustersbomreports.aquasecurity.github.io
+ - configauditreports.aquasecurity.github.io
+ - exposedsecretreports.aquasecurity.github.io
+ - infraassessmentreports.aquasecurity.github.io
+ - rbacassessmentreports.aquasecurity.github.io
+ - sbomreports.aquasecurity.github.io
+ - vulnerabilityreports.aquasecurity.github.io
+hooks: {}
+itemOperationTimeout: 0s
+labelSelector:
+ matchExpressions:
+ - key: compliantkubernetes.io/nobackup
+ operator: DoesNotExist
+metadata: {}
+storageLocation: default
+ttl: 720h0m0s

tests/end-to-end/velero/resources/test-application.yaml

@@ -0,0 +1,48 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: velero-test
+ namespace: velero-test
+spec:
+ podSelector: {}
+ policyTypes:
+ - Ingress
+ - Egress
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: velero-test
+ namespace: velero-test
+spec:
+ containers:
+ - image: ${image}
+ args:
+ - sleep
+ - "3600"
+ name: velero-test
+ resources:
+ limits:
+ cpu: 100m
+ memory: 128Mi
+ securityContext:
+ runAsUser: 1000
+ volumeMounts:
+ - name: velero-test
+ mountPath: /test
+ volumes:
+ - name: velero-test
+ persistentVolumeClaim:
+ claimName: velero-test
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: velero-test
+ namespace: velero-test
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi

tests/end-to-end/velero/resources/test-namespace.yaml

@@ -0,0 +1,5 @@
+apiVersion: hnc.x-k8s.io/v1alpha2
+kind: SubnamespaceAnchor
+metadata:
+ name: velero-test
+ namespace: production