diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index a83896faba..337517c172 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -10,6 +10,7 @@ on: env: DOCKER_USER: testuser DOCKER_PASSWORD: testpassword + NAMESPACE: nydus-system jobs: e2e_tests_k8s: @@ -25,115 +26,22 @@ jobs: with: go-version-file: 'go.mod' cache-dependency-path: "go.sum" - - name: Setup Kind - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.16.0 - config: tests/e2e/k8s/kind.yaml - - name: Build nydus snapshotter dev image - run: | - make - cp bin/containerd-nydus-grpc ./ - cp bin/nydus-overlayfs ./ - cp -r misc/snapshotter/* ./ - ls -tl ./ - NYDUS_VER=v$(curl -fsSL --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') - docker build --build-arg NYDUS_VER=${NYDUS_VER} -t local-dev:e2e . - - ## load local test image into kind node - kind load docker-image local-dev:e2e - - name: Setup registry - run: | - mkdir auth - docker run \ - --entrypoint htpasswd \ - httpd:2 -Bbn ${{ env.DOCKER_USER }} ${{ env.DOCKER_PASSWORD }} > auth/htpasswd - - docker run -d \ - -p 5000:5000 \ - --restart=always \ - --name registry \ - -v "$(pwd)"/auth:/auth \ - -e "REGISTRY_AUTH=htpasswd" \ - -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ - -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ - registry:2 - - name: Login to GitHub Container Registry - run: | - registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) - ## sudo cat can't modify daemon.json - sudo chmod a+w /etc/docker/daemon.json - cat << EOF > /etc/docker/daemon.json - { - "exec-opts": ["native.cgroupdriver=cgroupfs"], - "cgroup-parent": "/actions_job", - "insecure-registries" : [ "${registry_ip}:5000" ] - } - EOF - sudo systemctl restart docker - docker login --username=${{ env.DOCKER_USER }} --password=${{ env.DOCKER_PASSWORD }} $registry_ip:5000 - - name: Setup nydus snapshotter + - name: Test run: | - kubectl create -f tests/e2e/k8s/snapshotter-${{ inputs.auth-type }}.yaml - export ns=nydus-system - p=`kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name` - echo "snapshotter pod name ${p}" - kubectl -n $ns wait po $p --for=condition=ready --timeout=2m - - # change snapshotter to nydus after nydus snapshotter started - registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) - docker cp kind-control-plane:/etc/containerd/config.toml containerd.config.toml.bak - cat << EOF >> containerd.config.toml.bak - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${registry_ip}:5000"] - endpoint = ["http://${registry_ip}:5000"] - EOF - - docker cp containerd.config.toml.bak kind-control-plane:/etc/containerd/config.toml.bak - docker exec kind-control-plane sh -c "cat /etc/containerd/config.toml.bak > /etc/containerd/config.toml" - docker exec kind-control-plane systemctl restart containerd - - name: Install Nydus binaries and convert nydus image - run: | - NYDUS_VER=v$(curl -fsSL --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') - wget -q https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz - tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz - sudo cp nydus-static/nydusify nydus-static/nydus-image /usr/local/bin/ - - registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) - - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source busybox:latest \ - --target ${registry_ip}:5000/busybox:nydus-v6-latest \ - --fs-version 6 - - name: Run E2E test - run: | - registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) - kubectl create --namespace nydus-system secret generic regcred \ - --from-file=.dockerconfigjson=$HOME/.docker/config.json \ - --type=kubernetes.io/dockerconfigjson - - sed -e "s|REGISTRY_IP|${registry_ip}|" tests/e2e/k8s/test-pod.yaml.tpl > tests/e2e/k8s/test-pod.yaml - - if [[ "${{ inputs.auth-type }}" == "cri" ]]; then - docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet' - docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet' - fi - - kubectl apply -f tests/e2e/k8s/test-pod.yaml - kubectl wait po test-pod -n nydus-system --for=condition=ready --timeout=1m - kubectl delete -f tests/e2e/k8s/test-pod.yaml + AUTH_TYPE='${{ inputs.auth-type }}' + ./tests/helpers/kind.sh - name: Dump logs if: failure() continue-on-error: true run: | log_dir="/tmp/nydus-log" mkdir -p $log_dir - export ns=nydus-system - for p in `kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do - kubectl -n $ns get pod $p -o yaml >> $log_dir/nydus-pods.conf - kubectl -n $ns describe pod $p >> $log_dir/nydus-pods.conf - kubectl -n $ns logs $p -c nydus-snapshotter >> $log_dir/nydus-snapshotter.log || echo "failed to get snapshotter log" + for p in `kubectl --namespace "$NAMESPACE" get pods --no-headers -o custom-columns=NAME:metadata.name`; do + kubectl --namespace "$NAMESPACE" get pod $p -o yaml >> $log_dir/nydus-pods.conf + kubectl --namespace "$NAMESPACE" describe pod $p >> $log_dir/nydus-pods.conf + kubectl --namespace "$NAMESPACE" logs $p -c nydus-snapshotter >> $log_dir/nydus-snapshotter.log || echo "failed to get snapshotter log" done - kubectl -n $ns get secrets -o yaml >> $log_dir/nydus-secrets.log + kubectl --namespace "$NAMESPACE" get secrets -o yaml >> $log_dir/nydus-secrets.log docker exec kind-control-plane cat /etc/containerd/config.toml >> $log_dir/containerd-config.toml docker exec kind-control-plane containerd config dump >> $log_dir/containerd-config-dump.toml diff --git a/Makefile b/Makefile index f3d9b53172..a9e2b93d79 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,11 @@ build: GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs +.PHONY: static +static: + CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc + CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs + debug: GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(DEBUG_LDFLAGS)" -gcflags "-N -l" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(DEBUG_LDFLAGS)" -gcflags "-N -l" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs diff --git a/docs/tarfs.md b/docs/tarfs.md index da7c0a568f..726305700e 100644 --- a/docs/tarfs.md +++ b/docs/tarfs.md @@ -24,21 +24,21 @@ $ nerdctl run --snapshotter nydus --rm nginx # Show mounted rootfs a container $ mount -/dev/loop17 on /var/lib/containerd-nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) +/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) # Show loop devices used to mount layers and bootstrap for a container image $ losetup NAME SIZELIMIT OFFSET AUTOCLEAR RO BACK-FILE DIO LOG-SEC -/dev/loop11 0 0 0 0 /var/lib/containerd-nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0 512 -/dev/loop12 0 0 0 0 /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0 512 -/dev/loop13 0 0 0 0 /var/lib/containerd-nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0 512 -/dev/loop14 0 0 0 0 /var/lib/containerd-nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0 512 -/dev/loop15 0 0 0 0 /var/lib/containerd-nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0 512 -/dev/loop16 0 0 0 0 /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0 512 -/dev/loop17 0 0 0 0 /var/lib/containerd-nydus/snapshots/7/fs/image/image.boot 0 512 +/dev/loop11 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0 512 +/dev/loop12 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0 512 +/dev/loop13 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0 512 +/dev/loop14 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0 512 +/dev/loop15 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0 512 +/dev/loop16 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0 512 +/dev/loop17 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/fs/image/image.boot 0 512 # Files without suffix are tar files, files with suffix `layer.disk` are raw disk image for container image layers -$ ls -l /var/lib/containerd-nydus/cache/ +$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/ total 376800 -rw-r--r-- 1 root root 3584 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -rw-r--r-- 1 root root 527872 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk @@ -54,15 +54,15 @@ total 376800 -rw-r--r-- 1 root root 529408 Aug 30 23:18 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665.layer.disk -rw-r--r-- 1 root root 112968704 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa -rw-r--r-- 1 root root 113492992 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa.layer.disk -$ file /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -/var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive +$ file /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 +/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive # Mount the raw disk image for a container image layer -$ losetup /dev/loop100 /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk +$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk $ mount -t erofs /dev/loop100 ./mnt/ $ mount tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1544836k,nr_inodes=386209,mode=700,inode64) -/dev/loop17 on /var/lib/containerd-nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) +/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) /dev/loop100 on /root/ws/nydus-snapshotter.git/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) ``` @@ -81,7 +81,7 @@ $ containerd-nydus-grpc --config /etc/nydus/config.toml & $ nerdctl run --snapshotter nydus --rm nginx # Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image -$ ls -l /var/lib/containerd-nydus/cache/ +$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/ total 376320 -rw-r--r-- 1 root root 3584 Aug 30 23:35 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -rw-r--r-- 1 root root 77814784 Aug 30 23:35 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5 @@ -109,7 +109,7 @@ $ containerd-nydus-grpc --config /etc/nydus/config.toml & $ nerdctl run --snapshotter nydus --rm nginx # Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image -$ ls -l /var/lib/containerd-nydus/cache/ +$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/ total 388296 -rw-r--r-- 1 root root 3584 Aug 30 23:45 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -rw-r--r-- 1 root root 77814784 Aug 30 23:46 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5 @@ -120,7 +120,7 @@ total 388296 -rw-r--r-- 1 root root 5120 Aug 30 23:45 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 -rw-r--r-- 1 root root 112968704 Aug 30 23:46 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa -$ losetup /dev/loop100 /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk +$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk $ veritysetup open --no-superblock --format=1 -s "" --hash=sha256 --data-block-size=512 --hash-block-size=4096 --data-blocks 379918 --hash-offset 194519040 /dev/loop100 image1 /dev/loop100 8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd $ lsblk loop100 7:100 0 197.2M 0 loop @@ -136,11 +136,11 @@ $ veritysetup status dm-0 hash name: sha256 salt: - data device: /dev/loop100 - data loop: /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk + data loop: /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk size: 379918 sectors mode: readonly hash device: /dev/loop100 - hash loop: /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk + hash loop: /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk hash offset: 379920 sectors root hash: 8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd diff --git a/integration/Dockerfile b/integration/Dockerfile index b1860e4a69..3edae1401d 100644 --- a/integration/Dockerfile +++ b/integration/Dockerfile @@ -1,11 +1,11 @@ - ARG CONTAINERD_VER=1.6.15 ARG CONTAINERD_PROJECT=/containerd ARG RUNC_VERSION=1.1.4 ARG NYDUS_SNAPSHOTTER_PROJECT=/nydus-snapshotter ARG DOWNLOADS_MIRROR="https://github.com" ARG NYDUS_VER=2.2.4 -ARG NERDCTL_VER=1.0.0 +ARG NERDCTL_VER=1.7.6 +ARG DELVE_VER=1.23.0 FROM golang:1.21.12-bookworm AS golang-base @@ -16,6 +16,7 @@ ARG NYDUS_SNAPSHOTTER_PROJECT ARG DOWNLOADS_MIRROR ARG NYDUS_VER ARG NERDCTL_VER +ARG DELVE_VER # RUN echo '\ # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye main contrib non-free\n\ @@ -25,7 +26,7 @@ ARG NERDCTL_VER RUN apt-get update -qq && apt-get install -qq libbtrfs-dev libseccomp-dev sudo psmisc jq lsof net-tools -RUN go install github.com/go-delve/delve/cmd/dlv@latest +RUN go install github.com/go-delve/delve/cmd/dlv@v"$DELVE_VER" # Install containerd RUN wget -q ${DOWNLOADS_MIRROR}/containerd/containerd/releases/download/v${CONTAINERD_VER}/containerd-${CONTAINERD_VER}-linux-amd64.tar.gz && \ diff --git a/misc/example/containerd-config-v1.toml b/misc/example/containerd-config-v1.toml deleted file mode 100644 index 67fb9ed523..0000000000 --- a/misc/example/containerd-config-v1.toml +++ /dev/null @@ -1,30 +0,0 @@ -root = "/var/lib/containerd" -state = "/run/containerd" -oom_score = 0 - -[grpc] - address = "/run/containerd/containerd.sock" - -[debug] - level = "debug" - -[plugins] - [plugins.cgroups] - no_prometheus = false - [plugins.cri] - [plugins.cri.containerd] - snapshotter = "nydus" - disable_snapshot_annotations = false - [plugins.cri.containerd.runtimes] - [plugins.cri.containerd.runtimes.kata] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - [plugins.cri.cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - -[proxy_plugins] - [proxy_plugins.nydus] - type = "snapshot" - address = "/run/containerd-nydus/containerd-nydus-grpc.sock" - diff --git a/misc/snapshotter/Dockerfile b/misc/snapshotter/Dockerfile index c31124270e..91a43c34e7 100644 --- a/misc/snapshotter/Dockerfile +++ b/misc/snapshotter/Dockerfile @@ -34,7 +34,7 @@ VOLUME /var/lib/containerd/io.containerd.snapshotter.v1.nydus /run/containerd-ny COPY --from=sourcer /.nydus_version /.nydus_version COPY --from=kubectl-sourcer /usr/bin/kubectl /usr/bin/kubectl -RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd-nydus/cache /tmp/blobs/ +RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache /tmp/blobs/ COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/ COPY --chmod=755 containerd-nydus-grpc nydus-overlayfs ${BINARY_DESTINATION}/ COPY --chmod=755 snapshotter.sh ${SCRIPT_DESTINATION}/snapshotter.sh diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 3b6a61946b..416e1e7854 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -1033,7 +1033,7 @@ func (o *snapshotter) cleanupDirectories(ctx context.Context) ([]string, error) } func (o *snapshotter) cleanupSnapshotDirectory(ctx context.Context, dir string) error { - // For example: cleanupSnapshotDirectory /var/lib/containerd-nydus/snapshots/34" dir=/var/lib/containerd-nydus/snapshots/34 + // For example: cleanupSnapshotDirectory /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/34" dir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/34 snapshotID := filepath.Base(dir) if err := o.fs.Umount(ctx, snapshotID); err != nil && !os.IsNotExist(err) { diff --git a/tests/e2e/k8s/snapshotter-cri.yaml b/tests/e2e/k8s/snapshotter-cri.yaml index 99e87437c3..29b5d67922 100644 --- a/tests/e2e/k8s/snapshotter-cri.yaml +++ b/tests/e2e/k8s/snapshotter-cri.yaml @@ -168,7 +168,7 @@ data: ENABLE_SYSTEMD_SERVICE: "true" config.toml: |- version = 1 - root = "/var/lib/containerd-nydus" + root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus" address = "/run/containerd-nydus/containerd-nydus-grpc.sock" daemon_mode = "multiple" enable_system_controller = true diff --git a/tests/e2e/k8s/snapshotter-kubeconf.yaml b/tests/e2e/k8s/snapshotter-kubeconf.yaml index 3868a9c70f..b266ef1153 100644 --- a/tests/e2e/k8s/snapshotter-kubeconf.yaml +++ b/tests/e2e/k8s/snapshotter-kubeconf.yaml @@ -177,7 +177,7 @@ data: ENABLE_SYSTEMD_SERVICE: "false" config.toml: |- version = 1 - root = "/var/lib/containerd-nydus" + root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus" address = "/run/containerd-nydus/containerd-nydus-grpc.sock" daemon_mode = "multiple" enable_system_controller = true diff --git a/tests/e2e/k8s/test-pod.yaml.tpl b/tests/e2e/k8s/test-pod.yaml.tpl index cfc12be11f..8807ea822b 100644 --- a/tests/e2e/k8s/test-pod.yaml.tpl +++ b/tests/e2e/k8s/test-pod.yaml.tpl @@ -6,7 +6,7 @@ metadata: spec: containers: - name: busybox - image: REGISTRY_IP:5000/busybox:nydus-v6-latest + image: REGISTRY_URL/busybox:nydus-v6-latest imagePullPolicy: Always command: ["sh", "-c"] args: diff --git a/tests/helpers/helpers.sh b/tests/helpers/helpers.sh new file mode 100644 index 0000000000..efca0b2762 --- /dev/null +++ b/tests/helpers/helpers.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o errtrace -o functrace -o nounset -o pipefail + +## Helpers specific to nydus + +_rootful= + +configure::rootful(){ + log::debug "Configuring rootful to: ${1:+true}" + _rootful="${1:+true}" +} + +configure::dockerd(){ + local config="$1" + + log::debug "Reconfiguring docker and restarting" + log::debug "$config" + sudo touch /etc/docker/daemon.json + printf "%s" "$config" | sudo tee /etc/docker/daemon.json >/dev/null + sudo systemctl restart docker +} + +exec::docker(){ + local args=() + [ ! "$_rootful" ] || args=(sudo) + args+=(docker) + + log::debug "${args[*]} $*" + "${args[@]}" "$@" +} + +exec::kind(){ + local args=() + [ ! "$_rootful" ] || args=(sudo) + args+=(kind) + + log::debug "${args[*]} $*" + "${args[@]}" "$@" +} + +exec::kubectl(){ + local args=() + [ ! "$_rootful" ] || args=(sudo) + args+=(kubectl) + + log::debug "${args[*]} $*" + "${args[@]}" "$@" +} + +exec::nydusify(){ + local args=() + [ ! "$_rootful" ] || args=(sudo) + args+=(nydusify) + + log::debug "${args[*]} $*" + "${args[@]}" "$@" +} + +docker::configpath(){ + [ "$_rootful" ] && printf "/root/.docker/config.json" || printf "%s/.docker/config.json" "$HOME" +} + +docker::login(){ + local user="$1" + local password="$2" + local registry_url="$3" + + local args=() + [ ! "$_rootful" ] || args=(sudo) + args+=(docker login --password-stdin) + + log::debug "${args[*]} --username=$user $registry_url" + "${args[@]}" --username="$user" "$registry_url" <<<"$password" >/dev/null +} + +# Installation helpers +install::kind(){ + local version="$1" + local temp + temp="$(fs::mktemp "install")" + + http::get "$temp"/kind "https://kind.sigs.k8s.io/dl/$version/kind-linux-${GOARCH:-amd64}" + host::install "$temp"/kind +} + +install::kubectl(){ + local version="${1:-}" + [ "$version" ] || version="$(http::get /dev/stdout https://dl.k8s.io/release/stable.txt)" + local temp + temp="$(fs::mktemp "install")" + + http::get "$temp"/kubectl "https://storage.googleapis.com/kubernetes-release/release/$version/bin/linux/${GOARCH:-amd64}/kubectl" + host::install "$temp"/kubectl +} + +install::nydus(){ + local version="$1" + local temp + temp="$(fs::mktemp "install")" + + http::get "$temp"/nydus-static.tgz "https://github.com/dragonflyoss/nydus/releases/download/$version/nydus-static-$version-linux-${GOARCH:-amd64}.tgz" + tar::expand "$temp" "$temp"/nydus-static.tgz + host::install "$temp"/nydus-static/nydus-image + host::install "$temp"/nydus-static/nydusify + host::install "$temp"/nydus-static/nydusd + host::install "$temp"/nydus-static/nydusctl +} + +install::nerdctl(){ + local version="$1" + local temp + temp="$(fs::mktemp "install")" + + http::get "$temp"/nerdctl.tar.gz "https://github.com/containerd/nerdctl/releases/download/v$version/nerdctl-$version-linux-${GOARCH:-amd64}.tar.gz" + tar::expand "$temp" "$temp"/nerdctl.tar.gz + host::install "$temp"/nerdctl +} + +start::registry(){ + local user="$1" + local password="$2" + local tempdir + tempdir="$(fs::mktemp registry)" + local port=5000 + local ip + + exec::docker rm -f registry-auth-"$port" >/dev/null 2>&1 || true + + exec::docker run \ + --rm \ + --entrypoint htpasswd \ + httpd:2 -Bbn "$user" "$password" > "$tempdir"/htpasswd + + exec::docker run -d \ + -p "$port":5000 \ + --restart=always \ + --name registry-auth-"$port" \ + -v "$tempdir":/auth \ + -e "REGISTRY_AUTH=htpasswd" \ + -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + registry:2 >/dev/null + + ip="$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)" + printf "%s:%s" "$ip" "$port" +} diff --git a/tests/helpers/kind.sh b/tests/helpers/kind.sh new file mode 100755 index 0000000000..937ee7ecd8 --- /dev/null +++ b/tests/helpers/kind.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -o errexit -o errtrace -o functrace -o nounset -o pipefail + +AUTH_TYPE="${AUTH_TYPE:-kubeconf}" +[ "$(uname -m)" == "aarch64" ] && GOARCH=arm64 || GOARCH=amd64 +ROOTFUL="${ROOTFUL:-}" + +root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)" +readonly root +# shellcheck source=/dev/null +. "$root/lib.sh" +# shellcheck source=/dev/null +. "$root/helpers.sh" + + +KIND_VERSION=v0.23.0 +NYDUS_VERSION=v2.2.5 +DOCKER_USER=testuser +DOCKER_PASSWORD=testpassword +NAMESPACE=nydus-system + +log::info "Configuring rootful and github" +configure::rootful "${ROOTFUL:-}" +github::settoken '${{ secrets.GITHUB_TOKEN }}' + +NYDUS_VERSION="$(github::releases::latest dragonflyoss/nydus | jq -rc 'select(.tag_name != null) | .tag_name' || printf "%s" "$NYDUS_VERSION")" +log::info "Get latest nydus version $NYDUS_VERSION" + +# Build +log::info "Building... make" +# Binaries are going to be copied over inside the kind container that likely has a different glibc we do not control. +# So, build static +if [ "$LOG_LEVEL" == debug ]; then + make -d static +else + make -s static +fi +cp bin/containerd-nydus-grpc ./ +cp bin/nydus-overlayfs ./ +cp -r misc/snapshotter/* ./ + +log::info "Building... docker image" +pwd +exec::docker build --build-arg NYDUS_VER="$NYDUS_VERSION" -t local-dev:e2e . + +# Start local registry, and configure docker +log::info "Starting registry" +registry_url="$(start::registry "$DOCKER_USER" "$DOCKER_PASSWORD")" +# Configure template +sed -e "s|REGISTRY_URL|$registry_url|" tests/e2e/k8s/test-pod.yaml.tpl > tests/e2e/k8s/test-pod.yaml +configure::dockerd '{ + "exec-opts": ["native.cgroupdriver=cgroupfs"], + "cgroup-parent": "/actions_job", + "insecure-registries" : [ "'"$registry_url"'" ] +}' +http::healthcheck "$registry_url"/v2/ 5 1 "$DOCKER_USER" "$DOCKER_PASSWORD" +log::info "Login" +docker::login "$DOCKER_USER" "$DOCKER_PASSWORD" "$registry_url" + +# Install dependencies +log::info "Installing host dependencies" +install::kind "$KIND_VERSION" +install::kubectl +install::nydus "$NYDUS_VERSION" + +# Convert a nydus image and push it +log::info "Converting test image to nydus and push" +exec::nydusify convert \ + --source busybox:latest \ + --target $registry_url/busybox:nydus-v6-latest \ + --fs-version 6 + +# Create fresh cluster +log::info "Creating new cluster" +exec::kind delete cluster 2>/dev/null || true +exec::kind create cluster +exec::kind load docker-image local-dev:e2e + +# Deploy nydus +log::info "Deploying nydus" +exec::kubectl create -f tests/e2e/k8s/snapshotter-"$AUTH_TYPE".yaml +pod="$(exec::kubectl --namespace "$NAMESPACE" get pods --no-headers -o custom-columns=NAME:metadata.name)" +exec::kubectl --namespace "$NAMESPACE" wait po "$pod" --for=condition=ready --timeout=1m + +# Reconfigure and restart kind containerd +log::info "Restarting containerd" +echo '[plugins."io.containerd.grpc.v1.cri".registry.mirrors."'"$registry_url"'"] + endpoint = ["http://'"$registry_url"'"]' | + exec::docker exec -i kind-control-plane sh -c 'cat /dev/stdin >> /etc/containerd/config.toml' + +exec::docker exec kind-control-plane systemctl restart containerd + +# Actual testing +exec::kubectl delete --namespace "$NAMESPACE" secret generic regcred 2>/dev/null || true +exec::kubectl create --namespace "$NAMESPACE" secret generic regcred \ + --from-file=.dockerconfigjson="$(docker::configpath)" \ + --type=kubernetes.io/dockerconfigjson + +if [ "$AUTH_TYPE" == "cri" ]; then + exec::docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet' + exec::docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet' +fi + +exec::kubectl apply -f tests/e2e/k8s/test-pod.yaml +exec::kubectl wait po test-pod --namespace nydus-system --for=condition=ready --timeout=1m || { + exec::kubectl --namespace nydus-system get pods + exec::kubectl --namespace nydus-system get events + exec::kubectl --namespace nydus-system logs nydus-snapshotter + exec::kubectl --namespace nydus-system logs test-pod + exit 1 +} +exec::kubectl delete -f tests/e2e/k8s/test-pod.yaml + diff --git a/tests/helpers/lib.sh b/tests/helpers/lib.sh new file mode 100755 index 0000000000..28a48ae505 --- /dev/null +++ b/tests/helpers/lib.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2034 +set -o errexit -o errtrace -o functrace -o nounset -o pipefail + +## This is a library of generic helpers that can be used accross many different projects + +# Simple logger +readonly LOG_LEVEL_DEBUG=0 +readonly LOG_LEVEL_INFO=1 +readonly LOG_LEVEL_WARNING=2 +readonly LOG_LEVEL_ERROR=3 + +readonly LOG_COLOR_BLACK=0 +readonly LOG_COLOR_RED=1 +readonly LOG_COLOR_GREEN=2 +readonly LOG_COLOR_YELLOW=3 +readonly LOG_COLOR_BLUE=4 +readonly LOG_COLOR_MAGENTA=5 +readonly LOG_COLOR_CYAN=6 +readonly LOG_COLOR_WHITE=7 +readonly LOG_COLOR_DEFAULT=9 + +readonly LOG_STYLE_DEBUG=( setaf "$LOG_COLOR_WHITE" ) +readonly LOG_STYLE_INFO=( setaf "$LOG_COLOR_GREEN" ) +readonly LOG_STYLE_WARNING=( setaf "$LOG_COLOR_YELLOW" ) +readonly LOG_STYLE_ERROR=( setaf "$LOG_COLOR_RED" ) + +_log::log(){ + local level + local style + local numeric_level + local message="$2" + + level="$(printf "%s" "$1" | tr '[:lower:]' '[:upper:]')" + numeric_level="$(printf "LOG_LEVEL_%s" "$level")" + style="LOG_STYLE_${level}[@]" + + [ "${!numeric_level}" -ge "$LOG_LEVEL" ] || return 0 + + [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput "${!style}" 2>/dev/null || true + >&2 printf "[%s] %s: %s\n" "$(date 2>/dev/null || true)" "$(printf "%s" "$level" | tr '[:lower:]' '[:upper:]')" "$message" + [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput op 2>/dev/null || true +} + +log::init(){ + local _ll + # Default log to warning if unspecified + _ll="$(printf "LOG_LEVEL_%s" "${LOG_LEVEL:-warning}" | tr '[:lower:]' '[:upper:]')" + # Default to 3 (warning) if unrecognized + LOG_LEVEL="${!_ll:-3}" +} + +log::debug(){ + _log::log debug "$@" +} + +log::info(){ + _log::log info "$@" +} + +log::warning(){ + _log::log warning "$@" +} + +log::error(){ + _log::log error "$@" +} + +# Helpers +host::require(){ + local binary="$1" + + log::debug "Checking presence of $binary" + command -v "$binary" >/dev/null || { + log::error "You need $binary for this script to work, and it cannot be found in your path" + return 1 + } +} + +host::install(){ + local binary + + for binary in "$@"; do + log::debug "sudo install -D -m 755 $binary /usr/local/bin/$(basename "$binary")" + sudo install -D -m 755 "$binary" /usr/local/bin/"$(basename "$binary")" + done +} + +fs::mktemp(){ + local prefix="${1:-temporary}" + + mktemp -dq "${TMPDIR:-/tmp}/$prefix.XXXXXX" 2>/dev/null || mktemp -dq || { + log::error "Failed to create temporary directory" + return 1 + } +} + +tar::expand(){ + local dir="$1" + local arc="$2" + + log::debug "tar -C $dir -xzf $arc" + tar -C "$dir" -xzf "$arc" +} + +_http::get(){ + local url="$1" + local output="$2" + local retry="$3" + local delay="$4" + local user="${5:-}" + local password="${6:-}" + shift + shift + shift + shift + shift + shift + + local header + local command=(curl -fsSL --retry "$retry" --retry-delay "$delay" -o "$output") + # Add a basic auth user if necessary + [ "$user" == "" ] || command+=(--user "$user:$password") + # Force tls v1.2 and no redirect to http if url scheme is https + [ "${url:0:5}" != "https" ] || command+=(--proto '=https' --tlsv1.2) + # Stuff in any additional arguments as headers + for header in "$@"; do + command+=(-H "$header") + done + # Debug + log::debug "${command[*]} $url" + # Exec + "${command[@]}" "$url" || { + log::error "Failed to connect to $url with $retry retries every $delay seconds" + return 1 + } +} + +http::get(){ + local output="$1" + local url="$2" + shift + shift + + _http::get "$url" "$output" "2" "1" "" "" "$@" +} + +http::healthcheck(){ + local url="$1" + local retry="${2:-5}" + local delay="${3:-1}" + local user="${4:-}" + local password="${5:-}" + shift + shift + shift + shift + shift + _http::get "$url" /dev/null "$retry" "$delay" "$user" "$password" "$@" +} + +http::checksum(){ + local urls=("$@") + local url + + local temp + temp="$(fs::mktemp "http-checksum")" + + for url in "${urls[@]}"; do + http::get -o "$temp/${url##*/}" "$url" + done + + cd "$temp" + shasum -a 256 ./* + cd - >/dev/null || true +} + +# Github API helpers +# Set GITHUB_TOKEN to use authenticated requests to workaround limitations + +github::settoken(){ + local token="$1" + # If passed token is a github action pattern replace, and we are NOT on github, ignore it + [ "${token:0:3}" == '${{' ] || GITHUB_TOKEN="$token" +} + +github::request(){ + local endpoint="$1" + local args=( + "Accept: application/vnd.github+json" + "X-GitHub-Api-Version: 2022-11-28" + ) + + [ "${GITHUB_TOKEN:-}" == "" ] || args+=("Authorization: Bearer $GITHUB_TOKEN") + + http::get /dev/stdout https://api.github.com/"$endpoint" "${args[@]}" +} + +github::tags::latest(){ + local repo="$1" + github::request "repos/$repo/tags" | jq -rc .[0].name +} + +github::releases::latest(){ + local repo="$1" + github::request "repos/$repo/releases/latest" | jq -rc . +} + +log::init +host::require jq +host::require tar +host::require curl +host::require shasum