Enable concurrent running of the image generation jobs (#4)

Group GPU image generate for GHCR and QUAY in one job, and similarly group CPU only image generation for GHCR and QUAY in another job. These two jobs can now run concurrently. By grouping the jobs for the same image type, it can use the docker layer cache to run the second job faster. If we separately build these images in four diffent and concurrent jobs, they won't be able to use the cache. If this still takes significantly long, we probably have to generate image in it's own job, so that they can run concurrently, without leveraging cache of docker layers. Also added make file target to lint the github workflows. Signed-off-by: Anil Vishnoi <[email protected]>
DS4SD · Sep 13, 2024 · fc11dc2 · fc11dc2
1 parent 4dd517c
commit fc11dc2
Show file tree

Hide file tree

Showing 3 changed files with 109 additions and 39 deletions.
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
@@ -13,9 +13,13 @@ env:
   QUAY_DOCLING_SERVE_CPU_IMAGE_NAME: ds4sd/docling-serve-cpu
   QUAY_DOCLING_SERVE_GPU_IMAGE_NAME: ds4sd/docling-serve
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build_and_publish_ui_image:
-    name: Push docling-serve container images to GHCR and QUAY
+  build_and_publish_cpu_images:
+    name: Push docling-serve "CPU only" container image to GHCR and QUAY
     runs-on: ubuntu-latest
     environment: registry-creds
     permissions:
@@ -53,31 +57,13 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-buildx-
 
-      - name: Extract metadata (tags, labels) for docling-serve/cpu ghcr image
+      - name: Extract metadata (tags, labels) for docling-serve (CPU only) ghcr image
         id: ghcr_serve_cpu_meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_CPU_IMAGE_NAME }}
 
-      - name: Extract metadata (tags, labels) for docling-serve/gpu ghcr image
-        id: ghcr_serve_gpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_GPU_IMAGE_NAME }}
-
-      - name: Extract metadata (tags, labels) for docling-serve/cpu quay image
-        id: quay_serve_cpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_CPU_IMAGE_NAME }}
-
-      - name: Extract metadata (tags, labels) for docking-serve/gpu quay image
-        id: quay_serve_gpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_GPU_IMAGE_NAME }}
-
-      - name: Build and push docling-serve/cpu image to ghcr.io
+      - name: Build and push docling-serve-cpu image to ghcr.io
         id: push-serve-cpu-ghcr
         uses: docker/build-push-action@v5
         with:
@@ -99,7 +85,73 @@ jobs:
           subject-digest: ${{ steps.push-serve-cpu-ghcr.outputs.digest }}
           push-to-registry: true
 
-      - name: Build and push docling-serve/gpu image to ghcr.io
+      - name: Extract metadata (tags, labels) for docling-serve (CPU only) quay image
+        id: quay_serve_cpu_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_CPU_IMAGE_NAME }}
+
+      - name: Build and push docling-serve-cpu image to quay.io
+        id: push-serve-cpu-quay
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.quay_serve_cpu_meta.outputs.tags }}
+          labels: ${{ steps.quay_serve_cpu_meta.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          file: Containerfile
+          build-args: |
+            --build-arg CPU_ONLY=true
+
+  build_and_publish_gpu_images:
+    name: Push docling-serve (with GPU support) container image to GHCR and QUAY
+    runs-on: ubuntu-latest
+    environment: registry-creds
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to the GHCR container image registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.GHCR_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Log in to the Quay container image registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.QUAY_REGISTRY }}
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Extract metadata (tags, labels) for docling-serve (GPU) ghcr image
+        id: ghcr_serve_gpu_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_GPU_IMAGE_NAME }}
+
+      - name: Build and push docling-serve (GPU) image to ghcr.io
         id: push-serve-gpu-ghcr
         uses: docker/build-push-action@v5
         with:
@@ -121,22 +173,13 @@ jobs:
           subject-digest: ${{ steps.push-serve-gpu-ghcr.outputs.digest }}
           push-to-registry: true
 
-      - name: Build and push docling-serve/cpu image to quay.io
-        id: push-serve-cpu-quay
-        uses: docker/build-push-action@v5
+      - name: Extract metadata (tags, labels) for docling-serve (GPU) quay image
+        id: quay_serve_gpu_meta
+        uses: docker/metadata-action@v5
         with:
-          context: .
-          push: true
-          tags: ${{ steps.quay_serve_cpu_meta.outputs.tags }}
-          labels: ${{ steps.quay_serve_cpu_meta.outputs.labels }}
-          platforms: linux/amd64
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          file: Containerfile
-          build-args: |
-            --build-arg CPU_ONLY=true
+          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_GPU_IMAGE_NAME }}
 
-      - name: Build and push docling-serve/gpu image to quay.io
+      - name: Build and push docling-serve (GPU) image to quay.io
         id: push-serve-gpu-quay
         uses: docker/build-push-action@v5
         with:
@@ -149,4 +192,4 @@ jobs:
           cache-to: type=gha,mode=max
           file: Containerfile
           build-args: |
-            --build-arg CPU_ONLY=false
+            --build-arg CPU_ONLY=false
diff --git a/.gitignore b/.gitignore
@@ -438,3 +438,6 @@ pip-selfcheck.json
 # Docs
 # docs/**/*.png
 # docs/**/*.svg
+
+# Makefile
+.action-lint
diff --git a/Makefile b/Makefile
@@ -9,21 +9,45 @@ help:
 ifeq ($(NOISY_BUILD),)
     ECHO_PREFIX=@
     CMD_PREFIX=@
+    PIPE_DEV_NULL=> /dev/null 2> /dev/null
 else
     ECHO_PREFIX=@\#
-    CMD_PREFIX=    PIPE_DEV_NULL=
+    CMD_PREFIX=
+    PIPE_DEV_NULL=
 endif
 
 TAG=$(shell git rev-parse HEAD)
 
+lint:
+	$(CMD_PREFIX) touch .action-lint
+
+.PHONY: docling-serve-cpu-image
 docling-serve-cpu-image: Containerfile ## Build docling-serve "cpu only" continaer image
 	$(ECHO_PREFIX) printf "  %-12s Containerfile\n" "[docling-serve CPU ONLY]"
 	$(CMD_PREFIX) docker build --build-arg CPU_ONLY=true -f Containerfile --platform linux/amd64 -t ghcr.io/ds4sd/docling-serve-cpu:$(TAG) .
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve-cpu:$(TAG) ghcr.io/ds4sd/docling-serve-cpu:main
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve-cpu:$(TAG) quay.io/ds4sd/docling-serve-cpu:main
 
+.PHONY: docling-serve-gpu-image
 docling-serve-gpu-image: Containerfile ## Build docling-serve continaer image with GPU support
 	$(ECHO_PREFIX) printf "  %-12s Containerfile\n" "[docling-serve with GPU]"
 	$(CMD_PREFIX) docker build --build-arg CPU_ONLY=false -f Containerfile --platform linux/amd64 -t ghcr.io/ds4sd/docling-serve:$(TAG) .
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve:$(TAG) ghcr.io/ds4sd/docling-serve:main
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve:$(TAG) quay.io/ds4sd/docling-serve:main
+
+.PHONY: action-lint
+action-lint: .action-lint ##      Lint GitHub Action workflows
+.action-lint: $(shell find .github -type f) | lint
+	$(ECHO_PREFIX) printf "  %-12s .github/...\n" "[ACTION LINT]"
+	$(CMD_PREFIX) if ! which actionlint $(PIPE_DEV_NULL) ; then \
+		echo "Please install actionlint." ; \
+		echo "go install github.com/rhysd/actionlint/cmd/actionlint@latest" ; \
+		exit 1 ; \
+	fi
+	$(CMD_PREFIX) if ! which shellcheck $(PIPE_DEV_NULL) ; then \
+		echo "Please install shellcheck." ; \
+		echo "https://github.com/koalaman/shellcheck#user-content-installing" ; \
+		exit 1 ; \
+	fi
+	$(CMD_PREFIX) actionlint -color
+	$(CMD_PREFIX) touch $@