updates CI

unicef · May 28, 2024 · b82d1b1 · b82d1b1
1 parent 0eb94fd
commit b82d1b1
Show file tree

Hide file tree

Showing 5 changed files with 148 additions and 86 deletions.
diff --git a/.github/actions/hash/action.yml b/.github/actions/hash/action.yml
@@ -0,0 +1,19 @@
+# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action
+name: 'Calculate version hash'
+description: 'Calculate deps and os hash'
+inputs:
+  files:
+    description: 'Files to use to calculate the hash'
+    required: true
+    default: "pdm.lock docker/bin/* docker/conf/* docker/Dockerfile"
+outputs:
+  hash: # id of output
+    description: 'The time we greeted you'
+runs:
+  using: 'composite'
+#  args:
+#    - ${{ inputs.files }}
+  steps:
+    - run: |
+        LOCK_SHA=$(echo sha1sum ${{ inputs.files }} | sha1sum | awk '{print $1}' | cut -c 1-8)
+        echo "hash=$LOCK_SHA" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,7 +8,7 @@ on:
     branches:
     - develop
 env:
-  HASH_SEEDS: pdm.lock docker/bin/* docker/conf/* docker/Dockerfile
+#  HASH_SEEDS: pdm.lock docker/bin/* docker/conf/* docker/Dockerfile
   DOCKER_CACHE_IMAGE: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images
   BUILD_DATE: $(date +"%Y-%m-%d %H:%M")
   DOCKER_DEFAULT_PLATFORM: linux/amd64
@@ -17,43 +17,43 @@ jobs:
 
     build:
       runs-on: ubuntu-20.04
+      outputs:
+        docker-image: ${{ steps.image_name.outputs.name }}
       steps:
         - name: Checkout code
           uses: actions/checkout@v4
-        - name: DockerHub login
-          uses: docker/login-action@v3
+
+        - uses: ./.github/actions/hash
+          id: release_hash
+
+        - name: Image name
+          id: image_name
+          run: |
+            image_name="$DOCKER_CACHE_IMAGE:hde-dev-${{ steps.release_hash.outputs.hash }}"
+            image_name_latest="$DOCKER_CACHE_IMAGE:hde-dev-latest"
+            echo "name=$image_name" >> $GITHUB_OUTPUT
+            echo "latest=$image_name_latest" >> $GITHUB_OUTPUT
+        - name: Check if image exists
+          uses: ./.github/actions/image_exists
+          id: image_exists
           with:
+            image: ${{ steps.image_name.outputs.name }}
             username: ${{ secrets.DOCKERHUB_USERNAME }}
             password: ${{ secrets.DOCKERHUB_TOKEN }}
-        - name: Check Dev Image
-          continue-on-error: true
-          id: image_exists
-          run: |
-            set +e
-            LOCK_SHA=$(echo sha1sum ${{env.HASH_SEEDS}}| sha1sum | awk '{print $1}' | cut -c 1-8)
-            exists=$(docker manifest inspect ${DOCKER_CACHE_IMAGE}:hde-dev-${LOCK_SHA} > /dev/null 2>&1 && echo "exists" || echo "not_found")
-            exists="exists"
-            echo "result=$exists" >> "$GITHUB_OUTPUT"
-            echo "SHA=$LOCK_SHA" >> "$GITHUB_OUTPUT"
-            if [ $exists = "exists" ];then
-              echo "Tag ${{ steps.image_exists.outputs.SHA }} found"
-            else
-              echo "Tag ${{ steps.image_exists.outputs.SHA }} does not exist"            
-            fi
         - name: Build Dev Image
-          if: ${{ steps.image_exists.outputs.result == 'not_found' || contains(github.event.head_commit.message, 'rebuild') }}
+          if: ${{ !steps.image_exists.outputs.exists || contains(github.event.head_commit.message, 'rebuild') }}
+          id: docker_build
           run: |
-            LOCK_SHA=${{ steps.image_exists.outputs.SHA }}
             docker buildx create --use --platform x86_64 --name builder --driver docker-container
-            docker buildx build \            
+            docker buildx build \
               --platform x86_64 \
               --builder builder \
               --build-arg BUILD_DATE="${BUILD_DATE}" \
               --progress=plain \
               --cache-to type=local,ref=${DOCKER_CACHE_IMAGE}:hde-dev-latest,dest=./.AAA \
               --cache-from ${DOCKER_CACHE_IMAGE}:hde-dev-latest \
-              -t ${DOCKER_CACHE_IMAGE}:hde-dev-${LOCK_SHA} \
-              -t ${DOCKER_CACHE_IMAGE}:hde-dev-latest \
+              -t ${{ !steps.image_name.outputs.name }} \
+              -t ${{ !steps.image_name.outputs.latest }} \
               -f ./docker/Dockerfile \
               --push \
               --target python_dev_deps .
@@ -73,64 +73,3 @@ jobs:
           run: black --check src/
         - name: Flake8
           run: flake8 src/
-
-    test:
-      runs-on: ubuntu-20.04
-      needs: [build]
-      container:
-        image: unicef/hope-support-images:hde-dev-latest
-        credentials:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-        ports:
-          - 8000:8000
-      services:
-        redis:
-          image: redis
-        db:
-          image: postgres:14
-          env:
-            POSTGRES_DATABASE: dedupe
-            POSTGRES_PASSWORD: postgres
-            POSTGRES_USERNAME: postgres
-          options: >-
-            --health-cmd pg_isready
-            --health-interval 10s
-            --health-timeout 5s
-            --health-retries 5
-      env:
-        DATABASE_URL: postgres://postgres:postgres@db:5432/dedupe
-        SECRET_KEY: secret_key
-        CACHE_URL: redis://redis:6379/0
-        CELERY_BROKER_URL: redis://redis:6379/0
-        PYTHONPATH: "/hde/code/src:/hde/__pypackages__/3.12/lib"
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v2
-        - name: Run tests
-          run: |
-            pytest tests
-#              echo "==================================="
-#              ls -al
-#                docker run --rm \
-#                -e PYTHONPATH=/hde/code/src:/hde/__pypackages__/3.12/lib \
-#                -e CACHE_URL="${CACHE_URL}" \
-#                -e DATABASE_URL="${DATABASE_URL}" \
-#                -e SECRET_KEY="${SECRET_KEY}" \
-#                -e CELERY_BROKER_URL="${CELERY_BROKER_URL}" \
-#                -v ${PWD}:/hde/code/ \
-#                -w /hde/code/ \
-#                -t ${DOCKER_CACHE_IMAGE}:hde-dev-latest \
-#                pytest tests/ --create-db -v --cov --cov-report xml:coverage.xml
-#
-#        - name: Upload coverage to Codecov
-#          uses: codecov/codecov-action@v4
-#          with:
-#            directory: ./coverage/reports/
-#            env_vars: OS,PYTHON
-#            fail_ci_if_error: true
-#            files: /hde/code/coverage1.xml
-#            flags: unittests
-#            name: codecov-umbrella
-#            token: ${{ secrets.CODECOV_TOKEN }}
-#            verbose: true
diff --git a/.github/workflows/debug.yml b/.github/workflows/debug.yml
@@ -0,0 +1,43 @@
+name: CI
+
+on: always
+
+env:
+  DOCKER_DEFAULT_PLATFORM: linux/amd64
+  DOCKER_CACHE_IMAGE: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images
+
+jobs:
+
+    debug:
+      runs-on: ubuntu-20.04
+      if: ${{ github.event.act }}
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v4
+
+        - name: Calculate hash
+          uses: ./.github/actions/hash
+          id: release_hash
+
+        - name: Image name
+          id: image_name
+          run: |
+            image_name="$DOCKER_CACHE_IMAGE:hde-dev-${{ steps.release_hash.outputs.hash }}"
+            image_name_latest="$DOCKER_CACHE_IMAGE:hde-dev-latest"
+            echo "name=$image_name" >> $GITHUB_OUTPUT
+            echo "latest=$image_name_latest" >> $GITHUB_OUTPUT
+
+        - name: Check if image exists
+          uses: ./.github/actions/image_exists
+          id: image_exists
+          with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}
+            image: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images
+        - name: Dump info
+          run: |
+            echo "======================================================="
+            echo "Name  : ${{steps.image_name.outputs.name}}"
+            echo "Hash  : ${{steps.release_hash.outputs.hash}}"
+            echo "Exists: ${{steps.image_exists.outputs.exists}}"
+            echo "======================================================="
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,62 @@
+name: CI
+
+on:
+  push:
+    branches:
+    - develop
+  pull_request:
+    branches:
+    - develop
+  workflow_run:
+    workflows: [ci]
+    types:
+      - completed
+
+jobs:
+
+    test:
+      runs-on: ubuntu-20.04
+      container:
+        image: unicef/hope-support-images:hde-dev-latest
+        credentials:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        ports:
+          - 8000:8000
+      services:
+        redis:
+          image: redis
+        db:
+          image: postgres:14
+          env:
+            POSTGRES_DATABASE: dedupe
+            POSTGRES_PASSWORD: postgres
+            POSTGRES_USERNAME: postgres
+          options: >-
+            --health-cmd pg_isready
+            --health-interval 10s
+            --health-timeout 5s
+            --health-retries 5
+      env:
+        DATABASE_URL: postgres://postgres:postgres@db:5432/dedupe
+        SECRET_KEY: secret_key
+        CACHE_URL: redis://redis:6379/0
+        CELERY_BROKER_URL: redis://redis:6379/0
+        PYTHONPATH: "/hde/code/src:/hde/__pypackages__/3.12/lib"
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v2
+        - name: Run tests
+          run: |
+            pytest tests --create-db -v --cov --cov-report xml:coverage.xml
+        - name: Upload coverage to Codecov
+          uses: codecov/codecov-action@v4
+          with:
+            directory: ./coverage/reports/
+            env_vars: OS,PYTHON
+            fail_ci_if_error: true
+            files: /hde/code/coverage1.xml
+            flags: unittests
+            name: codecov-umbrella
+            token: ${{ secrets.CODECOV_TOKEN }}
+            verbose: true
diff --git a/.gitignore b/.gitignore
@@ -8,10 +8,9 @@ __pycache__/
 !.dockerignore
 !.flake8
 !.gitignore
-!.gitlab/
+!.github/*
 !.tx/config
 !.mypy.ini
-!.gitlab-ci.yml
 !.pre-commit-config.yaml
 !.bumpversion.cfg
 !.trivyignore