diff --git a/.github/actions/hash/action.yml b/.github/actions/hash/action.yml new file mode 100644 index 00000000..85a6aee8 --- /dev/null +++ b/.github/actions/hash/action.yml @@ -0,0 +1,19 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Calculate version hash' +description: 'Calculate deps and os hash' +inputs: + files: + description: 'Files to use to calculate the hash' + required: true + default: "pdm.lock docker/bin/* docker/conf/* docker/Dockerfile" +outputs: + hash: # id of output + description: 'The time we greeted you' +runs: + using: 'composite' +# args: +# - ${{ inputs.files }} + steps: + - run: | + LOCK_SHA=$(echo sha1sum ${{ inputs.files }} | sha1sum | awk '{print $1}' | cut -c 1-8) + echo "hash=$LOCK_SHA" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f643f682..76f396d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: branches: - develop env: - HASH_SEEDS: pdm.lock docker/bin/* docker/conf/* docker/Dockerfile +# HASH_SEEDS: pdm.lock docker/bin/* docker/conf/* docker/Dockerfile DOCKER_CACHE_IMAGE: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images BUILD_DATE: $(date +"%Y-%m-%d %H:%M") DOCKER_DEFAULT_PLATFORM: linux/amd64 @@ -17,43 +17,43 @@ jobs: build: runs-on: ubuntu-20.04 + outputs: + docker-image: ${{ steps.image_name.outputs.name }} steps: - name: Checkout code uses: actions/checkout@v4 - - name: DockerHub login - uses: docker/login-action@v3 + + - uses: ./.github/actions/hash + id: release_hash + + - name: Image name + id: image_name + run: | + image_name="$DOCKER_CACHE_IMAGE:hde-dev-${{ steps.release_hash.outputs.hash }}" + image_name_latest="$DOCKER_CACHE_IMAGE:hde-dev-latest" + echo "name=$image_name" >> $GITHUB_OUTPUT + echo "latest=$image_name_latest" >> $GITHUB_OUTPUT + - name: Check if image exists + uses: ./.github/actions/image_exists + id: image_exists with: + image: ${{ steps.image_name.outputs.name }} username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Check Dev Image - continue-on-error: true - id: image_exists - run: | - set +e - LOCK_SHA=$(echo sha1sum ${{env.HASH_SEEDS}}| sha1sum | awk '{print $1}' | cut -c 1-8) - exists=$(docker manifest inspect ${DOCKER_CACHE_IMAGE}:hde-dev-${LOCK_SHA} > /dev/null 2>&1 && echo "exists" || echo "not_found") - exists="exists" - echo "result=$exists" >> "$GITHUB_OUTPUT" - echo "SHA=$LOCK_SHA" >> "$GITHUB_OUTPUT" - if [ $exists = "exists" ];then - echo "Tag ${{ steps.image_exists.outputs.SHA }} found" - else - echo "Tag ${{ steps.image_exists.outputs.SHA }} does not exist" - fi - name: Build Dev Image - if: ${{ steps.image_exists.outputs.result == 'not_found' || contains(github.event.head_commit.message, 'rebuild') }} + if: ${{ !steps.image_exists.outputs.exists || contains(github.event.head_commit.message, 'rebuild') }} + id: docker_build run: | - LOCK_SHA=${{ steps.image_exists.outputs.SHA }} docker buildx create --use --platform x86_64 --name builder --driver docker-container - docker buildx build \ + docker buildx build \ --platform x86_64 \ --builder builder \ --build-arg BUILD_DATE="${BUILD_DATE}" \ --progress=plain \ --cache-to type=local,ref=${DOCKER_CACHE_IMAGE}:hde-dev-latest,dest=./.AAA \ --cache-from ${DOCKER_CACHE_IMAGE}:hde-dev-latest \ - -t ${DOCKER_CACHE_IMAGE}:hde-dev-${LOCK_SHA} \ - -t ${DOCKER_CACHE_IMAGE}:hde-dev-latest \ + -t ${{ !steps.image_name.outputs.name }} \ + -t ${{ !steps.image_name.outputs.latest }} \ -f ./docker/Dockerfile \ --push \ --target python_dev_deps . @@ -73,64 +73,3 @@ jobs: run: black --check src/ - name: Flake8 run: flake8 src/ - - test: - runs-on: ubuntu-20.04 - needs: [build] - container: - image: unicef/hope-support-images:hde-dev-latest - credentials: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - ports: - - 8000:8000 - services: - redis: - image: redis - db: - image: postgres:14 - env: - POSTGRES_DATABASE: dedupe - POSTGRES_PASSWORD: postgres - POSTGRES_USERNAME: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: - DATABASE_URL: postgres://postgres:postgres@db:5432/dedupe - SECRET_KEY: secret_key - CACHE_URL: redis://redis:6379/0 - CELERY_BROKER_URL: redis://redis:6379/0 - PYTHONPATH: "/hde/code/src:/hde/__pypackages__/3.12/lib" - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Run tests - run: | - pytest tests -# echo "===================================" -# ls -al -# docker run --rm \ -# -e PYTHONPATH=/hde/code/src:/hde/__pypackages__/3.12/lib \ -# -e CACHE_URL="${CACHE_URL}" \ -# -e DATABASE_URL="${DATABASE_URL}" \ -# -e SECRET_KEY="${SECRET_KEY}" \ -# -e CELERY_BROKER_URL="${CELERY_BROKER_URL}" \ -# -v ${PWD}:/hde/code/ \ -# -w /hde/code/ \ -# -t ${DOCKER_CACHE_IMAGE}:hde-dev-latest \ -# pytest tests/ --create-db -v --cov --cov-report xml:coverage.xml -# -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v4 -# with: -# directory: ./coverage/reports/ -# env_vars: OS,PYTHON -# fail_ci_if_error: true -# files: /hde/code/coverage1.xml -# flags: unittests -# name: codecov-umbrella -# token: ${{ secrets.CODECOV_TOKEN }} -# verbose: true \ No newline at end of file diff --git a/.github/workflows/debug.yml b/.github/workflows/debug.yml new file mode 100644 index 00000000..339f73fe --- /dev/null +++ b/.github/workflows/debug.yml @@ -0,0 +1,43 @@ +name: CI + +on: always + +env: + DOCKER_DEFAULT_PLATFORM: linux/amd64 + DOCKER_CACHE_IMAGE: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images + +jobs: + + debug: + runs-on: ubuntu-20.04 + if: ${{ github.event.act }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Calculate hash + uses: ./.github/actions/hash + id: release_hash + + - name: Image name + id: image_name + run: | + image_name="$DOCKER_CACHE_IMAGE:hde-dev-${{ steps.release_hash.outputs.hash }}" + image_name_latest="$DOCKER_CACHE_IMAGE:hde-dev-latest" + echo "name=$image_name" >> $GITHUB_OUTPUT + echo "latest=$image_name_latest" >> $GITHUB_OUTPUT + + - name: Check if image exists + uses: ./.github/actions/image_exists + id: image_exists + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + image: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images + - name: Dump info + run: | + echo "=======================================================" + echo "Name : ${{steps.image_name.outputs.name}}" + echo "Hash : ${{steps.release_hash.outputs.hash}}" + echo "Exists: ${{steps.image_exists.outputs.exists}}" + echo "=======================================================" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..9f0178a2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,62 @@ +name: CI + +on: + push: + branches: + - develop + pull_request: + branches: + - develop + workflow_run: + workflows: [ci] + types: + - completed + +jobs: + + test: + runs-on: ubuntu-20.04 + container: + image: unicef/hope-support-images:hde-dev-latest + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + ports: + - 8000:8000 + services: + redis: + image: redis + db: + image: postgres:14 + env: + POSTGRES_DATABASE: dedupe + POSTGRES_PASSWORD: postgres + POSTGRES_USERNAME: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DATABASE_URL: postgres://postgres:postgres@db:5432/dedupe + SECRET_KEY: secret_key + CACHE_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + PYTHONPATH: "/hde/code/src:/hde/__pypackages__/3.12/lib" + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Run tests + run: | + pytest tests --create-db -v --cov --cov-report xml:coverage.xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + directory: ./coverage/reports/ + env_vars: OS,PYTHON + fail_ci_if_error: true + files: /hde/code/coverage1.xml + flags: unittests + name: codecov-umbrella + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index e72af819..72e9d96d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,10 +8,9 @@ __pycache__/ !.dockerignore !.flake8 !.gitignore -!.gitlab/ +!.github/* !.tx/config !.mypy.ini -!.gitlab-ci.yml !.pre-commit-config.yaml !.bumpversion.cfg !.trivyignore