diff --git a/.github/actions/delete_docker/action.yml b/.github/actions/delete_docker/action.yml new file mode 100644 index 00000000..30576b80 --- /dev/null +++ b/.github/actions/delete_docker/action.yml @@ -0,0 +1,72 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Delete Docker Image' +description: 'Delete Docker Image from Registry' +inputs: + image: + description: 'Image name to delete' + required: true + tag: + description: 'Tag to delete' + required: true + username: + description: 'Docker Username' + required: true + password: + description: 'Docker Password' + required: true + registry: + description: 'Files to use to calculate the hash' + required: false + default: "https://hub.docker.com/v2" + + +outputs: + checksum: # id of output + description: 'The time we greeted you' + value: ${{ steps.calc.outputs.checksum }} + +runs: + using: 'composite' + steps: + - id: calc + shell: bash + run: | + registry=${{ inputs.registry }} + name=${{ inputs.image }} + tag=${{ inputs.tag }} + + echo "::notice:: 111111111 Deleting Image ${name}:${tag}" + + TOKEN=$(\ + curl \ + --silent \ + --header "Content-Type: application/json" \ + --request POST \ + --data '{"username": "'${{ inputs.username }}'", "password": "'${{ inputs.password }}'" }' \ + ${registry}/users/login/ \ + | jq -r .token\ + ) + + curl -X DELETE \ + --header "Authorization: JWT ${TOKEN}" \ + --header "Accept: application/json" \ + ${registry}/repositories/${name}/tags/${tag} + + +# echo "::notice:: curl $auth -sI -k \ +# -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ +# https://${registry}/v2/${name}/manifests/${tag}" +# +# echo "::notice:: 111111111 $auth" +# +# digest=$(curl $auth -sI -k \ +# -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +# "https://${registry}/v2/${name}/manifests/${tag}") +# echo "::notice:: 111111111 ${digest}" +# +# curl $auth -X DELETE -sI -k "https://${registry}/v2/${name}/manifests/$( +# curl $auth -sI -k \ +# -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +# "https://${registry}/v2/${name}/manifests/${tag}" \ +# | tr -d '\r' | sed -En 's/^Docker-Content-Digest: (.*)/\1/pi' +# )" diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml index ac97b6bc..336204fd 100644 --- a/.github/actions/docker_build/action.yml +++ b/.github/actions/docker_build/action.yml @@ -43,13 +43,13 @@ outputs: value: ${{ steps.meta.outputs.version }} created: description: 'True if new image has been created' - value: ${{ !steps.image_status.outputs.updated }} + value: ${{ steps.status.outputs.created }} digest: description: 'Built image digest' - value: ${{ !steps.build_push.outputs.digest }} + value: ${{ steps.build_push.outputs.digest }} imageId: description: 'Built image ID' - value: ${{ !steps.build_push.outputs.imageId }} + value: ${{ steps.build_push.outputs.imageId }} runs: @@ -122,8 +122,10 @@ runs: -v fatal \ ${{ steps.image_name.outputs.name }} 2>/dev/null) code_checksum="${{ inputs.code_checksum }}" - - if [[ -z "$image_checksum" ]]; then + if [[ "${{ contains(github.event.head_commit.message, 'ci:all') }}" == "true" ]];then + echo "::warning::🤔 Forced rebuild" + echo "updated=false" >> $GITHUB_OUTPUT + elif [[ -z "$image_checksum" ]]; then echo "::warning::🤔 No image checksum found" echo "updated=false" >> $GITHUB_OUTPUT elif [[ $image_checksum == $code_checksum ]]; then @@ -145,7 +147,7 @@ runs: driver-opts: | image=moby/buildkit:v0.13.2 network=host - - name: Build and push + - name: Build and push info if: (steps.image_status.outputs.updated != 'true' || inputs.rebuild == 'true') && inputs.dryrun == 'true' shell: bash run: | @@ -164,8 +166,8 @@ runs: with: context: . tags: ${{ steps.meta.outputs.tags }} - labels: "${{ steps.meta.outputs.labels }}\na=1\nb=2" - annotations: "${{ steps.meta.outputs.annotations }}\nchecksum=${{ inputs.checksum }}" + labels: "${{ steps.meta.outputs.labels }}\nchecksum=${{ inputs.code_checksum }}\ndistro=${{ inputs.target }}" + annotations: "${{ steps.meta.outputs.annotations }}\nchecksum=${{ inputs.code_checksum }}\ndistro=${{ inputs.target }}" target: ${{ inputs.target }} file: ./docker/Dockerfile platforms: linux/amd64 @@ -178,11 +180,16 @@ runs: GITHUB_SERVER_URL=${{ github.server_url }} GITHUB_REPOSITORY=${{ github.repository }} BUILD_DATE=${{ env.BUILD_DATE }} + DISTRO=${{ inputs.target }} CHECKSUM=${{ inputs.code_checksum }} VERSION=${{ steps.meta.outputs.version }} SOURCE_COMMIT=${{ steps.last_commit.outputs.last_commit_short_sha }} - name: Status + id: status if: (steps.image_status.outputs.updated != 'true' || inputs.rebuild == 'true') && inputs.dryrun != 'true' shell: bash run: | + echo "${{ toJSON(steps.build_push.outputs) }}" + regctl image inspect -p linux/amd64 ${{ steps.image_name.outputs.name }} echo "::notice:: Image ${{ steps.meta.outputs.tags }} successfully built and pushed" + echo "created=true" >> $GITHUB_OUTPUT diff --git a/.github/file-filters.yml b/.github/file-filters.yml index 2579804c..c9602022 100644 --- a/.github/file-filters.yml +++ b/.github/file-filters.yml @@ -1,6 +1,7 @@ # This is used by the action https://github.com/dorny/paths-filter docker: &docker - - added|modified: './docker/**' + - added|modified: './docker/**/*' + - added|modified: './docker/*' dependencies: &dependencies - 'pdm.lock' diff --git a/.github/workflows/delete_image.yml b/.github/workflows/delete_image.yml index 80d5ab23..c3aae144 100644 --- a/.github/workflows/delete_image.yml +++ b/.github/workflows/delete_image.yml @@ -1,39 +1,30 @@ -name: Branch Deleted +name: Delete Outdated Docker Image on: delete jobs: - delete: - if: github.event.ref_type == 'branch' + delete_branch: + if: github.event.ref_type == 'branch' || github.event.ref_type == 'tag' runs-on: ubuntu-latest steps: - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - - - name: Delete Test Docker Image - shell: bash - run: | - name="${{vars.DOCKER_IMAGE}}:test-${{steps.meta.outputs.version}}" - registry="https://registry-1.docker.io" - curl -v -sSL -X DELETE "http://${registry}/v2/${name}/manifests/$( - curl -sSL -I \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - "http://${registry}/v2/${name}/manifests/$( - curl -sSL "http://${registry}/v2/${name}/tags/list" | jq -r '.tags[0]' - )" \ - | awk '$1 == "Docker-Content-Digest:" { print $2 }' \ - | tr -d $'\r' \ - )" - - name: Delete linked Docker Image - shell: bash + - name: Checkout code + uses: actions/checkout@v4.1.7 + - shell: bash run: | - name="${{vars.DOCKER_IMAGE}}:${{steps.meta.outputs.version}}" - registry="https://registry-1.docker.io" - curl -v -sSL -X DELETE "http://${registry}/v2/${name}/manifests/$( - curl -sSL -I \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - "http://${registry}/v2/${name}/manifests/$( - curl -sSL "http://${registry}/v2/${name}/tags/list" | jq -r '.tags[0]' - )" \ - | awk '$1 == "Docker-Content-Digest:" { print $2 }' \ - | tr -d $'\r' \ - )" + ref="${{github.ref}}" + tag=$(echo $ref | sed -e "s#refs/heads/##g" | sed -e s#/#-##g) + echo "ref=$ref" >> $GITHUB_ENV + echo "dist_tag=$tag" >> $GITHUB_ENV + echo "test_tag=test-${tag}" >> $GITHUB_ENV + - uses: ./.github/actions/delete_docker + if: github.event.ref_type == 'tag' + with: + image: ${{vars.DOCKER_IMAGE}} + tag: ${{ env.dist_tag }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: ./.github/actions/delete_docker + with: + image: ${{vars.DOCKER_IMAGE}} + tag: ${{ env.test_tag }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 44b35a9f..2ba79248 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,12 +3,15 @@ on: push: branches: - develop -# - master -# - staging -# - releases/* - pull_request: - branches: [develop, master] - types: [synchronize, opened, reopened, ready_for_review] + - master + - staging + - release/* + - feature/* + - bugfix/* + - hotfix/* +# pull_request: +# branches: [develop, master] +# types: [synchronize, opened, reopened, ready_for_review] defaults: run: @@ -49,7 +52,7 @@ jobs: if: github.event.pull_request.draft == false && needs.changes.outputs.lint steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install requirements @@ -66,7 +69,7 @@ jobs: if: github.event.pull_request.draft == false && needs.changes.outputs.lint steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install requirements @@ -79,7 +82,7 @@ jobs: if: github.event.pull_request.draft == false && needs.changes.outputs.lint steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install requirements diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..8b1bbb06 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,185 @@ +name: Publish Release + +on: + release: + types: [published] + + +concurrency: + group: "${{ github.workflow }}" + cancel-in-progress: true + +defaults: + run: + shell: bash + +permissions: + id-token: write + attestations: write + + +jobs: + check: + runs-on: ubuntu-latest + steps: + - id: killswitch + name: Check release blockers + shell: bash + run: | + if [[ -z '${{ inputs.force }}' ]] && gh issue list -l release-blocker -s open | grep -q '^[0-9]\+[[:space:]]'; then + echo "Open release-blocking issues found, cancelling release..."; + gh api -X POST repos/:owner/:repo/actions/runs/$GITHUB_RUN_ID/cancel; + fi + + build: + name: Build Release Test Docker + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [ check ] + defaults: + run: + shell: bash + outputs: + image: ${{ steps.build.outputs.image }} + version: ${{ steps.build.outputs.version }} + created: ${{ steps.build.outputs.created }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.7 + - id: checksum + uses: ./.github/actions/checksum + - name: Build Image + id: build + uses: ./.github/actions/docker_build + with: + image: ${{ vars.DOCKER_IMAGE }} + target: 'python_dev_deps' + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + code_checksum: ${{ steps.checksum.outputs.checksum }} + + test: + name: Run Pre-Release Test + needs: [ build ] + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 16379:6379 + db: + image: postgres:14 + env: + POSTGRES_DATABASE: bitcaster + POSTGRES_PASSWORD: postgres + POSTGRES_USERNAME: postgres + ports: + - 15432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DOCKER_DEFAULT_PLATFORM: linux/amd64 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run tests + run: | + docker run --rm \ + -e DATABASE_URL=postgres://postgres:postgres@localhost:15432/bitcaster \ + -e SECRET_KEY=secret_key \ + -e CACHE_URL=redis://localhost:16379/0 \ + -e CELERY_BROKER_URL=redis://localhost:16379/0 \ + --network host \ + -v $PWD:/code/app \ + -w /code/app \ + -t ${{ vars.DOCKER_IMAGE }}:${{needs.build.outputs.version}} \ + pytest tests -v --create-db --maxfail=10 --cov -n auto --cov-report xml:coverage.xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + token: ${{ secrets.CODECOV_TOKEN }} + if: env.token != null + with: + env_vars: OS,PYTHON + fail_ci_if_error: true + files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} + verbose: false + name: codecov-${{env.GITHUB_REF_NAME}} + + deployable: + name: "Build Release Docker" + needs: [ test ] + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + outputs: + image: ${{ steps.build.outputs.image }} + version: ${{ steps.build.outputs.version }} + created: ${{ steps.build.outputs.created }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.7 + - id: checksum + uses: ./.github/actions/checksum + - name: Build + id: build + uses: ./.github/actions/docker_build + with: + dryrun: ${{ env.ACT || 'false' }} + push: 'true' + rebuild: ${{ env.BUILD == 'true'}} + image: ${{ vars.DOCKER_IMAGE }} + target: 'dist' + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + code_checksum: ${{ steps.checksum.outputs.checksum }} + - shell: bash + run: | + echo "${{ toJSON(steps.build.outputs) }}" + - name: Generate artifact attestation + if: steps.build.outputs.digest + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ vars.DOCKER_IMAGE }}:${{ steps.build.outputs.version }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + + - uses: ./.github/actions/delete_docker + with: + image: ${{vars.DOCKER_IMAGE}} + tag: test-${{ steps.build.outputs.version }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + trivy: + name: Check Release with Trivy + runs-on: ubuntu-latest + needs: [ deployable ] + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + if: needs.deployable.outputs.created == 'true' + || contains(github.event.head_commit.message, 'ci:scan') + || contains(github.event.head_commit.message, 'ci:all') + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{needs.deployable.outputs.image}} + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index b5ed8fb1..1efe142a 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -3,12 +3,15 @@ on: push: branches: - develop -# - master -# - staging -# - releases/* - pull_request: - branches: [develop, master] - types: [synchronize, opened, reopened, ready_for_review] + - master + - staging + - release/* + - feature/* + - bugfix/* + - hotfix/* +# pull_request: +# branches: [develop, master] +# types: [synchronize, opened, reopened, ready_for_review] defaults: run: @@ -61,7 +64,7 @@ jobs: # Github token of the repository (automatically created by Github) GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information. # File or directory to run bandit on - # path: # optional, default is . + path: src # optional, default is . # Report only issues of a given severity level or higher. Can be LOW, MEDIUM or HIGH. Default is UNDEFINED (everything) # level: # optional, default is UNDEFINED # Report only issues of a given confidence level or higher. Can be LOW, MEDIUM or HIGH. Default is UNDEFINED (everything) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d4fed1a9..5970dae9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,9 @@ name: Test on: + create: + branches: + - releases/* push: branches: - develop @@ -10,9 +13,9 @@ on: - feature/* - bugfix/* - hotfix/* - pull_request: - branches: [ develop, master ] - types: [ synchronize, opened, reopened, ready_for_review ] +# pull_request: +# branches: [ develop, master ] +# types: [ synchronize, opened, reopened, ready_for_review ] concurrency: group: "${{ github.workflow }}-${{ github.ref }}" @@ -29,19 +32,20 @@ permissions: jobs: changes: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name - + if: (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + || github.event_name == 'create' runs-on: ubuntu-latest timeout-minutes: 1 defaults: run: shell: bash outputs: - run_tests: ${{ steps.changes.outputs.run_tests }} + run_tests: ${{ steps.changed-files.outputs.run_tests }} steps: - name: Checkout code uses: actions/checkout@v4.1.7 - - id: changes + - id: changed-files name: Check for file changes uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 with: @@ -50,8 +54,9 @@ jobs: filters: .github/file-filters.yml - name: info shell: bash - run: | - force_build="${{ contains(github.event.head_commit.message, 'ci:build') }}" + run: | + github_ref="${{ github.ref }}" + force_build="${{ contains(github.event.head_commit.message, 'ci:build') || contains(github.event.head_commit.message, 'ci:release')}}" force_scan="${{ contains(github.event.head_commit.message, 'ci:scan') }}" force_test="${{ contains(github.event.head_commit.message, 'ci:test') }}" @@ -62,14 +67,14 @@ jobs: elif [[ $force_scan == "true" ]]; then echo "::notice:: Forced trivy scan due to commit message" fi - if [[ $force_build == "true" || "${{needs.changes.outputs.run_tests}}" == "true" ]]; then + if [[ $force_build == "true" || "${{steps.changed-files.outputs.run_tests}}" == "true" ]]; then echo "BUILD=true" >> $GITHUB_ENV fi build: needs: [ changes ] runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 30 defaults: run: shell: bash @@ -82,12 +87,12 @@ jobs: uses: actions/checkout@v4.1.7 - id: checksum uses: ./.github/actions/checksum - - name: Build and Test + - name: Build Image id: build uses: ./.github/actions/docker_build with: dryrun: ${{ env.ACT || 'false' }} - rebuild: ${{ contains(github.event.head_commit.message, 'ci:build') }} + rebuild: ${{ env.BUILD == 'true'}} image: ${{ vars.DOCKER_IMAGE }} target: 'python_dev_deps' username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -97,13 +102,23 @@ jobs: test: name: Run Test Suite needs: [ changes,build ] - if: needs.changes.outputs.run_tests == 'true' || contains(github.event.head_commit.message, 'ci:test') + if: (needs.changes.outputs.run_tests == 'true' + || contains(github.event.head_commit.message, 'ci:test') + || contains(github.event.head_commit.message, 'ci:all') + || github.event_name == 'create') runs-on: ubuntu-latest services: redis: image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 db: - image: postgres:14 + image: postgres:16 env: POSTGRES_DATABASE: dedupe POSTGRES_PASSWORD: postgres @@ -125,8 +140,8 @@ jobs: docker run --rm \ -e DATABASE_URL=postgres://postgres:postgres@localhost:5432/dedupe \ -e SECRET_KEY=secret_key \ - -e CACHE_URL=redis://redis:6379/0 \ - -e CELERY_BROKER_URL=redis://redis:6379/0 \ + -e CACHE_URL=redis://localhost:6379/0 \ + -e CELERY_BROKER_URL=redis://localhost:6379/0 \ --network host \ -v $PWD:/code/app \ -w /code/app \ @@ -142,43 +157,19 @@ jobs: verbose: false name: codecov-${{env.GITHUB_REF_NAME}} - trivy: - name: Check Image with Trivy - runs-on: ubuntu-latest - needs: [ build ] - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - if: needs.build.outputs.created == 'true' || contains(github.event.head_commit.message, 'ci:scan') - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ${{needs.build.outputs.image}} - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: 'trivy-results.sarif' - - release: + deployable: if: - contains(' - refs/heads/develop - refs/heads/staging - refs/heads/master - ', github.ref) || contains(github.event.head_commit.message, 'ci:release') + contains(github.ref, '/release/') + || endsWith(github.ref, '/develop') + || endsWith(github.ref, '/master') + || endsWith(github.ref, '/staging') + || contains(github.event.head_commit.message, 'ci:release') + || contains(github.event.head_commit.message, 'ci:all') - name: "Release Docker" + name: "Build deployable Docker" needs: [ test ] runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 30 defaults: run: shell: bash @@ -196,15 +187,40 @@ jobs: uses: ./.github/actions/docker_build with: dryrun: ${{ env.ACT || 'false' }} - rebuild: ${{ contains(github.event.head_commit.message, 'ci:build') }} + rebuild: ${{ env.BUILD == 'true'}} image: ${{ vars.DOCKER_IMAGE }} target: 'dist' username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} code_checksum: ${{ contains(github.event.head_commit.message, 'ci:build') && steps.checksum.outputs.checksum || '' }} - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 + - shell: bash + run: | + echo "${{ toJSON(steps.build.outputs) }}" + + trivy: + name: Check Image with Trivy + runs-on: ubuntu-latest + needs: [ deployable ] + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + if: needs.release.outputs.created == 'true' + || contains(github.event.head_commit.message, 'ci:scan') + || contains(github.event.head_commit.message, 'ci:all') + || github.event_name == 'create' + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master with: - subject-name: ${{ steps.build.outputs.image }} - subject-digest: ${{ steps.build.outputs.digest }} - push-to-registry: true + image-ref: ${{needs.deployable.outputs.image}} + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' diff --git a/.gitignore b/.gitignore index b9fb62bc..6523fb3e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.egg-info __pycache__/ *.py[cod] +__pypackages__/ !tests/.coveragerc !.dockerignore @@ -32,5 +33,3 @@ act.* !.pylintrc !.isort.cfg !.git - - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f2aef2b..9b2fcc78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,25 +5,26 @@ repos: - id: isort stages: [commit] - repo: https://github.com/ambv/black - rev: 24.1.1 + rev: 24.4.2 hooks: - id: black args: [--config=pyproject.toml] exclude: "migrations|snapshots" stages: [commit] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.1.0 hooks: - id: flake8 args: [--config=.flake8] + additional_dependencies: [flake8-bugbear==22.9.23] stages: [ commit ] - repo: https://github.com/PyCQA/bandit - rev: '1.7.8' # Update me! + rev: '1.7.9' # Update me! hooks: - id: bandit args: ["-c", "bandit.yaml"] - repo: https://github.com/twisted/towncrier - rev: 22.13.0 + rev: 23.11.0 hooks: - - id: towncrier-check \ No newline at end of file + - id: towncrier-check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..a23fe5c9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Contributing + +## System Requirements + +- python 3.12 +- [direnv](https://direnv.net/) - not mandatory but strongly recommended +- [pdm](https://pdm.fming.dev/2.9/) + + + + +**WARNING** +> Hope Deduplication Engine implements **security first** policy. It means that configuration default values are "almost" production compliant. +> +> Es. `DEBUG=False` or `SECURE_SSL_REDIRECT=True`. +> +> Be sure to run `./manage.py env --check` and `./manage.py env -g all` to check and display your configuration + + + +### 1. Clone repo and install requirements + git clone https://github.com/unicef/hope-dedup-engine + pdm venv create 3.11 + pdm install + pdm venv activate in-project + pre-commit install + +### 2. configure your environment + +Uses `./manage.py env` to check required (and optional) variables to put + + ./manage.py env --check + + +### 3. Run upgrade to run migrations and initial setup + + ./manage.py upgrade diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 00000000..4414e08c --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,108 @@ +## DB Schema +```mermaid +erDiagram + DeduplicationSet { + string name + string reference_pk + string state + string notification_url + } + + Entry { + string reference_pk + + date birth_date + + string filename + + string given_name + string middle_name + string patronymic + string matronymic + string family_name + + string[] middle_names + string[] family_names + } + + DocumentType { + string name + } + + Document { + string number + } + + Duplicate { + string first_reference_pk + string second_reference_pk + float similarity + } + + IgnoredKeyPair { + string first_reference_pk + string second_reference_pk + } + + Similarity { + float raw + float scaled + } + + Rule { + int scale_factor + bool enabled + } + + DeduplicationSet ||--o{ Entry : "" + DeduplicationSet ||--o{ Duplicate : "" + DeduplicationSet ||--o{ IgnoredKeyPair : "" + + Entry ||--o{ Document : "" + + DocumentType ||--o{ Document : "" + + Duplicate ||--o{ Similarity : "" + + Similarity }o--|| Rule : "" + +``` +### Notes + +**Entry** + +Possible name parts are taken from [Wikipedia](https://en.wikipedia.org/wiki/Personal_name#Structure_in_humans). `middle_name` and `middle_names` are merged into a single list before comparison as well as `family_name` and `family_names`. + +**DocumentType** + +New document types can be added through admin. It only makes sense to compare documents of the same type. + +**Rule** + +Rule executes code to compare specific entries. The result of excution is multipled by scale factor. Both raw and scaled result is stored in `Similarity` table. + +**Similarity** + +`raw` field contains value between 0 and 1. `scaled` field contains value between 0 and 1 calculated using the formula below + +```math +\left. {r (x) s} \middle/ {\sum_{i=1}^n s_i} \right. +``` + +where + +- $r (x)$ - rule execution result +- $s$ - specific rule scale factor +- $s_i$ - *ith* enabled rule scale factor + +**Duplicate** + +`similarity` takes values between 0 and 1. And is computed using formula below + +```math +\left. {\sum_{i=1}^n r_i (x) s_i} \middle/ {\sum_{i=1}^n s_i} \right. +``` + +where +- $r_i (x)$ - *ith* enabled rule execution result +- $s_i$ - *ith* enabled rule scale factor diff --git a/LICENSE b/LICENSE index 94621be2..be3f7b28 100644 --- a/LICENSE +++ b/LICENSE @@ -1,20 +1,7 @@ -Copyright (c) 2014 - 2024 UNICEF. All rights reserved. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License below for more details. - ------------------------------------------------------------------------- - GNU AFFERO GENERAL PUBLIC LICENSE + GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 - Copyright (C) 2007 Free Software Foundation, Inc. + Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. @@ -656,7 +643,7 @@ the "copyright" line and a pointer to where the full notice is found. GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . + along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. @@ -671,4 +658,4 @@ specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see -. +. diff --git a/README.md b/README.md index a0f22857..81bba4e9 100644 --- a/README.md +++ b/README.md @@ -4,44 +4,105 @@ ABOUT HOPE Deduplication Engine [![Test](https://github.com/unicef/hope-dedup-engine/actions/workflows/test.yml/badge.svg)](https://github.com/unicef/hope-dedup-engine/actions/workflows/test.yml) [![Lint](https://github.com/unicef/hope-dedup-engine/actions/workflows/lint.yml/badge.svg)](https://github.com/unicef/hope-dedup-engine/actions/workflows/lint.yml) [![codecov](https://codecov.io/gh/unicef/hope-dedup-engine/graph/badge.svg?token=kAuZEX5k5o)](https://codecov.io/gh/unicef/hope-dedup-engine) -![Version](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Fsaxix%2Ftrash%2Fdevelop%2Fpyproject.toml&query=%24.project.version&label=version) -![License](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Fsaxix%2Ftrash%2Fdevelop%2Fpyproject.toml&query=%24.project.license.text&label=license) +![Version](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%unicef%2Fhope-dedup-engine%2Fdevelop%2Fpyproject.toml&query=%24.project.version&label=version) +![License](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Funicef%2Fhope-dedup-engine%2Fdevelop%2Fpyproject.toml&query=%24.project.license.text&label=license) -## Contributing +## Configuration and Usage -### System Requirements +#### Display the Current Configuration -- python 3.12 -- [direnv](https://direnv.net/) - not mandatory but strongly recommended -- [pdm](https://pdm.fming.dev/2.9/) + $ docker run -it -t unicef/hope-dedupe-engine:release-0.1 django-admin env + +#### Check Mandatory Environment Variables + $ docker run -it -t unicef/hope-dedupe-engine:release-0.1 django-admin env --check +Ensure the following environment variables are properly configured: +- **DATABASE_URL**: The URL for the database connection. + *Example:* `postgres://hde:password@db:5432/hope_dedupe_engine` -**WARNING** -> Hope Deduplication Engine implements **security first** policy. It means that configuration default values are "almost" production compliant. -> -> Es. `DEBUG=False` or `SECURE_SSL_REDIRECT=True`. -> -> Be sure to run `./manage.py env --check` and `./manage.py env -g all` to check and display your configuration - +- **SECRET_KEY**: A secret key for the Django installation. + *Example:* `django-insecure-pretty-strong` +- **CACHE_URL**: The URL for the cache server. + *Example:* `redis://redis:6379/1` -### 1. Clone repo and install requirements - git clone https://github.com/unicef/hope-dedup-engine - pdm venv create 3.11 - pdm install - pdm venv activate in-project - pre-commit install +- **CELERY_BROKER_URL**: The URL for the Celery broker. + *Example:* `redis://redis:6379/9` -### 2. configure your environment +- **DEFAULT_ROOT**: The root directory for locally stored files. + *Example:* `/var/hope_dedupe_engine/default` -Uses `./manage.py env` to check required (and optional) variables to put +- **MEDIA_ROOT**: The root directory for media files. + *Example:* `/var/hope_dedupe_engine/media` - ./manage.py env --check +- **STATIC_ROOT**: The root directory for static files. + *Example:* `/var/hope_dedupe_engine/static` +#### Storage Configuration +The service uses the following storage backends: +- **FILE_STORAGE_DEFAULT**: This backend is used for storing locally downloaded DNN model files and encoded data. + ``` + FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage + ``` +- **FILE_STORAGE_DNN**: +This backend is dedicated to storing DNN model files. Ensure that the following two files are present in this storage: + 1. *deploy.prototxt*: Defines the model architecture. + 2. *res10_300x300_ssd_iter_140000.caffemodel*: Contains the pre-trained model weights. -### 3. Run upgrade to run migrations and initial setup + The current process involves downloading files from a [GitHub repository](https://github.com/sr6033/face-detection-with-OpenCV-and-DNN) and saving them to Azure Blob Storage **FILE_STORAGE_DNN** using command `django-admin upgrade --with-dnn-setup`, or through dedicated command `django-admin dnnsetup`. + In the future, an automated pipeline related to model training could handle file updates. - ./manage.py upgrade + The storage configuration for this backend is as follows: + ``` + FILE_STORAGE_DNN="storages.backends.azure_storage.AzureStorage?account_name=&account_key=&overwrite_files=true&azure_container=dnn" + ``` + +- **FILE_STORAGE_HOPE**: This backend is used for storing HOPE dataset images. It should be configured as read-only for the service. + ``` + FILE_STORAGE_HOPE="storages.backends.azure_storage.AzureStorage?account_name=&account_key=&azure_container=hope" + ``` +- **FILE_STORAGE_MEDIA**: This backend is used for storing media files. + +- **FILE_STORAGE_STATIC**: This backend is used for storing static files, such as CSS, JavaScript, and images. + +#### To run server and support services + + $ docker run -d -t unicef/hope-dedupe-engine:release-0.1 + $ docker run -d -t unicef/hope-dedupe-engine:release-0.1 worker + $ docker run -d -t unicef/hope-dedupe-engine:release-0.1 beat + +## Demo application + +#### To run locally demo server with the provided sample data + + $ docker compose -f tests/extras/demoapp/compose.yml up --build + +You can access the demo server admin panel at http://localhost:8000/admin/ with the following credentials: `adm@hde.org`/`123` + +#### API documentation and interaction +API documentation is available at [Swagger UI](http://localhost:8000/api/rest/swagger/) and [Redoc](http://localhost:8000/api/rest/redoc/) + +Scripts for API interaction are located in the `tests/extras/demoapp/scripts` directory. These scripts require `httpie` and `jq` to be installed. + +For more information, refer to the [demoapp README](tests/extras/demoapp/scripts/README.md) + +## Development + +To develop the service locally, use the provided `compose.yml` file. This will start the service and all necessary dependencies. + + $ docker compose up --build + +To run the tests, use: + + $ docker compose run --rm backend pytest tests -v --create-db + +After running the tests, you can view the coverage report at the `~build/coverage` directory. + + +## Help +**Got a question?** We got answers. + +File a GitHub [issue](https://github.com/unicef/hope-dedup-engine/issues) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..c238822c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security + +## Reporting Security Issues + +If you've found a security issue in HDE, you can submit your report to hope-security[@]unicef.org via email. + +Please include as much information as possible in your report to better help us understand and resolve the issue: + +- Where the security issue exists (ie. HDE Core, API subsystem, etc.) +- The type of issue (ex. SQL injection, cross-site scripting, missing authorization, etc.) +- Full paths or links to the source files where the security issue exists, if possible +- Any special configuration required to reproduce the issue +- Step-by-step instructions to reproduce the issue +- Proof of concept or exploit code, if available + +If you need to encrypt sensitive information sent to us, please use our [PGP key](https://keys.openpgp.org/vks/v1/by-fingerprint/F72BF087F3A94FE4A305CE449061F6AC06E40F32): + +F72B F087 F3A9 4FE4 A305 CE44 9061 F6AC 06E4 0F32 diff --git a/compose.yml b/compose.yml index 5d6fc8e5..7b3c81a3 100644 --- a/compose.yml +++ b/compose.yml @@ -2,52 +2,74 @@ x-common: &common build: context: . dockerfile: docker/Dockerfile - target: dev - env_file: - - .env + target: python_dev_deps + platform: linux/amd64 + environment: + - ADMIN_EMAIL=adm@hde.org + - ADMIN_PASSWORD=123 + - ALLOWED_HOSTS=localhost,127.0.0.1 + - CACHE_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/9 + - CELERY_TASK_ALWAYS_EAGER=False + - CSRF_COOKIE_SECURE=False + - DATABASE_URL=postgres://hde:password@db:5432/hope_dedupe_engine + - DEFAULT_ROOT=/var/hope_dedupe_engine/default + - DJANGO_SETTINGS_MODULE=hope_dedup_engine.config.settings + - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage + - FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - MEDIA_ROOT=/var/hope_dedupe_engine/media + - PYTHONPATH=/code/src/:/code/__pypackages__/3.12/lib/ + - SECRET_KEY=very-secret-key + - SECURE_SSL_REDIRECT=False + - SESSION_COOKIE_DOMAIN= + - SESSION_COOKIE_SECURE=False + - SOCIAL_AUTH_REDIRECT_IS_HTTPS=False + - STATIC_ROOT=/var/hope_dedupe_engine/static volumes: - .:/code - /var/run/docker.sock:/var/run/docker.sock - restart: unless-stopped + restart: always depends_on: db: condition: service_healthy redis: condition: service_healthy + services: backend: <<: *common ports: - 8000:8000 - command: ["docker-entrypoint.sh", "dev"] + # command: ["tail", "-f", "/dev/null"] + command: > + /bin/sh -c " + django-admin demo --skip-checks && + django-admin upgrade && + django-admin runserver 0.0.0.0:8000 + " healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/healthcheck"] interval: 10s timeout: 5s retries: 5 - celery_worker: - <<: *common - command: ["docker-entrypoint.sh", "worker"] - - celery_beat: - <<: *common - command: ["docker-entrypoint.sh", "beat"] - db: - image: postgres:15 + image: postgres:16 environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=postgres + - POSTGRES_USER=hde + - POSTGRES_PASSWORD=password + - POSTGRES_DB=hope_dedupe_engine volumes: - postgres_data:/var/lib/postgresql/data/ ports: - 5432:5432 restart: always healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres"] + test: ["CMD", "pg_isready", "-U", "hde", "-d", "hope_dedupe_engine"] start_period: 5s start_interval: 1s interval: 5s @@ -69,13 +91,40 @@ services: azurite: image: mcr.microsoft.com/azure-storage/azurite - command: "azurite -l /workspace -d /workspace/debug.log --blobPort 10000 --blobHost 0.0.0.0 --loose" + command: "azurite -l /workspace -d /workspace/debug.log --blobPort 10000 --blobHost 0.0.0.0 --loose --silent" restart: always ports: - "10000:10000" # Blob service volumes: - azurite_data:/workspace + celery-worker: + <<: *common + # entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""] + # command: worker + command: > + sh -c ' + mkdir -p /var/hope_dedupe_engine/default && + chown -R user:app /var/hope_dedupe_engine && + gosu user:app django-admin syncdnn && + gosu user:app celery -A hope_dedup_engine.config.celery worker -E --loglevel=WARNING --concurrency=4 + ' + + celery-beat: + <<: *common + entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""] + command: beat + + celery-flower: + <<: *common + ports: + - 5555:5555 + command: > + sh -c " + exec celery -A hope_dedup_engine.config.celery flower --address=0.0.0.0 + " + + volumes: - postgres_data: azurite_data: + postgres_data: diff --git a/docker/Dockerfile b/docker/Dockerfile index cfce73b8..45ca5e6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -121,6 +121,8 @@ ARG VERSION ENV VERSION=$VERSION ARG BUILD_DATE ENV BUILD_DATE=$BUILD_DATE +ARG DISTRO +ENV DISTRO=$DISTRO ARG SOURCE_COMMIT ENV SOURCE_COMMIT=$SOURCE_COMMIT ARG GITHUB_SERVER_URL @@ -132,7 +134,7 @@ ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY LABEL date=$BUILD_DATE LABEL version=$VERSION LABEL checksum=$CHECKSUM -LABEL distro="builder-test" +LABEL distro="test" #COPY pyproject.toml pdm.lock ./ #COPY docker/conf/config.toml /etc/xdg/pdm/config.toml @@ -145,7 +147,13 @@ RUN set -x \ && pdm sync --no-editable -v --no-self RUN < /RELEASE -{"version": "$VERSION", "commit": "$SOURCE_COMMIT", "date": "$BUILD_DATE", "checksum": "$CHECKSUM", "source": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/"} +{"version": "$VERSION", + "commit": "$SOURCE_COMMIT", + "date": "$BUILD_DATE", + "distro": "test", + "checksum": "$CHECKSUM", + "source": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" +} EOF FROM build_deps AS python_prod_deps @@ -199,11 +207,20 @@ ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY WORKDIR /code -COPY --chown=user:app --from=python_prod_deps /code /code +COPY --chown=user:app --from=python_prod_deps /code/__pypackages__ /code/__pypackages__ +COPY --chown=user:app --from=python_prod_deps /code/README.md /code/LICENSE / + +ENV PATH=${APATH}:${PATH} \ + PYTHONPATH=${APYTHONPATH} \ + PYTHONDBUFFERED=1 \ + PYTHONDONTWRITEBYTCODE=1 \ + DJANGO_SETTINGS_MODULE="hope_dedup_engine.config.settings" + RUN < /RELEASE {"version": "$VERSION", "commit": "$SOURCE_COMMIT", "date": "$BUILD_DATE", + "distro": "dist", "checksum": "$CHECKSUM", "source": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" } @@ -211,19 +228,18 @@ EOF VOLUME /var/run/app/ EXPOSE 8000 -ENTRYPOINT exec docker-entrypoint.sh "$0" "$@" CMD ["run"] +ENTRYPOINT exec docker-entrypoint.sh "$0" "$@" LABEL distro="final" -LABEL maintainer="hope@app.io" -LABEL org.opencontainers.image.authors="author@app.io" +LABEL maintainer="hope@unicef.org" +LABEL org.opencontainers.image.authors="hope@unicef.org" LABEL org.opencontainers.image.created="$BUILD_DATE" -LABEL org.opencontainers.image.description="App runtime image" -LABEL org.opencontainers.image.documentation="https://github.com/saxix/trash" +LABEL org.opencontainers.image.description="HOPE Deduplication Engine" +LABEL org.opencontainers.image.documentation="https://github.com/unicef/hope-dedup-engine" LABEL org.opencontainers.image.licenses="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_COMMIT:-master}/LICENSE" LABEL org.opencontainers.image.revision=$SOURCE_COMMIT LABEL org.opencontainers.image.source="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" LABEL org.opencontainers.image.title="Hope Deduplication Engine" LABEL org.opencontainers.image.version="$VERSION" -#LABEL org.opencontainers.image.url="https://app.io/" -#LABEL org.opencontainers.image.vendor="App ltd" +LABEL org.opencontainers.image.vendor="UNICEF" diff --git a/docker/bin/docker-entrypoint.sh b/docker/bin/docker-entrypoint.sh index 7c67a60f..3ede941b 100755 --- a/docker/bin/docker-entrypoint.sh +++ b/docker/bin/docker-entrypoint.sh @@ -1,56 +1,46 @@ -#!/bin/sh -e - +#!/bin/sh export MEDIA_ROOT="${MEDIA_ROOT:-/var/run/app/media}" export STATIC_ROOT="${STATIC_ROOT:-/var/run/app/static}" +export DEFAULT_ROOT="${DEFAULT_ROOT:-/var/run/app/default}" export UWSGI_PROCESSES="${UWSGI_PROCESSES:-"4"}" -mkdir -p "${MEDIA_ROOT}" "${STATIC_ROOT}" || echo "Cannot create dirs ${MEDIA_ROOT} ${STATIC_ROOT}" +export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-"hope_dedup_engine.config.settings"}" +mkdir -p "${MEDIA_ROOT}" "${STATIC_ROOT}" "${DEFAULT_ROOT}" || echo "Cannot create dirs ${MEDIA_ROOT} ${STATIC_ROOT} ${DEFAULT_ROOT}" + +if [ -d "${STATIC_ROOT}" ];then + chown -R user:app ${STATIC_ROOT} +fi + +if [ -d "${DEFAULT_ROOT}" ];then + chown -R user:app ${DEFAULT_ROOT} +fi + + +echo "MEDIA_ROOT ${MEDIA_ROOT}" +echo "STATIC_ROOT ${STATIC_ROOT}" +echo "DEFAULT_ROOT ${DEFAULT_ROOT}" +echo "Docker run command: $1" case "$1" in + setup) + django-admin check --deploy || exit 1 + django-admin upgrade --no-static || exit 1 + exit 0 + ;; + worker) + gosu user:app django-admin syncdnn || exit 1 + set -- tini -- "$@" + set -- gosu user:app celery -A hope_dedup_engine.config.celery worker -E --loglevel=ERROR --concurrency=4 + ;; + beat) + set -- tini -- "$@" + set -- gosu user:app celery -A hope_dedup_engine.config.celery beat --loglevel=ERROR --scheduler django_celery_beat.schedulers:DatabaseScheduler + ;; run) + django-admin check --deploy || exit 1 set -- tini -- "$@" set -- gosu user:app uwsgi --ini /conf/uwsgi.ini ;; esac exec "$@" - -# -#case "$1" in -# run) -# if [ "$INIT_RUN_CHECK" = "1" ];then -# echo "Running Django checks..." -# django-admin check --deploy -# fi -# OPTS="--no-check -v 1" -# if [ "$INIT_RUN_UPGRADE" = "1" ];then -# if [ "$INIT_RUN_COLLECTSTATIC" != "1" ];then -# OPTS="$OPTS --no-static" -# fi -# if [ "$INIT_RUN_MIGRATATIONS" != "1" ];then -# OPTS="$OPTS --no-migrate" -# fi -# echo "Running 'upgrade $OPTS'" -# django-admin upgrade $OPTS -# fi -# set -- tini -- "$@" -# echo "Starting uwsgi..." -# exec uwsgi --ini /conf/uwsgi.ini -# ;; -# worker) -# exec celery -A hope_dedup_engine.celery worker -E --loglevel=ERROR --concurrency=4 -# ;; -# beat) -# exec celery -A hope_dedup_engine.celery beat -E --loglevel=ERROR ---scheduler django_celery_beat.schedulers:DatabaseScheduler -# ;; -# dev) -# until pg_isready -h db -p 5432; -# do echo "waiting for database"; sleep 2; done; -# django-admin collectstatic --no-input -# django-admin migrate -# django-admin runserver 0.0.0.0:8000 -# ;; -# *) -# exec "$@" -# ;; -#esac diff --git a/docker/bin/release-info.sh b/docker/bin/release-info.sh index caf4d5f8..938c3bde 100755 --- a/docker/bin/release-info.sh +++ b/docker/bin/release-info.sh @@ -1,5 +1,5 @@ #!/bin/bash cat /RELEASE -uwsgi --version -django-admin --version +exho "uwsgi `uwsgi --version`" +echo "Django `django-admin --version`" diff --git a/docker/conf/uwsgi.ini b/docker/conf/uwsgi.ini index 919740de..5eb7327a 100644 --- a/docker/conf/uwsgi.ini +++ b/docker/conf/uwsgi.ini @@ -3,17 +3,20 @@ http=0.0.0.0:8000 enable-threads=0 honour-range=1 master=1 -module=trash.wsgi +module=hope_dedup_engine.config.wsgi:application processes=$(UWSGI_PROCESSES) ;virtualenv=/code/.venv/ ;virtualenv=%(_) ;venv=%(_) ;chdir=code/ -username = user -gropuname = app +;username = user +;gropuname = app ;offload-threads=%k ;static-gzip-all=true -route = /static/(.*) static:$(STATIC_ROOT)/$1 +;route = /static/(.*) static:$(STATIC_ROOT)/$1 +static-map = /static=$(STATIC_ROOT) http-keepalive = 1 -collect-header=Content-Type RESPONSE_CONTENT_TYPE -mimefile=/etc/mime.types +;collect-header=Content-Type RESPONSE_CONTENT_TYPE +mimefile=/conf/mime.types +buffer-size = 8192 +http-buffer-size = 8192 diff --git a/pdm.lock b/pdm.lock index f9eba18d..ef4cdf81 100644 --- a/pdm.lock +++ b/pdm.lock @@ -3,9 +3,12 @@ [metadata] groups = ["default", "dev"] -strategy = ["cross_platform", "inherit_metadata"] -lock_version = "4.4.1" -content_hash = "sha256:07d1e2f3b569e6b60d3825c075de0d525c36024c441e7d4fcbace92bb8cd931b" +strategy = ["inherit_metadata"] +lock_version = "4.5.0" +content_hash = "sha256:e929a0f5bde349def56aecf6e2ae3ca3ef6227aa8be9b84f37c520c5af489ed2" + +[[metadata.targets]] +requires_python = ">=3.12" [[package]] name = "amqp" @@ -27,6 +30,9 @@ version = "3.8.1" requires_python = ">=3.8" summary = "ASGI specs, helper code, and adapters" groups = ["default"] +dependencies = [ + "typing-extensions>=4; python_version < \"3.11\"", +] files = [ {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"}, {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"}, @@ -40,6 +46,7 @@ groups = ["dev"] marker = "python_version >= \"3.11\"" dependencies = [ "six>=1.12.0", + "typing; python_version < \"3.5\"", ] files = [ {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, @@ -52,6 +59,9 @@ version = "23.2.0" requires_python = ">=3.7" summary = "Classes Without Boilerplate" groups = ["default", "dev"] +dependencies = [ + "importlib-metadata; python_version < \"3.8\"", +] files = [ {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, @@ -59,8 +69,8 @@ files = [ [[package]] name = "azure-core" -version = "1.30.1" -requires_python = ">=3.7" +version = "1.30.2" +requires_python = ">=3.8" summary = "Microsoft Azure Core Library for Python" groups = ["default"] dependencies = [ @@ -69,13 +79,13 @@ dependencies = [ "typing-extensions>=4.6.0", ] files = [ - {file = "azure-core-1.30.1.tar.gz", hash = "sha256:26273a254131f84269e8ea4464f3560c731f29c0c1f69ac99010845f239c1a8f"}, - {file = "azure_core-1.30.1-py3-none-any.whl", hash = "sha256:7c5ee397e48f281ec4dd773d67a0a47a0962ed6fa833036057f9ea067f688e74"}, + {file = "azure-core-1.30.2.tar.gz", hash = "sha256:a14dc210efcd608821aa472d9fb8e8d035d29b68993819147bc290a8ac224472"}, + {file = "azure_core-1.30.2-py3-none-any.whl", hash = "sha256:cf019c1ca832e96274ae85abd3d9f752397194d9fea3b41487290562ac8abe4a"}, ] [[package]] name = "azure-storage-blob" -version = "12.20.0" +version = "12.21.0" requires_python = ">=3.8" summary = "Microsoft Azure Blob Storage Client Library for Python" groups = ["default"] @@ -86,8 +96,8 @@ dependencies = [ "typing-extensions>=4.6.0", ] files = [ - {file = "azure-storage-blob-12.20.0.tar.gz", hash = "sha256:eeb91256e41d4b5b9bad6a87fd0a8ade07dd58aa52344e2c8d2746e27a017d3b"}, - {file = "azure_storage_blob-12.20.0-py3-none-any.whl", hash = "sha256:de6b3bf3a90e9341a6bcb96a2ebe981dffff993e9045818f6549afea827a52a9"}, + {file = "azure-storage-blob-12.21.0.tar.gz", hash = "sha256:b9722725072f5b7373c0f4dd6d78fbae2bb37bffc5c3e01731ab8c750ee8dd7e"}, + {file = "azure_storage_blob-12.21.0-py3-none-any.whl", hash = "sha256:f9ede187dd5a0ef296b583a7c1861c6938ddd6708d6e70f4203a163c2ab42d43"}, ] [[package]] @@ -127,6 +137,8 @@ dependencies = [ "packaging>=22.0", "pathspec>=0.9.0", "platformdirs>=2", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions>=4.0.1; python_version < \"3.11\"", ] files = [ {file = "black-24.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d"}, @@ -144,11 +156,13 @@ requires_python = ">=3.8" summary = "Distributed Task Queue." groups = ["default", "dev"] dependencies = [ + "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", "billiard<5.0,>=4.2.0", "click-didyoumean>=0.3.0", "click-plugins>=1.1.1", "click-repl>=0.2.0", "click<9.0,>=8.1.2", + "importlib-metadata>=3.6; python_version < \"3.8\"", "kombu<6.0,>=5.3.4", "python-dateutil>=2.8.2", "tzdata>=2022.7", @@ -177,13 +191,13 @@ files = [ [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.7.4" requires_python = ">=3.6" summary = "Python package for providing Mozilla's CA Bundle." groups = ["default", "dev"] files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] @@ -255,6 +269,7 @@ summary = "Composable command line interface toolkit" groups = ["default", "dev"] dependencies = [ "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -317,48 +332,47 @@ files = [ [[package]] name = "coverage" -version = "7.5.3" +version = "7.6.0" requires_python = ">=3.8" summary = "Code coverage measurement for Python" groups = ["dev"] files = [ - {file = "coverage-7.5.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:296a7d9bbc598e8744c00f7a6cecf1da9b30ae9ad51c566291ff1314e6cbbed8"}, - {file = "coverage-7.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:34d6d21d8795a97b14d503dcaf74226ae51eb1f2bd41015d3ef332a24d0a17b3"}, - {file = "coverage-7.5.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e317953bb4c074c06c798a11dbdd2cf9979dbcaa8ccc0fa4701d80042d4ebf1"}, - {file = "coverage-7.5.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:705f3d7c2b098c40f5b81790a5fedb274113373d4d1a69e65f8b68b0cc26f6db"}, - {file = "coverage-7.5.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1196e13c45e327d6cd0b6e471530a1882f1017eb83c6229fc613cd1a11b53cd"}, - {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:015eddc5ccd5364dcb902eaecf9515636806fa1e0d5bef5769d06d0f31b54523"}, - {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:fd27d8b49e574e50caa65196d908f80e4dff64d7e592d0c59788b45aad7e8b35"}, - {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:33fc65740267222fc02975c061eb7167185fef4cc8f2770267ee8bf7d6a42f84"}, - {file = "coverage-7.5.3-cp312-cp312-win32.whl", hash = "sha256:7b2a19e13dfb5c8e145c7a6ea959485ee8e2204699903c88c7d25283584bfc08"}, - {file = "coverage-7.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:0bbddc54bbacfc09b3edaec644d4ac90c08ee8ed4844b0f86227dcda2d428fcb"}, - {file = "coverage-7.5.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:3538d8fb1ee9bdd2e2692b3b18c22bb1c19ffbefd06880f5ac496e42d7bb3884"}, - {file = "coverage-7.5.3.tar.gz", hash = "sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f"}, + {file = "coverage-7.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b"}, + {file = "coverage-7.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605"}, + {file = "coverage-7.6.0-cp312-cp312-win32.whl", hash = "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da"}, + {file = "coverage-7.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67"}, + {file = "coverage-7.6.0.tar.gz", hash = "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51"}, ] [[package]] name = "coverage" -version = "7.5.3" +version = "7.6.0" extras = ["toml"] requires_python = ">=3.8" summary = "Code coverage measurement for Python" groups = ["dev"] dependencies = [ - "coverage==7.5.3", + "coverage==7.6.0", + "tomli; python_full_version <= \"3.11.0a6\"", ] files = [ - {file = "coverage-7.5.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:296a7d9bbc598e8744c00f7a6cecf1da9b30ae9ad51c566291ff1314e6cbbed8"}, - {file = "coverage-7.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:34d6d21d8795a97b14d503dcaf74226ae51eb1f2bd41015d3ef332a24d0a17b3"}, - {file = "coverage-7.5.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e317953bb4c074c06c798a11dbdd2cf9979dbcaa8ccc0fa4701d80042d4ebf1"}, - {file = "coverage-7.5.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:705f3d7c2b098c40f5b81790a5fedb274113373d4d1a69e65f8b68b0cc26f6db"}, - {file = "coverage-7.5.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1196e13c45e327d6cd0b6e471530a1882f1017eb83c6229fc613cd1a11b53cd"}, - {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:015eddc5ccd5364dcb902eaecf9515636806fa1e0d5bef5769d06d0f31b54523"}, - {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:fd27d8b49e574e50caa65196d908f80e4dff64d7e592d0c59788b45aad7e8b35"}, - {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:33fc65740267222fc02975c061eb7167185fef4cc8f2770267ee8bf7d6a42f84"}, - {file = "coverage-7.5.3-cp312-cp312-win32.whl", hash = "sha256:7b2a19e13dfb5c8e145c7a6ea959485ee8e2204699903c88c7d25283584bfc08"}, - {file = "coverage-7.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:0bbddc54bbacfc09b3edaec644d4ac90c08ee8ed4844b0f86227dcda2d428fcb"}, - {file = "coverage-7.5.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:3538d8fb1ee9bdd2e2692b3b18c22bb1c19ffbefd06880f5ac496e42d7bb3884"}, - {file = "coverage-7.5.3.tar.gz", hash = "sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f"}, + {file = "coverage-7.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b"}, + {file = "coverage-7.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605"}, + {file = "coverage-7.6.0-cp312-cp312-win32.whl", hash = "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da"}, + {file = "coverage-7.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67"}, + {file = "coverage-7.6.0.tar.gz", hash = "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51"}, ] [[package]] @@ -373,7 +387,7 @@ files = [ [[package]] name = "cryptography" -version = "42.0.7" +version = "43.0.0" requires_python = ">=3.7" summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." groups = ["default", "dev"] @@ -381,53 +395,40 @@ dependencies = [ "cffi>=1.12; platform_python_implementation != \"PyPy\"", ] files = [ - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, - {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, - {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, - {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, - {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, - {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, - {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, + {file = "cryptography-43.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431"}, + {file = "cryptography-43.0.0-cp37-abi3-win32.whl", hash = "sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc"}, + {file = "cryptography-43.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778"}, + {file = "cryptography-43.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b"}, + {file = "cryptography-43.0.0-cp39-abi3-win32.whl", hash = "sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf"}, + {file = "cryptography-43.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709"}, + {file = "cryptography-43.0.0.tar.gz", hash = "sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e"}, ] [[package]] name = "debugpy" -version = "1.8.1" +version = "1.8.2" requires_python = ">=3.8" summary = "An implementation of the Debug Adapter Protocol for Python" groups = ["dev"] files = [ - {file = "debugpy-1.8.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3ebb70ba1a6524d19fa7bb122f44b74170c447d5746a503e36adc244a20ac539"}, - {file = "debugpy-1.8.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2e658a9630f27534e63922ebf655a6ab60c370f4d2fc5c02a5b19baf4410ace"}, - {file = "debugpy-1.8.1-cp312-cp312-win32.whl", hash = "sha256:caad2846e21188797a1f17fc09c31b84c7c3c23baf2516fed5b40b378515bbf0"}, - {file = "debugpy-1.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:edcc9f58ec0fd121a25bc950d4578df47428d72e1a0d66c07403b04eb93bcf98"}, - {file = "debugpy-1.8.1-py2.py3-none-any.whl", hash = "sha256:28acbe2241222b87e255260c76741e1fbf04fdc3b6d094fcf57b6c6f75ce1242"}, - {file = "debugpy-1.8.1.zip", hash = "sha256:f696d6be15be87aef621917585f9bb94b1dc9e8aced570db1b8a6fc14e8f9b42"}, + {file = "debugpy-1.8.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:5d3ccd39e4021f2eb86b8d748a96c766058b39443c1f18b2dc52c10ac2757835"}, + {file = "debugpy-1.8.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62658aefe289598680193ff655ff3940e2a601765259b123dc7f89c0239b8cd3"}, + {file = "debugpy-1.8.2-cp312-cp312-win32.whl", hash = "sha256:bd11fe35d6fd3431f1546d94121322c0ac572e1bfb1f6be0e9b8655fb4ea941e"}, + {file = "debugpy-1.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:15bc2f4b0f5e99bf86c162c91a74c0631dbd9cef3c6a1d1329c946586255e859"}, + {file = "debugpy-1.8.2-py2.py3-none-any.whl", hash = "sha256:16e16df3a98a35c63c3ab1e4d19be4cbc7fdda92d9ddc059294f18910928e0ca"}, + {file = "debugpy-1.8.2.zip", hash = "sha256:95378ed08ed2089221896b9b3a8d021e642c24edc8fef20e5d4342ca8be65c00"}, ] [[package]] @@ -436,6 +437,7 @@ version = "5.1.1" requires_python = ">=3.5" summary = "Decorators for Humans" groups = ["dev"] +marker = "python_version >= \"3.11\"" files = [ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, @@ -464,7 +466,7 @@ files = [ [[package]] name = "django" -version = "5.0.6" +version = "5.0.7" requires_python = ">=3.10" summary = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." groups = ["default"] @@ -474,8 +476,8 @@ dependencies = [ "tzdata; sys_platform == \"win32\"", ] files = [ - {file = "Django-5.0.6-py3-none-any.whl", hash = "sha256:8363ac062bb4ef7c3f12d078f6fa5d154031d129a15170a1066412af49d30905"}, - {file = "Django-5.0.6.tar.gz", hash = "sha256:ff1b61005004e476e0aeea47c7f79b85864c70124030e95146315396f1e7951f"}, + {file = "Django-5.0.7-py3-none-any.whl", hash = "sha256:f216510ace3de5de01329463a315a629f33480e893a9024fc93d8c32c22913da"}, + {file = "Django-5.0.7.tar.gz", hash = "sha256:bd4505cae0b9bd642313e8fb71810893df5dc2ffcacaa67a33af2d5cd61888f2"}, ] [[package]] @@ -517,9 +519,11 @@ summary = "Database-backed Periodic Tasks." groups = ["default"] dependencies = [ "Django<5.1,>=2.2", + "backports-zoneinfo; python_version < \"3.9\"", "celery<6.0,>=5.2.3", "cron-descriptor>=1.2.32", "django-timezone-field>=5.0", + "importlib-metadata<5.0; python_version < \"3.8\"", "python-crontab>=2.3.4", "tzdata", ] @@ -593,7 +597,7 @@ files = [ [[package]] name = "django-debug-toolbar" -version = "4.4.2" +version = "4.4.6" requires_python = ">=3.8" summary = "A configurable set of panels that display various debug information about the current request/response." groups = ["default"] @@ -602,8 +606,8 @@ dependencies = [ "sqlparse>=0.2", ] files = [ - {file = "django_debug_toolbar-4.4.2-py3-none-any.whl", hash = "sha256:5d7afb2ea5f8730241e5b0735396e16cd1fd8c6b53a2f3e1e30bbab9abb23728"}, - {file = "django_debug_toolbar-4.4.2.tar.gz", hash = "sha256:9204050fcb1e4f74216c5b024bc76081451926a6303993d6c513f5e142675927"}, + {file = "django_debug_toolbar-4.4.6-py3-none-any.whl", hash = "sha256:3beb671c9ec44ffb817fad2780667f172bd1c067dbcabad6268ce39a81335f45"}, + {file = "django_debug_toolbar-4.4.6.tar.gz", hash = "sha256:36e421cb908c2f0675e07f9f41e3d1d8618dc386392ec82d23bcfcd5d29c7044"}, ] [[package]] @@ -682,9 +686,23 @@ files = [ {file = "django-regex-0.5.0.tar.gz", hash = "sha256:6af1add11ae5232f133a42754c9291f9113996b1294b048305d9f1a427bca27c"}, ] +[[package]] +name = "django-smart-env" +version = "0.1.0" +requires_python = ">=3.12" +summary = "Add your description here" +groups = ["default"] +dependencies = [ + "django-environ>=0.11.2", +] +files = [ + {file = "django_smart_env-0.1.0-py3-none-any.whl", hash = "sha256:ffcbc03ab2b28808d1ac80b5165543549396dde4a24107e969a9635ba9321849"}, + {file = "django_smart_env-0.1.0.tar.gz", hash = "sha256:09ef06a2ae9223c68ba893dae2b6188938f41e464cb38e4714c341950fc1caf3"}, +] + [[package]] name = "django-storages" -version = "1.14.3" +version = "1.14.4" requires_python = ">=3.7" summary = "Support for many storage backends in Django" groups = ["default"] @@ -692,13 +710,13 @@ dependencies = [ "Django>=3.2", ] files = [ - {file = "django-storages-1.14.3.tar.gz", hash = "sha256:95a12836cd998d4c7a4512347322331c662d9114c4344f932f5e9c0fce000608"}, - {file = "django_storages-1.14.3-py3-none-any.whl", hash = "sha256:31f263389e95ce3a1b902fb5f739a7ed32895f7d8b80179fe7453ecc0dfe102e"}, + {file = "django-storages-1.14.4.tar.gz", hash = "sha256:69aca94d26e6714d14ad63f33d13619e697508ee33ede184e462ed766dc2a73f"}, + {file = "django_storages-1.14.4-py3-none-any.whl", hash = "sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3"}, ] [[package]] name = "django-storages" -version = "1.14.3" +version = "1.14.4" extras = ["azure"] requires_python = ">=3.7" summary = "Support for many storage backends in Django" @@ -706,25 +724,26 @@ groups = ["default"] dependencies = [ "azure-core>=1.13", "azure-storage-blob>=12", - "django-storages==1.14.3", + "django-storages==1.14.4", ] files = [ - {file = "django-storages-1.14.3.tar.gz", hash = "sha256:95a12836cd998d4c7a4512347322331c662d9114c4344f932f5e9c0fce000608"}, - {file = "django_storages-1.14.3-py3-none-any.whl", hash = "sha256:31f263389e95ce3a1b902fb5f739a7ed32895f7d8b80179fe7453ecc0dfe102e"}, + {file = "django-storages-1.14.4.tar.gz", hash = "sha256:69aca94d26e6714d14ad63f33d13619e697508ee33ede184e462ed766dc2a73f"}, + {file = "django_storages-1.14.4-py3-none-any.whl", hash = "sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3"}, ] [[package]] name = "django-timezone-field" -version = "6.1.0" -requires_python = ">=3.8,<4.0" +version = "7.0" +requires_python = "<4.0,>=3.8" summary = "A Django app providing DB, form, and REST framework fields for zoneinfo and pytz timezone objects." groups = ["default"] dependencies = [ "Django<6.0,>=3.2", + "backports-zoneinfo<0.3.0,>=0.2.1; python_version < \"3.9\"", ] files = [ - {file = "django_timezone_field-6.1.0-py3-none-any.whl", hash = "sha256:0095f43da716552fcc606783cfb42cb025892514f1ec660ebfa96186eb83b74c"}, - {file = "django_timezone_field-6.1.0.tar.gz", hash = "sha256:d40f7059d7bae4075725d04a9dae601af9fe3c7f0119a69b0e2c6194a782f797"}, + {file = "django_timezone_field-7.0-py3-none-any.whl", hash = "sha256:3232e7ecde66ba4464abb6f9e6b8cc739b914efb9b29dc2cf2eee451f7cc2acb"}, + {file = "django_timezone_field-7.0.tar.gz", hash = "sha256:aa6f4965838484317b7f08d22c0d91a53d64e7bbbd34264468ae83d4023898a7"}, ] [[package]] @@ -742,16 +761,17 @@ files = [ [[package]] name = "djangorestframework" -version = "3.15.1" -requires_python = ">=3.6" +version = "3.15.2" +requires_python = ">=3.8" summary = "Web APIs for Django, made easy." groups = ["default"] dependencies = [ - "django>=3.0", + "backports-zoneinfo; python_version < \"3.9\"", + "django>=4.2", ] files = [ - {file = "djangorestframework-3.15.1-py3-none-any.whl", hash = "sha256:3ccc0475bce968608cf30d07fb17d8e52d1d7fc8bfe779c905463200750cbca6"}, - {file = "djangorestframework-3.15.1.tar.gz", hash = "sha256:f88fad74183dfc7144b2756d0d2ac716ea5b4c7c9840995ac3bfd8ec034333c1"}, + {file = "djangorestframework-3.15.2-py3-none-any.whl", hash = "sha256:2b8871b062ba1aefc2de01f773875441a961fefbf79f5eed1e32b2f096944b20"}, + {file = "djangorestframework-3.15.2.tar.gz", hash = "sha256:36fe88cd2d6c6bec23dca9804bab2ba5517a8bb9d8f47ebc68981b56840107ad"}, ] [[package]] @@ -806,6 +826,7 @@ dependencies = [ "djangorestframework>=3.10.3", "inflection>=0.3.1", "jsonschema>=2.6.0", + "typing-extensions; python_version < \"3.10\"", "uritemplate>=2.0.0", ] files = [ @@ -815,7 +836,7 @@ files = [ [[package]] name = "drf-spectacular-sidecar" -version = "2024.5.1" +version = "2024.7.1" requires_python = ">=3.6" summary = "Serve self-contained distribution builds of Swagger UI and Redoc with Django" groups = ["default"] @@ -823,8 +844,8 @@ dependencies = [ "Django>=2.2", ] files = [ - {file = "drf_spectacular_sidecar-2024.5.1-py3-none-any.whl", hash = "sha256:089fdef46b520b7b1c8a497a398cde9336c3f20b115835baeb158dc4138d743d"}, - {file = "drf_spectacular_sidecar-2024.5.1.tar.gz", hash = "sha256:1ecfbe86174461e3cf78a9cd49f69aa8d9e0710cb5e8b35107d3f8cc0f380c21"}, + {file = "drf_spectacular_sidecar-2024.7.1-py3-none-any.whl", hash = "sha256:5dc8b38ad153e90b328152674c7959bf114bf86360a617a5a4516e135cb832bc"}, + {file = "drf_spectacular_sidecar-2024.7.1.tar.gz", hash = "sha256:beb992d6ece806a2d422ad626983e2472c0a5550de9647a7ed6764716a5abdfe"}, ] [[package]] @@ -911,6 +932,7 @@ summary = "A versatile test fixtures replacement based on thoughtbot's factory_b groups = ["dev"] dependencies = [ "Faker>=0.7.0", + "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "factory_boy-3.3.0-py2.py3-none-any.whl", hash = "sha256:a2cdbdb63228177aa4f1c52f4b6d83fab2b8623bf602c7dedd7eb83c0f69c04c"}, @@ -919,7 +941,7 @@ files = [ [[package]] name = "faker" -version = "25.3.0" +version = "26.0.0" requires_python = ">=3.8" summary = "Faker is a Python package that generates fake data for you." groups = ["dev"] @@ -927,35 +949,49 @@ dependencies = [ "python-dateutil>=2.4", ] files = [ - {file = "Faker-25.3.0-py3-none-any.whl", hash = "sha256:0158d47e955b6ec22134c0a74ebb7ed34fe600896208bafbf1008db831b17f04"}, - {file = "Faker-25.3.0.tar.gz", hash = "sha256:bcbe31eee5ef4bbf87ce36c4eba53c01e2a1d912fde2a4d3528b430d2beb784f"}, + {file = "Faker-26.0.0-py3-none-any.whl", hash = "sha256:886ee28219be96949cd21ecc96c4c742ee1680e77f687b095202c8def1a08f06"}, + {file = "Faker-26.0.0.tar.gz", hash = "sha256:0f60978314973de02c00474c2ae899785a42b2cf4f41b7987e93c132a2b8a4a9"}, +] + +[[package]] +name = "fancycompleter" +version = "0.9.1" +summary = "colorful TAB completion for Python prompt" +groups = ["dev"] +dependencies = [ + "pyreadline; platform_system == \"Windows\"", + "pyrepl>=0.8.2", +] +files = [ + {file = "fancycompleter-0.9.1-py3-none-any.whl", hash = "sha256:dd076bca7d9d524cc7f25ec8f35ef95388ffef9ef46def4d3d25e9b044ad7080"}, + {file = "fancycompleter-0.9.1.tar.gz", hash = "sha256:09e0feb8ae242abdfd7ef2ba55069a46f011814a80fe5476be48f51b00247272"}, ] [[package]] name = "filelock" -version = "3.14.0" +version = "3.15.4" requires_python = ">=3.8" summary = "A platform independent file lock." groups = ["dev"] files = [ - {file = "filelock-3.14.0-py3-none-any.whl", hash = "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f"}, - {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, ] [[package]] name = "flake8" -version = "7.0.0" +version = "7.1.0" requires_python = ">=3.8.1" summary = "the modular source code checker: pep8 pyflakes and co" groups = ["dev"] dependencies = [ "mccabe<0.8.0,>=0.7.0", - "pycodestyle<2.12.0,>=2.11.0", + "pycodestyle<2.13.0,>=2.12.0", "pyflakes<3.3.0,>=3.2.0", ] files = [ - {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, - {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, + {file = "flake8-7.1.0-py2.py3-none-any.whl", hash = "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a"}, + {file = "flake8-7.1.0.tar.gz", hash = "sha256:48a07b626b55236e0fb4784ee69a465fbf59d79eec1f5b4785c3d3bc57d17aa5"}, ] [[package]] @@ -994,6 +1030,7 @@ summary = "Generate HTML reports of flake8 violations" groups = ["dev"] dependencies = [ "flake8>=3.3.0", + "importlib-metadata; python_version < \"3.8\"", "jinja2>=3.1.0", "pygments>=2.2.0", ] @@ -1002,6 +1039,24 @@ files = [ {file = "flake8_html-0.4.3-py2.py3-none-any.whl", hash = "sha256:8f126748b1b0edd6cd39e87c6192df56e2f8655b0aa2bb00ffeac8cf27be4325"}, ] +[[package]] +name = "flower" +version = "2.0.1" +requires_python = ">=3.7" +summary = "Celery Flower" +groups = ["default"] +dependencies = [ + "celery>=5.0.5", + "humanize", + "prometheus-client>=0.8.0", + "pytz", + "tornado<7.0.0,>=5.0.0", +] +files = [ + {file = "flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2"}, + {file = "flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0"}, +] + [[package]] name = "freezegun" version = "1.5.1" @@ -1030,15 +1085,26 @@ files = [ {file = "graphene_stubs-0.16-py3-none-any.whl", hash = "sha256:7fae3ff663344db1b3ee5b187f054a1d018bb63c364c3624890fe02960c6affe"}, ] +[[package]] +name = "humanize" +version = "4.10.0" +requires_python = ">=3.8" +summary = "Python humanize utilities" +groups = ["default"] +files = [ + {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, + {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, +] + [[package]] name = "identify" -version = "2.5.36" +version = "2.6.0" requires_python = ">=3.8" summary = "File identification library for Python" groups = ["dev"] files = [ - {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"}, - {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"}, + {file = "identify-2.6.0-py2.py3-none-any.whl", hash = "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"}, + {file = "identify-2.6.0.tar.gz", hash = "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf"}, ] [[package]] @@ -1081,8 +1147,24 @@ requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" summary = "IPython-enabled pdb" groups = ["dev"] dependencies = [ + "decorator; python_version == \"3.5\"", + "decorator; python_version == \"3.6\"", + "decorator; python_version > \"3.6\" and python_version < \"3.11\"", "decorator; python_version >= \"3.11\"", + "decorator<5.0.0; python_version == \"2.7\"", + "decorator<5.0.0; python_version == \"3.4\"", + "ipython<6.0.0,>=5.1.0; python_version == \"2.7\"", + "ipython<7.0.0,>=6.0.0; python_version == \"3.4\"", + "ipython<7.10.0,>=7.0.0; python_version == \"3.5\"", + "ipython<7.17.0,>=7.16.3; python_version == \"3.6\"", + "ipython>=7.31.1; python_version > \"3.6\" and python_version < \"3.11\"", "ipython>=7.31.1; python_version >= \"3.11\"", + "pathlib; python_version == \"2.7\"", + "toml>=0.10.2; python_version == \"2.7\"", + "toml>=0.10.2; python_version == \"3.4\"", + "toml>=0.10.2; python_version == \"3.5\"", + "tomli; python_version == \"3.6\"", + "tomli; python_version > \"3.6\" and python_version < \"3.11\"", ] files = [ {file = "ipdb-0.13.13-py3-none-any.whl", hash = "sha256:45529994741c4ab6d2388bfa5d7b725c2cf7fe9deffabdb8a6113aa5ed449ed4"}, @@ -1091,7 +1173,7 @@ files = [ [[package]] name = "ipython" -version = "8.24.0" +version = "8.26.0" requires_python = ">=3.10" summary = "IPython: Productive Interactive Computing" groups = ["dev"] @@ -1099,6 +1181,7 @@ marker = "python_version >= \"3.11\"" dependencies = [ "colorama; sys_platform == \"win32\"", "decorator", + "exceptiongroup; python_version < \"3.11\"", "jedi>=0.16", "matplotlib-inline", "pexpect>4.3; sys_platform != \"win32\" and sys_platform != \"emscripten\"", @@ -1106,10 +1189,11 @@ dependencies = [ "pygments>=2.4.0", "stack-data", "traitlets>=5.13.0", + "typing-extensions>=4.6; python_version < \"3.12\"", ] files = [ - {file = "ipython-8.24.0-py3-none-any.whl", hash = "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3"}, - {file = "ipython-8.24.0.tar.gz", hash = "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501"}, + {file = "ipython-8.26.0-py3-none-any.whl", hash = "sha256:e6b347c27bdf9c32ee9d31ae85defc525755a1869f14057e900675b9e8d6e6ff"}, + {file = "ipython-8.26.0.tar.gz", hash = "sha256:1cec0fbba8404af13facebe83d04436a7434c7400e59f47acf467c64abd0956c"}, ] [[package]] @@ -1167,19 +1251,21 @@ files = [ [[package]] name = "jsonschema" -version = "4.22.0" +version = "4.23.0" requires_python = ">=3.8" summary = "An implementation of JSON Schema validation for Python" groups = ["default"] dependencies = [ "attrs>=22.2.0", + "importlib-resources>=1.4.0; python_version < \"3.9\"", "jsonschema-specifications>=2023.03.6", + "pkgutil-resolve-name>=1.3.10; python_version < \"3.9\"", "referencing>=0.28.4", "rpds-py>=0.7.1", ] files = [ - {file = "jsonschema-4.22.0-py3-none-any.whl", hash = "sha256:ff4cfd6b1367a40e7bc6411caec72effadd3db0bbe5017de188f2d6108335802"}, - {file = "jsonschema-4.22.0.tar.gz", hash = "sha256:5b22d434a45935119af990552c862e5d6d564e8f6601206b305a61fdf661a2b7"}, + {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, + {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, ] [[package]] @@ -1189,6 +1275,7 @@ requires_python = ">=3.8" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" groups = ["default"] dependencies = [ + "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", ] files = [ @@ -1217,6 +1304,8 @@ summary = "Messaging library for Python." groups = ["default", "dev"] dependencies = [ "amqp<6.0.0,>=5.1.1", + "backports-zoneinfo[tzdata]>=0.2.1; python_version < \"3.9\"", + "typing-extensions; python_version < \"3.10\"", "vine", ] files = [ @@ -1252,30 +1341,6 @@ files = [ {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f11ae142f3a322d44513de1018b50f474f8f736bc3cd91d969f464b5bfef8836"}, {file = "lxml-5.2.2-cp312-cp312-win32.whl", hash = "sha256:16a8326e51fcdffc886294c1e70b11ddccec836516a343f9ed0f82aac043c24a"}, {file = "lxml-5.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:bbc4b80af581e18568ff07f6395c02114d05f4865c2812a1f02f2eaecf0bfd48"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b537bd04d7ccd7c6350cdaaaad911f6312cbd61e6e6045542f781c7f8b2e99d2"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4820c02195d6dfb7b8508ff276752f6b2ff8b64ae5d13ebe02e7667e035000b9"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a09f6184f17a80897172863a655467da2b11151ec98ba8d7af89f17bf63dae"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:76acba4c66c47d27c8365e7c10b3d8016a7da83d3191d053a58382311a8bf4e1"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b128092c927eaf485928cec0c28f6b8bead277e28acf56800e972aa2c2abd7a2"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ae791f6bd43305aade8c0e22f816b34f3b72b6c820477aab4d18473a37e8090b"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a2f6a1bc2460e643785a2cde17293bd7a8f990884b822f7bca47bee0a82fc66b"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e8d351ff44c1638cb6e980623d517abd9f580d2e53bfcd18d8941c052a5a009"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bec4bd9133420c5c52d562469c754f27c5c9e36ee06abc169612c959bd7dbb07"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:55ce6b6d803890bd3cc89975fca9de1dff39729b43b73cb15ddd933b8bc20484"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ab6a358d1286498d80fe67bd3d69fcbc7d1359b45b41e74c4a26964ca99c3f8"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:06668e39e1f3c065349c51ac27ae430719d7806c026fec462e5693b08b95696b"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9cd5323344d8ebb9fb5e96da5de5ad4ebab993bbf51674259dbe9d7a18049525"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89feb82ca055af0fe797a2323ec9043b26bc371365847dbe83c7fd2e2f181c34"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e481bba1e11ba585fb06db666bfc23dbe181dbafc7b25776156120bf12e0d5a6"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d6c6ea6a11ca0ff9cd0390b885984ed31157c168565702959c25e2191674a14"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3d98de734abee23e61f6b8c2e08a88453ada7d6486dc7cdc82922a03968928db"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:69ab77a1373f1e7563e0fb5a29a8440367dec051da6c7405333699d07444f511"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:34e17913c431f5ae01d8658dbf792fdc457073dcdfbb31dc0cc6ab256e664a8d"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05f8757b03208c3f50097761be2dea0aba02e94f0dc7023ed73a7bb14ff11eb0"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a520b4f9974b0a0a6ed73c2154de57cdfd0c8800f4f15ab2b73238ffed0b36e"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5e097646944b66207023bc3c634827de858aebc226d5d4d6d16f0b77566ea182"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b5e4ef22ff25bfd4ede5f8fb30f7b24446345f3e79d9b7455aef2836437bc38a"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff69a9a0b4b17d78170c73abe2ab12084bdf1691550c5629ad1fe7849433f324"}, {file = "lxml-5.2.2.tar.gz", hash = "sha256:bb2dc4898180bea79863d5487e5f9c7c34297414bad54bcd0f0852aee9cfdb87"}, ] @@ -1327,22 +1392,23 @@ files = [ [[package]] name = "mypy" -version = "1.10.0" +version = "1.11.0" requires_python = ">=3.8" summary = "Optional static typing for Python" groups = ["dev"] dependencies = [ "mypy-extensions>=1.0.0", - "typing-extensions>=4.1.0", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions>=4.6.0", ] files = [ - {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, - {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, - {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, - {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, - {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, - {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, - {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, + {file = "mypy-1.11.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1d44c1e44a8be986b54b09f15f2c1a66368eb43861b4e82573026e04c48a9e20"}, + {file = "mypy-1.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cea3d0fb69637944dd321f41bc896e11d0fb0b0aa531d887a6da70f6e7473aba"}, + {file = "mypy-1.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a83ec98ae12d51c252be61521aa5731f5512231d0b738b4cb2498344f0b840cd"}, + {file = "mypy-1.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c7b73a856522417beb78e0fb6d33ef89474e7a622db2653bc1285af36e2e3e3d"}, + {file = "mypy-1.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:f2268d9fcd9686b61ab64f077be7ffbc6fbcdfb4103e5dd0cc5eaab53a8886c2"}, + {file = "mypy-1.11.0-py3-none-any.whl", hash = "sha256:56913ec8c7638b0091ef4da6fcc9136896914a9d60d54670a75880c3e5b99ace"}, + {file = "mypy-1.11.0.tar.gz", hash = "sha256:93743608c7348772fdc717af4aeee1997293a1ad04bc0ea6efa15bf65385c538"}, ] [[package]] @@ -1358,13 +1424,13 @@ files = [ [[package]] name = "nodeenv" -version = "1.9.0" +version = "1.9.1" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Node.js virtual environment builder" groups = ["dev"] files = [ - {file = "nodeenv-1.9.0-py2.py3-none-any.whl", hash = "sha256:508ecec98f9f3330b636d4448c0f1a56fc68017c68f1e7857ebc52acf0eb879a"}, - {file = "nodeenv-1.9.0.tar.gz", hash = "sha256:07f144e90dae547bf0d4ee8da0ee42664a42a04e02ed68e06324348dafe4bdb1"}, + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] [[package]] @@ -1382,9 +1448,6 @@ files = [ {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, - {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, - {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, - {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] @@ -1400,43 +1463,45 @@ files = [ ] [[package]] -name = "opencv-python" -version = "4.9.0.80" +name = "opencv-contrib-python-headless" +version = "4.10.0.84" requires_python = ">=3.6" summary = "Wrapper package for OpenCV python bindings." groups = ["default"] dependencies = [ + "numpy>=1.13.3; python_version < \"3.7\"", "numpy>=1.17.0; python_version >= \"3.7\"", "numpy>=1.17.3; python_version >= \"3.8\"", "numpy>=1.19.3; python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\"", "numpy>=1.19.3; python_version >= \"3.9\"", + "numpy>=1.21.0; python_version <= \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\"", "numpy>=1.21.2; python_version >= \"3.10\"", "numpy>=1.21.4; python_version >= \"3.10\" and platform_system == \"Darwin\"", "numpy>=1.23.5; python_version >= \"3.11\"", "numpy>=1.26.0; python_version >= \"3.12\"", ] files = [ - {file = "opencv-python-4.9.0.80.tar.gz", hash = "sha256:1a9f0e6267de3a1a1db0c54213d022c7c8b5b9ca4b580e80bdc58516c922c9e1"}, - {file = "opencv_python-4.9.0.80-cp37-abi3-macosx_10_16_x86_64.whl", hash = "sha256:7e5f7aa4486651a6ebfa8ed4b594b65bd2d2f41beeb4241a3e4b1b85acbbbadb"}, - {file = "opencv_python-4.9.0.80-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71dfb9555ccccdd77305fc3dcca5897fbf0cf28b297c51ee55e079c065d812a3"}, - {file = "opencv_python-4.9.0.80-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b34a52e9da36dda8c151c6394aed602e4b17fa041df0b9f5b93ae10b0fcca2a"}, - {file = "opencv_python-4.9.0.80-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4088cab82b66a3b37ffc452976b14a3c599269c247895ae9ceb4066d8188a57"}, - {file = "opencv_python-4.9.0.80-cp37-abi3-win32.whl", hash = "sha256:dcf000c36dd1651118a2462257e3a9e76db789a78432e1f303c7bac54f63ef6c"}, - {file = "opencv_python-4.9.0.80-cp37-abi3-win_amd64.whl", hash = "sha256:3f16f08e02b2a2da44259c7cc712e779eff1dd8b55fdb0323e8cab09548086c0"}, + {file = "opencv-contrib-python-headless-4.10.0.84.tar.gz", hash = "sha256:6351250db97e1f91f31afdec2436afb1c89594e3da02851e0f01e20ea16bbd9e"}, + {file = "opencv_contrib_python_headless-4.10.0.84-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:be91c6c81e839613c6f3b15755bf71789839289d0e3440fab093e0708516ffcf"}, + {file = "opencv_contrib_python_headless-4.10.0.84-cp37-abi3-macosx_12_0_x86_64.whl", hash = "sha256:252df47a7e1da280cef26ee0ecc1799841015ce3718214634bb15bc22d4cb308"}, + {file = "opencv_contrib_python_headless-4.10.0.84-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77eb20ee077ac0955704d391c00639df6063cb67cb62606c07b97d8b635feff6"}, + {file = "opencv_contrib_python_headless-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89c16eb5f888aee7bf664106e12c423705d29d1b094876b66aa4e33d4e8ec905"}, + {file = "opencv_contrib_python_headless-4.10.0.84-cp37-abi3-win32.whl", hash = "sha256:7581d7ffb7fff953436797dca2dfc5e70e100f721ea18ab84ebf11417ea21d0c"}, + {file = "opencv_contrib_python_headless-4.10.0.84-cp37-abi3-win_amd64.whl", hash = "sha256:660ded6b77b07f875f56065016677bbb6a3abca13903b9320164691a46474a7d"}, ] [[package]] name = "openpyxl" -version = "3.1.2" -requires_python = ">=3.6" +version = "3.1.5" +requires_python = ">=3.8" summary = "A Python library to read/write Excel 2010 xlsx/xlsm files" groups = ["dev"] dependencies = [ "et-xmlfile", ] files = [ - {file = "openpyxl-3.1.2-py2.py3-none-any.whl", hash = "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"}, - {file = "openpyxl-3.1.2.tar.gz", hash = "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184"}, + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, ] [[package]] @@ -1456,13 +1521,13 @@ files = [ [[package]] name = "packaging" -version = "24.0" -requires_python = ">=3.7" +version = "24.1" +requires_python = ">=3.8" summary = "Core utilities for Python packages" groups = ["dev"] files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] @@ -1488,6 +1553,21 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pdbpp" +version = "0.10.3" +summary = "pdb++, a drop-in replacement for pdb" +groups = ["dev"] +dependencies = [ + "fancycompleter>=0.8", + "pygments", + "wmctrl", +] +files = [ + {file = "pdbpp-0.10.3-py2.py3-none-any.whl", hash = "sha256:79580568e33eb3d6f6b462b1187f53e10cd8e4538f7d31495c9181e2cf9665d1"}, + {file = "pdbpp-0.10.3.tar.gz", hash = "sha256:d9e43f4fda388eeb365f2887f4e7b66ac09dce9b6236b76f63616530e2f669f5"}, +] + [[package]] name = "pexpect" version = "4.9.0" @@ -1504,37 +1584,34 @@ files = [ [[package]] name = "pillow" -version = "10.3.0" +version = "10.4.0" requires_python = ">=3.8" summary = "Python Imaging Library (Fork)" groups = ["default"] files = [ - {file = "pillow-10.3.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:e46f38133e5a060d46bd630faa4d9fa0202377495df1f068a8299fd78c84de84"}, - {file = "pillow-10.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:50b8eae8f7334ec826d6eeffaeeb00e36b5e24aa0b9df322c247539714c6df19"}, - {file = "pillow-10.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d3bea1c75f8c53ee4d505c3e67d8c158ad4df0d83170605b50b64025917f338"}, - {file = "pillow-10.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19aeb96d43902f0a783946a0a87dbdad5c84c936025b8419da0a0cd7724356b1"}, - {file = "pillow-10.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:74d28c17412d9caa1066f7a31df8403ec23d5268ba46cd0ad2c50fb82ae40462"}, - {file = "pillow-10.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a"}, - {file = "pillow-10.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d886f5d353333b4771d21267c7ecc75b710f1a73d72d03ca06df49b09015a9ef"}, - {file = "pillow-10.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4b5ec25d8b17217d635f8935dbc1b9aa5907962fae29dff220f2659487891cd3"}, - {file = "pillow-10.3.0-cp312-cp312-win32.whl", hash = "sha256:51243f1ed5161b9945011a7360e997729776f6e5d7005ba0c6879267d4c5139d"}, - {file = "pillow-10.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:412444afb8c4c7a6cc11a47dade32982439925537e483be7c0ae0cf96c4f6a0b"}, - {file = "pillow-10.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:798232c92e7665fe82ac085f9d8e8ca98826f8e27859d9a96b41d519ecd2e49a"}, - {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6b02471b72526ab8a18c39cb7967b72d194ec53c1fd0a70b050565a0f366d355"}, - {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8ab74c06ffdab957d7670c2a5a6e1a70181cd10b727cd788c4dd9005b6a8acd9"}, - {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:048eeade4c33fdf7e08da40ef402e748df113fd0b4584e32c4af74fe78baaeb2"}, - {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2ec1e921fd07c7cda7962bad283acc2f2a9ccc1b971ee4b216b75fad6f0463"}, - {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c8e73e99da7db1b4cad7f8d682cf6abad7844da39834c288fbfa394a47bbced"}, - {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:16563993329b79513f59142a6b02055e10514c1a8e86dca8b48a893e33cf91e3"}, - {file = "pillow-10.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dd78700f5788ae180b5ee8902c6aea5a5726bac7c364b202b4b3e3ba2d293170"}, - {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:aff76a55a8aa8364d25400a210a65ff59d0168e0b4285ba6bf2bd83cf675ba32"}, - {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7bc2176354defba3edc2b9a777744462da2f8e921fbaf61e52acb95bafa9828"}, - {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:793b4e24db2e8742ca6423d3fde8396db336698c55cd34b660663ee9e45ed37f"}, - {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93480005693d247f8346bc8ee28c72a2191bdf1f6b5db469c096c0c867ac015"}, - {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83341b89884e2b2e55886e8fbbf37c3fa5efd6c8907124aeb72f285ae5696e5"}, - {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1a1d1915db1a4fdb2754b9de292642a39a7fb28f1736699527bb649484fb966a"}, - {file = "pillow-10.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a0eaa93d054751ee9964afa21c06247779b90440ca41d184aeb5d410f20ff591"}, - {file = "pillow-10.3.0.tar.gz", hash = "sha256:9d2455fbf44c914840c793e89aa82d0e1763a14253a000743719ae5946814b2d"}, + {file = "pillow-10.4.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94"}, + {file = "pillow-10.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597"}, + {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80"}, + {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca"}, + {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef"}, + {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a"}, + {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b"}, + {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9"}, + {file = "pillow-10.4.0-cp312-cp312-win32.whl", hash = "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42"}, + {file = "pillow-10.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a"}, + {file = "pillow-10.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9"}, + {file = "pillow-10.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3"}, + {file = "pillow-10.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb"}, + {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70"}, + {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be"}, + {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0"}, + {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc"}, + {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a"}, + {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309"}, + {file = "pillow-10.4.0-cp313-cp313-win32.whl", hash = "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060"}, + {file = "pillow-10.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea"}, + {file = "pillow-10.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d"}, + {file = "pillow-10.4.0.tar.gz", hash = "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06"}, ] [[package]] @@ -1561,7 +1638,7 @@ files = [ [[package]] name = "pre-commit" -version = "3.7.1" +version = "3.8.0" requires_python = ">=3.9" summary = "A framework for managing and maintaining multi-language pre-commit hooks." groups = ["dev"] @@ -1573,13 +1650,24 @@ dependencies = [ "virtualenv>=20.10.0", ] files = [ - {file = "pre_commit-3.7.1-py2.py3-none-any.whl", hash = "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5"}, - {file = "pre_commit-3.7.1.tar.gz", hash = "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a"}, + {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, + {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, +] + +[[package]] +name = "prometheus-client" +version = "0.20.0" +requires_python = ">=3.8" +summary = "Python client for the Prometheus monitoring system." +groups = ["default"] +files = [ + {file = "prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"}, + {file = "prometheus_client-0.20.0.tar.gz", hash = "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89"}, ] [[package]] name = "prompt-toolkit" -version = "3.0.45" +version = "3.0.47" requires_python = ">=3.7.0" summary = "Library for building powerful interactive command lines in Python" groups = ["default", "dev"] @@ -1587,24 +1675,25 @@ dependencies = [ "wcwidth", ] files = [ - {file = "prompt_toolkit-3.0.45-py3-none-any.whl", hash = "sha256:a29b89160e494e3ea8622b09fa5897610b437884dcdcd054fdc1308883326c2a"}, - {file = "prompt_toolkit-3.0.45.tar.gz", hash = "sha256:07c60ee4ab7b7e90824b61afa840c8f5aad2d46b3e2e10acc33d8ecc94a49089"}, + {file = "prompt_toolkit-3.0.47-py3-none-any.whl", hash = "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10"}, + {file = "prompt_toolkit-3.0.47.tar.gz", hash = "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"}, ] [[package]] name = "psutil" -version = "5.9.8" -requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +version = "6.0.0" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" summary = "Cross-platform lib for process and system monitoring in Python." groups = ["dev"] files = [ - {file = "psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"}, - {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"}, - {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"}, - {file = "psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"}, - {file = "psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"}, - {file = "psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"}, - {file = "psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"}, + {file = "psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"}, + {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"}, + {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"}, + {file = "psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"}, + {file = "psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"}, + {file = "psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"}, + {file = "psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"}, + {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"}, ] [[package]] @@ -1642,35 +1731,24 @@ files = [ [[package]] name = "pure-eval" -version = "0.2.2" +version = "0.2.3" summary = "Safely evaluate AST nodes without side effects" groups = ["dev"] marker = "python_version >= \"3.11\"" files = [ - {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, - {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, -] - -[[package]] -name = "py" -version = "1.11.0" -requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -summary = "library with cross-python path, ini-parsing, io, code, log facilities" -groups = ["dev"] -files = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, + {file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"}, + {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"}, ] [[package]] name = "pycodestyle" -version = "2.11.1" +version = "2.12.0" requires_python = ">=3.8" summary = "Python style guide checker" groups = ["dev"] files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, + {file = "pycodestyle-2.12.0-py2.py3-none-any.whl", hash = "sha256:949a39f6b86c3e1515ba1787c2022131d165a8ad271b11370a8819aa070269e4"}, + {file = "pycodestyle-2.12.0.tar.gz", hash = "sha256:442f950141b4f43df752dd303511ffded3a04c2b6fb7f65980574f0c31e6e79c"}, ] [[package]] @@ -1713,46 +1791,70 @@ version = "2.8.0" requires_python = ">=3.7" summary = "JSON Web Token implementation in Python" groups = ["default"] +dependencies = [ + "typing-extensions; python_version <= \"3.7\"", +] files = [ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, ] +[[package]] +name = "pyreadline" +version = "2.1" +summary = "A python implmementation of GNU readline." +groups = ["dev"] +marker = "platform_system == \"Windows\"" +files = [ + {file = "pyreadline-2.1.zip", hash = "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1"}, +] + +[[package]] +name = "pyrepl" +version = "0.9.0" +summary = "A library for building flexible command line interfaces" +groups = ["dev"] +files = [ + {file = "pyrepl-0.9.0.tar.gz", hash = "sha256:292570f34b5502e871bbb966d639474f2b57fbfcd3373c2d6a2f3d56e681a775"}, +] + [[package]] name = "pytest" -version = "8.2.1" +version = "8.3.2" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" groups = ["dev"] dependencies = [ "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", "iniconfig", "packaging", - "pluggy<2.0,>=1.5", + "pluggy<2,>=1.5", + "tomli>=1; python_version < \"3.11\"", ] files = [ - {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"}, - {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"}, + {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, + {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, ] [[package]] name = "pytest-celery" -version = "1.0.0" +version = "1.0.1" requires_python = "<4.0,>=3.8" summary = "Pytest plugin for Celery" groups = ["dev"] dependencies = [ "celery", "debugpy<2.0.0,>=1.8.1", - "docker<8.0.0,>=7.0.0", + "docker<8.0.0,>=7.1.0", "psutil>=5.9.7", "pytest-docker-tools>=3.1.3", - "retry>=0.9.2", "setuptools>=69.1.0", + "tenacity>=8.5.0", ] files = [ - {file = "pytest_celery-1.0.0-py3-none-any.whl", hash = "sha256:c10bc7d16daa3ae4a5784efcbd1855d610c0e087c21d185e52fa018b3a6c4249"}, - {file = "pytest_celery-1.0.0.tar.gz", hash = "sha256:17a066b1554d4fa8797d4928e8b8cda1bfb441dae4688ca29fdbde28ffa49ff7"}, + {file = "pytest_celery-1.0.1-py3-none-any.whl", hash = "sha256:8f0068f0b5deb3123c76ae56327d40ece488c622daee54b3c5ff968c503df841"}, + {file = "pytest_celery-1.0.1.tar.gz", hash = "sha256:8ab12f2f16946e131c315efce2d71fa3b74a05269077fde04f96a6048b249377"}, ] [[package]] @@ -1862,14 +1964,15 @@ files = [ [[package]] name = "python-crontab" -version = "3.1.0" +version = "3.2.0" summary = "Python Crontab API" groups = ["default"] dependencies = [ "python-dateutil", ] files = [ - {file = "python-crontab-3.1.0.tar.gz", hash = "sha256:f4ea1605d24533b67fa7a634ef26cb59a5f2e7954f6e677d2d7a2229959a2fc8"}, + {file = "python_crontab-3.2.0-py3-none-any.whl", hash = "sha256:82cb9b6a312d41ff66fd3caf3eed7115c28c195bfb50711bc2b4b9592feb9fe5"}, + {file = "python_crontab-3.2.0.tar.gz", hash = "sha256:40067d1dd39ade3460b2ad8557c7651514cd3851deffff61c5c60e1227c5c36b"}, ] [[package]] @@ -1940,13 +2043,18 @@ files = [ [[package]] name = "redis" -version = "5.0.4" +version = "5.0.7" requires_python = ">=3.7" summary = "Python client for Redis database and key-value store" groups = ["default"] +dependencies = [ + "async-timeout>=4.0.3; python_full_version < \"3.11.3\"", + "importlib-metadata>=1.0; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", +] files = [ - {file = "redis-5.0.4-py3-none-any.whl", hash = "sha256:7adc2835c7a9b5033b7ad8f8918d09b7344188228809c98df07af226d39dec91"}, - {file = "redis-5.0.4.tar.gz", hash = "sha256:ec31f2ed9675cc54c21ba854cfe0462e6faf1d83c8ce5944709db8a4700b9c61"}, + {file = "redis-5.0.7-py3-none-any.whl", hash = "sha256:0e479e24da960c690be5d9b96d21f7b918a98c0cf49af3b6fafaa0753f93a0db"}, + {file = "redis-5.0.7.tar.gz", hash = "sha256:8f611490b93c8109b50adc317b31bfd84fff31def3475b92e7e80bf39f48175b"}, ] [[package]] @@ -2012,7 +2120,7 @@ files = [ [[package]] name = "responses" -version = "0.25.0" +version = "0.25.3" requires_python = ">=3.8" summary = "A utility library for mocking out the `requests` Python library." groups = ["dev"] @@ -2022,83 +2130,49 @@ dependencies = [ "urllib3<3.0,>=1.25.10", ] files = [ - {file = "responses-0.25.0-py3-none-any.whl", hash = "sha256:2f0b9c2b6437db4b528619a77e5d565e4ec2a9532162ac1a131a83529db7be1a"}, - {file = "responses-0.25.0.tar.gz", hash = "sha256:01ae6a02b4f34e39bffceb0fc6786b67a25eae919c6368d05eabc8d9576c2a66"}, -] - -[[package]] -name = "retry" -version = "0.9.2" -summary = "Easy to use retry decorator." -groups = ["dev"] -dependencies = [ - "decorator>=3.4.2", - "py<2.0.0,>=1.4.26", -] -files = [ - {file = "retry-0.9.2-py2.py3-none-any.whl", hash = "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606"}, - {file = "retry-0.9.2.tar.gz", hash = "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4"}, + {file = "responses-0.25.3-py3-none-any.whl", hash = "sha256:521efcbc82081ab8daa588e08f7e8a64ce79b91c39f6e62199b19159bea7dbcb"}, + {file = "responses-0.25.3.tar.gz", hash = "sha256:617b9247abd9ae28313d57a75880422d55ec63c29d33d629697590a034358dba"}, ] [[package]] name = "rpds-py" -version = "0.18.1" +version = "0.19.1" requires_python = ">=3.8" summary = "Python bindings to Rust's persistent data structures (rpds)" groups = ["default"] files = [ - {file = "rpds_py-0.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3dd3cd86e1db5aadd334e011eba4e29d37a104b403e8ca24dcd6703c68ca55b3"}, - {file = "rpds_py-0.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:05f3d615099bd9b13ecf2fc9cf2d839ad3f20239c678f461c753e93755d629ee"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35b2b771b13eee8729a5049c976197ff58a27a3829c018a04341bcf1ae409b2b"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee17cd26b97d537af8f33635ef38be873073d516fd425e80559f4585a7b90c43"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b646bf655b135ccf4522ed43d6902af37d3f5dbcf0da66c769a2b3938b9d8184"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19ba472b9606c36716062c023afa2484d1e4220548751bda14f725a7de17b4f6"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e30ac5e329098903262dc5bdd7e2086e0256aa762cc8b744f9e7bf2a427d3f8"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d58ad6317d188c43750cb76e9deacf6051d0f884d87dc6518e0280438648a9ac"}, - {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e1735502458621921cee039c47318cb90b51d532c2766593be6207eec53e5c4c"}, - {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f5bab211605d91db0e2995a17b5c6ee5edec1270e46223e513eaa20da20076ac"}, - {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2fc24a329a717f9e2448f8cd1f960f9dac4e45b6224d60734edeb67499bab03a"}, - {file = "rpds_py-0.18.1-cp312-none-win32.whl", hash = "sha256:1805d5901779662d599d0e2e4159d8a82c0b05faa86ef9222bf974572286b2b6"}, - {file = "rpds_py-0.18.1-cp312-none-win_amd64.whl", hash = "sha256:720edcb916df872d80f80a1cc5ea9058300b97721efda8651efcd938a9c70a72"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cbfbea39ba64f5e53ae2915de36f130588bba71245b418060ec3330ebf85678e"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3d456ff2a6a4d2adcdf3c1c960a36f4fd2fec6e3b4902a42a384d17cf4e7a65"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7700936ef9d006b7ef605dc53aa364da2de5a3aa65516a1f3ce73bf82ecfc7ae"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:51584acc5916212e1bf45edd17f3a6b05fe0cbb40482d25e619f824dccb679de"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:942695a206a58d2575033ff1e42b12b2aece98d6003c6bc739fbf33d1773b12f"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b906b5f58892813e5ba5c6056d6a5ad08f358ba49f046d910ad992196ea61397"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f8e3fecca256fefc91bb6765a693d96692459d7d4c644660a9fff32e517843"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7732770412bab81c5a9f6d20aeb60ae943a9b36dcd990d876a773526468e7163"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:bd1105b50ede37461c1d51b9698c4f4be6e13e69a908ab7751e3807985fc0346"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:618916f5535784960f3ecf8111581f4ad31d347c3de66d02e728de460a46303c"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:17c6d2155e2423f7e79e3bb18151c686d40db42d8645e7977442170c360194d4"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6c4c4c3f878df21faf5fac86eda32671c27889e13570645a9eea0a1abdd50922"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:fab6ce90574645a0d6c58890e9bcaac8d94dff54fb51c69e5522a7358b80ab64"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:531796fb842b53f2695e94dc338929e9f9dbf473b64710c28af5a160b2a8927d"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:740884bc62a5e2bbb31e584f5d23b32320fd75d79f916f15a788d527a5e83644"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:998125738de0158f088aef3cb264a34251908dd2e5d9966774fdab7402edfab7"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2be6e9dd4111d5b31ba3b74d17da54a8319d8168890fbaea4b9e5c3de630ae5"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0cee71bc618cd93716f3c1bf56653740d2d13ddbd47673efa8bf41435a60daa"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2c3caec4ec5cd1d18e5dd6ae5194d24ed12785212a90b37f5f7f06b8bedd7139"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:27bba383e8c5231cd559affe169ca0b96ec78d39909ffd817f28b166d7ddd4d8"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:a888e8bdb45916234b99da2d859566f1e8a1d2275a801bb8e4a9644e3c7e7909"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6031b25fb1b06327b43d841f33842b383beba399884f8228a6bb3df3088485ff"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48c2faaa8adfacefcbfdb5f2e2e7bdad081e5ace8d182e5f4ade971f128e6bb3"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d85164315bd68c0806768dc6bb0429c6f95c354f87485ee3593c4f6b14def2bd"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6afd80f6c79893cfc0574956f78a0add8c76e3696f2d6a15bca2c66c415cf2d4"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa242ac1ff583e4ec7771141606aafc92b361cd90a05c30d93e343a0c2d82a89"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21be4770ff4e08698e1e8e0bce06edb6ea0626e7c8f560bc08222880aca6a6f"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c45a639e93a0c5d4b788b2613bd637468edd62f8f95ebc6fcc303d58ab3f0a8"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910e71711d1055b2768181efa0a17537b2622afeb0424116619817007f8a2b10"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9bb1f182a97880f6078283b3505a707057c42bf55d8fca604f70dedfdc0772a"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1d54f74f40b1f7aaa595a02ff42ef38ca654b1469bef7d52867da474243cc633"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:8d2e182c9ee01135e11e9676e9a62dfad791a7a467738f06726872374a83db49"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:636a15acc588f70fda1661234761f9ed9ad79ebed3f2125d44be0862708b666e"}, - {file = "rpds_py-0.18.1.tar.gz", hash = "sha256:dc48b479d540770c811fbd1eb9ba2bb66951863e448efec2e2c102625328e92f"}, + {file = "rpds_py-0.19.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:56313be667a837ff1ea3508cebb1ef6681d418fa2913a0635386cf29cff35165"}, + {file = "rpds_py-0.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d1d7539043b2b31307f2c6c72957a97c839a88b2629a348ebabe5aa8b626d6b"}, + {file = "rpds_py-0.19.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e1dc59a5e7bc7f44bd0c048681f5e05356e479c50be4f2c1a7089103f1621d5"}, + {file = "rpds_py-0.19.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b8f78398e67a7227aefa95f876481485403eb974b29e9dc38b307bb6eb2315ea"}, + {file = "rpds_py-0.19.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef07a0a1d254eeb16455d839cef6e8c2ed127f47f014bbda64a58b5482b6c836"}, + {file = "rpds_py-0.19.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8124101e92c56827bebef084ff106e8ea11c743256149a95b9fd860d3a4f331f"}, + {file = "rpds_py-0.19.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08ce9c95a0b093b7aec75676b356a27879901488abc27e9d029273d280438505"}, + {file = "rpds_py-0.19.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0b02dd77a2de6e49078c8937aadabe933ceac04b41c5dde5eca13a69f3cf144e"}, + {file = "rpds_py-0.19.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4dd02e29c8cbed21a1875330b07246b71121a1c08e29f0ee3db5b4cfe16980c4"}, + {file = "rpds_py-0.19.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9c7042488165f7251dc7894cd533a875d2875af6d3b0e09eda9c4b334627ad1c"}, + {file = "rpds_py-0.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f809a17cc78bd331e137caa25262b507225854073fd319e987bd216bed911b7c"}, + {file = "rpds_py-0.19.1-cp312-none-win32.whl", hash = "sha256:3ddab996807c6b4227967fe1587febade4e48ac47bb0e2d3e7858bc621b1cace"}, + {file = "rpds_py-0.19.1-cp312-none-win_amd64.whl", hash = "sha256:32e0db3d6e4f45601b58e4ac75c6f24afbf99818c647cc2066f3e4b192dabb1f"}, + {file = "rpds_py-0.19.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:747251e428406b05fc86fee3904ee19550c4d2d19258cef274e2151f31ae9d38"}, + {file = "rpds_py-0.19.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dc733d35f861f8d78abfaf54035461e10423422999b360966bf1c443cbc42705"}, + {file = "rpds_py-0.19.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbda75f245caecff8faa7e32ee94dfaa8312a3367397975527f29654cd17a6ed"}, + {file = "rpds_py-0.19.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd04d8cab16cab5b0a9ffc7d10f0779cf1120ab16c3925404428f74a0a43205a"}, + {file = "rpds_py-0.19.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2d66eb41ffca6cc3c91d8387509d27ba73ad28371ef90255c50cb51f8953301"}, + {file = "rpds_py-0.19.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdf4890cda3b59170009d012fca3294c00140e7f2abe1910e6a730809d0f3f9b"}, + {file = "rpds_py-0.19.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1fa67ef839bad3815124f5f57e48cd50ff392f4911a9f3cf449d66fa3df62a5"}, + {file = "rpds_py-0.19.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b82c9514c6d74b89a370c4060bdb80d2299bc6857e462e4a215b4ef7aa7b090e"}, + {file = "rpds_py-0.19.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c7b07959866a6afb019abb9564d8a55046feb7a84506c74a6f197cbcdf8a208e"}, + {file = "rpds_py-0.19.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4f580ae79d0b861dfd912494ab9d477bea535bfb4756a2269130b6607a21802e"}, + {file = "rpds_py-0.19.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c6d20c8896c00775e6f62d8373aba32956aa0b850d02b5ec493f486c88e12859"}, + {file = "rpds_py-0.19.1-cp313-none-win32.whl", hash = "sha256:afedc35fe4b9e30ab240b208bb9dc8938cb4afe9187589e8d8d085e1aacb8309"}, + {file = "rpds_py-0.19.1-cp313-none-win_amd64.whl", hash = "sha256:1d4af2eb520d759f48f1073ad3caef997d1bfd910dc34e41261a595d3f038a94"}, + {file = "rpds_py-0.19.1.tar.gz", hash = "sha256:31dd5794837f00b46f4096aa8ccaa5972f73a938982e32ed817bb520c465e520"}, ] [[package]] name = "sentry-sdk" -version = "2.3.1" +version = "2.11.0" requires_python = ">=3.6" summary = "Python client for Sentry (https://sentry.io)" groups = ["default"] @@ -2107,13 +2181,13 @@ dependencies = [ "urllib3>=1.26.11", ] files = [ - {file = "sentry_sdk-2.3.1-py2.py3-none-any.whl", hash = "sha256:c5aeb095ba226391d337dd42a6f9470d86c9fc236ecc71cfc7cd1942b45010c6"}, - {file = "sentry_sdk-2.3.1.tar.gz", hash = "sha256:139a71a19f5e9eb5d3623942491ce03cf8ebc14ea2e39ba3e6fe79560d8a5b1f"}, + {file = "sentry_sdk-2.11.0-py2.py3-none-any.whl", hash = "sha256:d964710e2dbe015d9dc4ff0ad16225d68c3b36936b742a6fe0504565b760a3b7"}, + {file = "sentry_sdk-2.11.0.tar.gz", hash = "sha256:4ca16e9f5c7c6bc2fb2d5c956219f4926b148e511fffdbbde711dc94f1e0468f"}, ] [[package]] name = "sentry-sdk" -version = "2.3.1" +version = "2.11.0" extras = ["celery", "django"] requires_python = ">=3.6" summary = "Python client for Sentry (https://sentry.io)" @@ -2121,22 +2195,22 @@ groups = ["default"] dependencies = [ "celery>=3", "django>=1.8", - "sentry-sdk==2.3.1", + "sentry-sdk==2.11.0", ] files = [ - {file = "sentry_sdk-2.3.1-py2.py3-none-any.whl", hash = "sha256:c5aeb095ba226391d337dd42a6f9470d86c9fc236ecc71cfc7cd1942b45010c6"}, - {file = "sentry_sdk-2.3.1.tar.gz", hash = "sha256:139a71a19f5e9eb5d3623942491ce03cf8ebc14ea2e39ba3e6fe79560d8a5b1f"}, + {file = "sentry_sdk-2.11.0-py2.py3-none-any.whl", hash = "sha256:d964710e2dbe015d9dc4ff0ad16225d68c3b36936b742a6fe0504565b760a3b7"}, + {file = "sentry_sdk-2.11.0.tar.gz", hash = "sha256:4ca16e9f5c7c6bc2fb2d5c956219f4926b148e511fffdbbde711dc94f1e0468f"}, ] [[package]] name = "setuptools" -version = "70.0.0" +version = "74.1.2" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" -groups = ["dev"] +groups = ["default", "dev"] files = [ - {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, - {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, + {file = "setuptools-74.1.2-py3-none-any.whl", hash = "sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308"}, + {file = "setuptools-74.1.2.tar.gz", hash = "sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6"}, ] [[package]] @@ -2152,7 +2226,7 @@ files = [ [[package]] name = "social-auth-app-django" -version = "5.4.1" +version = "5.4.2" requires_python = ">=3.8" summary = "Python Social Authentication, Django integration." groups = ["default"] @@ -2161,8 +2235,8 @@ dependencies = [ "social-auth-core>=4.4.1", ] files = [ - {file = "social-auth-app-django-5.4.1.tar.gz", hash = "sha256:2a43cde559dd34fdc7132417b6c52c780fa99ec2332dee9f405b4763f371c367"}, - {file = "social_auth_app_django-5.4.1-py3-none-any.whl", hash = "sha256:7519f186c63c50f2d364457b236f051338d194bcface55e318a6a705c5213477"}, + {file = "social-auth-app-django-5.4.2.tar.gz", hash = "sha256:c8832c6cf13da6ad76f5613bcda2647d89ae7cfbc5217fadd13477a3406feaa8"}, + {file = "social_auth_app_django-5.4.2-py3-none-any.whl", hash = "sha256:0c041a31707921aef9a930f143183c65d8c7b364381364a50f3f7c6fcc9d62f6"}, ] [[package]] @@ -2198,13 +2272,13 @@ files = [ [[package]] name = "sqlparse" -version = "0.5.0" +version = "0.5.1" requires_python = ">=3.8" summary = "A non-validating SQL parser." groups = ["default"] files = [ - {file = "sqlparse-0.5.0-py3-none-any.whl", hash = "sha256:c204494cd97479d0e39f28c93d46c0b2d5959c7b9ab904762ea6c7af211c8663"}, - {file = "sqlparse-0.5.0.tar.gz", hash = "sha256:714d0a4932c059d16189f58ef5411ec2287a4360f17cdd0edd2d09d4c5087c93"}, + {file = "sqlparse-0.5.1-py3-none-any.whl", hash = "sha256:773dcbf9a5ab44a090f3441e2180efe2560220203dc2f8c0b0fa141e18b505e4"}, + {file = "sqlparse-0.5.1.tar.gz", hash = "sha256:bb6b4df465655ef332548e24f08e205afc81b9ab86cb1c45657a7ff173a3a00e"}, ] [[package]] @@ -2223,6 +2297,37 @@ files = [ {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, ] +[[package]] +name = "tenacity" +version = "8.5.0" +requires_python = ">=3.8" +summary = "Retry code until it succeeds" +groups = ["dev"] +files = [ + {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"}, + {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"}, +] + +[[package]] +name = "tornado" +version = "6.4.1" +requires_python = ">=3.8" +summary = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +groups = ["default"] +files = [ + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, + {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, + {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, + {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, +] + [[package]] name = "traitlets" version = "5.14.3" @@ -2261,7 +2366,7 @@ files = [ [[package]] name = "types-pyopenssl" -version = "24.1.0.20240425" +version = "24.1.0.20240722" requires_python = ">=3.8" summary = "Typing stubs for pyOpenSSL" groups = ["dev"] @@ -2270,8 +2375,8 @@ dependencies = [ "types-cffi", ] files = [ - {file = "types-pyOpenSSL-24.1.0.20240425.tar.gz", hash = "sha256:0a7e82626c1983dc8dc59292bf20654a51c3c3881bcbb9b337c1da6e32f0204e"}, - {file = "types_pyOpenSSL-24.1.0.20240425-py3-none-any.whl", hash = "sha256:f51a156835555dd2a1f025621e8c4fbe7493470331afeef96884d1d29bf3a473"}, + {file = "types-pyOpenSSL-24.1.0.20240722.tar.gz", hash = "sha256:47913b4678a01d879f503a12044468221ed8576263c1540dcb0484ca21b08c39"}, + {file = "types_pyOpenSSL-24.1.0.20240722-py3-none-any.whl", hash = "sha256:6a7a5d2ec042537934cfb4c9d4deb0e16c4c6250b09358df1f083682fe6fda54"}, ] [[package]] @@ -2298,7 +2403,7 @@ files = [ [[package]] name = "types-redis" -version = "4.6.0.20240425" +version = "4.6.0.20240726" requires_python = ">=3.8" summary = "Typing stubs for redis" groups = ["dev"] @@ -2307,13 +2412,13 @@ dependencies = [ "types-pyOpenSSL", ] files = [ - {file = "types-redis-4.6.0.20240425.tar.gz", hash = "sha256:9402a10ee931d241fdfcc04592ebf7a661d7bb92a8dea631279f0d8acbcf3a22"}, - {file = "types_redis-4.6.0.20240425-py3-none-any.whl", hash = "sha256:ac5bc19e8f5997b9e76ad5d9cf15d0392d9f28cf5fc7746ea4a64b989c45c6a8"}, + {file = "types-redis-4.6.0.20240726.tar.gz", hash = "sha256:de2aefcf7afe80057debada8c540463d06c8863de50b8016bd369ccdbcb59b5e"}, + {file = "types_redis-4.6.0.20240726-py3-none-any.whl", hash = "sha256:233062b7120a9908532ec9163d17af74b80fa49a89d510444cad4cac42717378"}, ] [[package]] name = "types-requests" -version = "2.32.0.20240523" +version = "2.32.0.20240712" requires_python = ">=3.8" summary = "Typing stubs for requests" groups = ["dev"] @@ -2321,30 +2426,30 @@ dependencies = [ "urllib3>=2", ] files = [ - {file = "types-requests-2.32.0.20240523.tar.gz", hash = "sha256:26b8a6de32d9f561192b9942b41c0ab2d8010df5677ca8aa146289d11d505f57"}, - {file = "types_requests-2.32.0.20240523-py3-none-any.whl", hash = "sha256:f19ed0e2daa74302069bbbbf9e82902854ffa780bc790742a810a9aaa52f65ec"}, + {file = "types-requests-2.32.0.20240712.tar.gz", hash = "sha256:90c079ff05e549f6bf50e02e910210b98b8ff1ebdd18e19c873cd237737c1358"}, + {file = "types_requests-2.32.0.20240712-py3-none-any.whl", hash = "sha256:f754283e152c752e46e70942fa2a146b5bc70393522257bb85bd1ef7e019dcc3"}, ] [[package]] name = "types-setuptools" -version = "70.0.0.20240524" +version = "71.1.0.20240726" requires_python = ">=3.8" summary = "Typing stubs for setuptools" groups = ["dev"] files = [ - {file = "types-setuptools-70.0.0.20240524.tar.gz", hash = "sha256:e31fee7b9d15ef53980526579ac6089b3ae51a005a281acf97178e90ac71aff6"}, - {file = "types_setuptools-70.0.0.20240524-py3-none-any.whl", hash = "sha256:8f5379b9948682d72a9ab531fbe52932e84c4f38deda570255f9bae3edd766bc"}, + {file = "types-setuptools-71.1.0.20240726.tar.gz", hash = "sha256:85ba28e9461bb1be86ebba4db0f1c2408f2b11115b1966334ea9dc464e29303e"}, + {file = "types_setuptools-71.1.0.20240726-py3-none-any.whl", hash = "sha256:a7775376f36e0ff09bcad236bf265777590a66b11623e48c20bfc30f1444ea36"}, ] [[package]] name = "typing-extensions" -version = "4.12.0" +version = "4.12.2" requires_python = ">=3.8" summary = "Backported and Experimental Type Hints for Python 3.8+" groups = ["default", "dev"] files = [ - {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, - {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] @@ -2407,22 +2512,22 @@ files = [ [[package]] name = "urllib3" -version = "2.2.1" +version = "2.2.2" requires_python = ">=3.8" summary = "HTTP library with thread-safe connection pooling, file post, and more." groups = ["default", "dev"] files = [ - {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, - {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, ] [[package]] name = "uwsgi" -version = "2.0.25.1" +version = "2.0.26" summary = "The uWSGI server" groups = ["default"] files = [ - {file = "uwsgi-2.0.25.1.tar.gz", hash = "sha256:d653d2d804c194c8cbe2585fa56efa2650313ae75c686a9d7931374d4dfbfc6e"}, + {file = "uwsgi-2.0.26.tar.gz", hash = "sha256:86e6bfcd4dc20529665f5b7777193cdc48622fb2c59f0a7f1e3dc32b3882e7f9"}, ] [[package]] @@ -2438,18 +2543,19 @@ files = [ [[package]] name = "virtualenv" -version = "20.26.2" +version = "20.26.3" requires_python = ">=3.7" summary = "Virtual Python Environment builder" groups = ["dev"] dependencies = [ "distlib<1,>=0.3.7", "filelock<4,>=3.12.2", + "importlib-metadata>=6.6; python_version < \"3.8\"", "platformdirs<5,>=3.9.1", ] files = [ - {file = "virtualenv-20.26.2-py3-none-any.whl", hash = "sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b"}, - {file = "virtualenv-20.26.2.tar.gz", hash = "sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c"}, + {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, + {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, ] [[package]] @@ -2473,12 +2579,6 @@ files = [ {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:093b23e6906a8b97051191a4a0c73a77ecc958121d42346274c6af6520dec175"}, {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:611be3904f9843f0529c35a3ff3fd617449463cb4b73b1633950b3d97fa4bfb7"}, {file = "watchdog-4.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:62c613ad689ddcb11707f030e722fa929f322ef7e4f18f5335d2b73c61a85c28"}, - {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0144c0ea9997b92615af1d94afc0c217e07ce2c14912c7b1a5731776329fcfc7"}, - {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:998d2be6976a0ee3a81fb8e2777900c28641fb5bfbd0c84717d89bca0addcdc5"}, - {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e7921319fe4430b11278d924ef66d4daa469fafb1da679a2e48c935fa27af193"}, - {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:f0de0f284248ab40188f23380b03b59126d1479cd59940f2a34f8852db710625"}, - {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bca36be5707e81b9e6ce3208d92d95540d4ca244c006b61511753583c81c70dd"}, - {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ab998f567ebdf6b1da7dc1e5accfaa7c6992244629c0fdaef062f43249bd8dee"}, {file = "watchdog-4.0.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:dddba7ca1c807045323b6af4ff80f5ddc4d654c8bce8317dde1bd96b128ed253"}, {file = "watchdog-4.0.1-py3-none-manylinux2014_armv7l.whl", hash = "sha256:4513ec234c68b14d4161440e07f995f231be21a09329051e67a2118a7a612d2d"}, {file = "watchdog-4.0.1-py3-none-manylinux2014_i686.whl", hash = "sha256:4107ac5ab936a63952dea2a46a734a23230aa2f6f9db1291bf171dac3ebd53c6"}, @@ -2497,6 +2597,9 @@ name = "wcwidth" version = "0.2.13" summary = "Measures the displayed width of unicode strings in a terminal" groups = ["default", "dev"] +dependencies = [ + "backports-functools-lru-cache>=1.2.1; python_version < \"3.2\"", +] files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, @@ -2529,6 +2632,20 @@ files = [ {file = "WebTest-3.0.0.tar.gz", hash = "sha256:54bd969725838d9861a9fa27f8d971f79d275d94ae255f5c501f53bb6d9929eb"}, ] +[[package]] +name = "wmctrl" +version = "0.5" +requires_python = ">=2.7" +summary = "A tool to programmatically control windows inside X" +groups = ["dev"] +dependencies = [ + "attrs", +] +files = [ + {file = "wmctrl-0.5-py2.py3-none-any.whl", hash = "sha256:ae695c1863a314c899e7cf113f07c0da02a394b968c4772e1936219d9234ddd7"}, + {file = "wmctrl-0.5.tar.gz", hash = "sha256:7839a36b6fe9e2d6fd22304e5dc372dbced2116ba41283ea938b2da57f53e962"}, +] + [[package]] name = "xlrd" version = "2.0.1" diff --git a/pyproject.toml b/pyproject.toml index ede66847..71797b64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ authors = [ {name = "Domenico DiNicola", email = "dom.dinicola@gmail.com"}, ] requires-python = ">=3.12" + dependencies = [ "Django", "celery[redis]", @@ -15,6 +16,7 @@ dependencies = [ "django-adminactions", "django-adminfilters", "django-celery-beat", + "django-celery-results>=2.5.1", "django-concurrency", "django-constance", "django-csp", @@ -28,23 +30,30 @@ dependencies = [ "djangorestframework", "drf-nested-routers>=0.94.1", "drf-spectacular[sidecar]", - "face-recognition>=1.3.0", - "opencv-python>=4.9.0.80", "psycopg2-binary>=2.9.9", "sentry-sdk[celery,django]>=2.2.1", "social-auth-app-django", "social-auth-core", + "opencv-contrib-python-headless>=4.10.0.84", + "face-recognition>=1.3.0", "unicef-security", "uwsgi>=2.0.25.1", - "drf-nested-routers>=0.94.1", - "face-recognition>=1.3.0", - "opencv-python>=4.9.0.80", - "django-celery-results>=2.5.1", "requests>=2.32.3", + "numpy>=1.26.4,<2.0.0", + "flower>=2.0.1", + "setuptools>=74.1.2", + "django-smart-env>=0.1.0", ] +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + [tool.pdm.build] -includes = [] +includes = ['src/hope_dedup_engine'] + +[tool.pdm] +distribution = true [tool.pdm.dev-dependencies] dev = [ @@ -62,6 +71,7 @@ dev = [ "isort", "mypy", "openpyxl-stubs", + "pdbpp", "pre-commit", "pytest", "pytest-celery>=1.0.0", diff --git a/src/hope_dedup_engine/apps/api/admin.py b/src/hope_dedup_engine/apps/api/admin.py deleted file mode 100644 index 7753cfe2..00000000 --- a/src/hope_dedup_engine/apps/api/admin.py +++ /dev/null @@ -1,13 +0,0 @@ -from django.contrib import admin - -from hope_dedup_engine.apps.api.models import ( - DeduplicationSet, - Duplicate, - HDEToken, - Image, -) - -admin.site.register(DeduplicationSet) -admin.site.register(Duplicate) -admin.site.register(HDEToken) -admin.site.register(Image) diff --git a/src/hope_dedup_engine/apps/api/admin/__init__.py b/src/hope_dedup_engine/apps/api/admin/__init__.py new file mode 100644 index 00000000..1f932710 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/admin/__init__.py @@ -0,0 +1,10 @@ +from django.contrib import admin + +from .deduplicationset import DeduplicationSetAdmin # noqa +from .duplicate import DuplicateAdmin # noqa +from .hdetoken import HDETokenAdmin # noqa +from .image import ImageAdmin # noqa + +admin.site.site_header = "HOPE Dedup Engine" +admin.site.site_title = "HOPE Deduplication Admin" +admin.site.index_title = "Welcome to the HOPE Deduplication Engine Admin" diff --git a/src/hope_dedup_engine/apps/api/admin/deduplicationset.py b/src/hope_dedup_engine/apps/api/admin/deduplicationset.py new file mode 100644 index 00000000..494d3a2c --- /dev/null +++ b/src/hope_dedup_engine/apps/api/admin/deduplicationset.py @@ -0,0 +1,55 @@ +from django.contrib.admin import ModelAdmin, register +from django.http import HttpRequest, HttpResponseRedirect +from django.urls import reverse + +from admin_extra_buttons.api import button +from admin_extra_buttons.mixins import ExtraButtonsMixin +from adminfilters.dates import DateRangeFilter +from adminfilters.filters import ChoicesFieldComboFilter, DjangoLookupFilter +from adminfilters.mixin import AdminFiltersMixin + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.utils.process import start_processing + + +@register(DeduplicationSet) +class DeduplicationSetAdmin(AdminFiltersMixin, ExtraButtonsMixin, ModelAdmin): + list_display = ( + "id", + "name", + "reference_pk", + "state_value", + "created_at", + "updated_at", + "deleted", + ) + readonly_fields = ( + "id", + "state_value", + "external_system", + "created_at", + "created_by", + "updated_at", + "updated_by", + "deleted", + ) + search_fields = ("name",) + list_filter = ( + ("state_value", ChoicesFieldComboFilter), + ("created_at", DateRangeFilter), + ("updated_at", DateRangeFilter), + DjangoLookupFilter, + ) + change_form_template = "admin/api/deduplicationset/change_form.html" + + def has_add_permission(self, request): + return False + + @button(label="Process") + def process(self, request: HttpRequest, pk: str) -> HttpResponseRedirect: + dd = DeduplicationSet.objects.get(pk=pk) + start_processing(dd) + self.message_user( + request, f"Processing for deduplication set '{dd}' has been started." + ) + return HttpResponseRedirect(reverse("admin:api_deduplicationset_changelist")) diff --git a/src/hope_dedup_engine/apps/api/admin/duplicate.py b/src/hope_dedup_engine/apps/api/admin/duplicate.py new file mode 100644 index 00000000..4ab05214 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/admin/duplicate.py @@ -0,0 +1,35 @@ +from django.contrib.admin import ModelAdmin, register + +from adminfilters.filters import ( + DjangoLookupFilter, + NumberFilter, + RelatedFieldComboFilter, +) +from adminfilters.mixin import AdminFiltersMixin + +from hope_dedup_engine.apps.api.models import Duplicate + + +@register(Duplicate) +class DuplicateAdmin(AdminFiltersMixin, ModelAdmin): + list_display = ( + "id", + "deduplication_set", + "score", + "first_reference_pk", + "second_reference_pk", + ) + list_filter = ( + ("deduplication_set", RelatedFieldComboFilter), + ("score", NumberFilter), + DjangoLookupFilter, + ) + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return obj is not None diff --git a/src/hope_dedup_engine/apps/api/admin/hdetoken.py b/src/hope_dedup_engine/apps/api/admin/hdetoken.py new file mode 100644 index 00000000..d299bd4c --- /dev/null +++ b/src/hope_dedup_engine/apps/api/admin/hdetoken.py @@ -0,0 +1,8 @@ +from django.contrib.admin import ModelAdmin, register + +from hope_dedup_engine.apps.api.models import HDEToken + + +@register(HDEToken) +class HDETokenAdmin(ModelAdmin): + pass diff --git a/src/hope_dedup_engine/apps/api/admin/image.py b/src/hope_dedup_engine/apps/api/admin/image.py new file mode 100644 index 00000000..8b721863 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/admin/image.py @@ -0,0 +1,32 @@ +from django.contrib.admin import ModelAdmin, register + +from adminfilters.dates import DateRangeFilter +from adminfilters.filters import DjangoLookupFilter, RelatedFieldComboFilter +from adminfilters.mixin import AdminFiltersMixin + +from hope_dedup_engine.apps.api.models import Image + + +@register(Image) +class ImageAdmin(AdminFiltersMixin, ModelAdmin): + list_display = ( + "id", + "filename", + "deduplication_set", + "created_at", + ) + + list_filter = ( + ("deduplication_set", RelatedFieldComboFilter), + ("created_at", DateRangeFilter), + DjangoLookupFilter, + ) + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return obj is not None diff --git a/src/hope_dedup_engine/apps/api/celery_tasks.py b/src/hope_dedup_engine/apps/api/celery_tasks.py new file mode 100644 index 00000000..2e323d29 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/celery_tasks.py @@ -0,0 +1,3 @@ +from hope_dedup_engine.apps.api.deduplication.process import ( # noqa: F401 + find_duplicates, +) diff --git a/src/hope_dedup_engine/apps/api/const.py b/src/hope_dedup_engine/apps/api/const.py index a5c8aeb1..573cb593 100644 --- a/src/hope_dedup_engine/apps/api/const.py +++ b/src/hope_dedup_engine/apps/api/const.py @@ -15,5 +15,8 @@ DUPLICATE = "duplicate" DUPLICATE_LIST = f"{DUPLICATE}s" -IGNORED_KEYS = "ignore" -IGNORED_KEYS_LIST = f"{IGNORED_KEYS}s" +IGNORED = "ignored" +REFERENCE_PK = "reference_pk" +FILENAME = "filename" +IGNORED_REFERENCE_PK_LIST = f"{IGNORED}/{REFERENCE_PK}s" +IGNORED_FILENAME_LIST = f"{IGNORED}/{FILENAME}s" diff --git a/src/hope_dedup_engine/apps/api/deduplication/__init__.py b/src/hope_dedup_engine/apps/api/deduplication/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hope_dedup_engine/apps/api/deduplication/adapters.py b/src/hope_dedup_engine/apps/api/deduplication/adapters.py new file mode 100644 index 00000000..9deaf9e0 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/deduplication/adapters.py @@ -0,0 +1,36 @@ +from collections.abc import Generator + +from constance import config + +from hope_dedup_engine.apps.api.deduplication.registry import DuplicateKeyPair +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.faces.services.duplication_detector import ( + DuplicationDetector, +) + + +class DuplicateFaceFinder: + weight = 1 + + def __init__(self, deduplication_set: DeduplicationSet): + self.deduplication_set = deduplication_set + + def run(self) -> Generator[DuplicateKeyPair, None, None]: + filename_to_reference_pk = { + filename: reference_pk + for reference_pk, filename in self.deduplication_set.image_set.values_list( + "reference_pk", "filename" + ) + } + face_distance_threshold: float = ( + self.deduplication_set.config + and self.deduplication_set.config.face_distance_threshold + ) or config.FACE_DISTANCE_THRESHOLD + # ignored key pairs are not handled correctly in DuplicationDetector + detector = DuplicationDetector( + tuple[str](filename_to_reference_pk.keys()), face_distance_threshold + ) + for first_filename, second_filename, distance in detector.find_duplicates(): + yield filename_to_reference_pk[first_filename], filename_to_reference_pk[ + second_filename + ], 1 - distance diff --git a/src/hope_dedup_engine/apps/api/deduplication/lock.py b/src/hope_dedup_engine/apps/api/deduplication/lock.py new file mode 100644 index 00000000..88ddf3f6 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/deduplication/lock.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from base64 import b64decode, b64encode +from typing import Final, Self + +from django.core.cache import cache + +from constance import config +from redis.exceptions import LockNotOwnedError +from redis.lock import Lock + +from hope_dedup_engine.apps.api.models import DeduplicationSet + +DELIMITER: Final[str] = "|" +LOCK_IS_NOT_ENABLED = "LOCK_IS_NOT_ENABLED" + + +class DeduplicationSetLock: + """ + A lock used to limit access to a specific deduplication set. + This lock can be serialized, passed to Celery worker, and then deserialized. + """ + + class LockNotOwnedException(Exception): + pass + + def __init__(self, name: str, token: bytes | None = None) -> None: + # we heavily rely on Redis being used as a cache framework backend. + redis = cache._cache.get_client() + lock = Lock( + redis, + name, + blocking=False, + thread_local=False, + timeout=config.DEDUPLICATION_SET_LAST_ACTION_TIMEOUT, + ) + + if token is None: + # new lock + if not lock.acquire(): + raise self.LockNotOwnedException + else: + # deserialized lock + lock.local.token = token + if not lock.owned(): + raise DeduplicationSetLock.LockNotOwnedException + + self.lock = lock + + def __str__(self) -> str: + name_bytes, token_bytes = self.lock.name.encode(), self.lock.local.token + encoded = map(b64encode, (name_bytes, token_bytes)) + string_values = map(bytes.decode, encoded) + return DELIMITER.join(string_values) + + def refresh(self) -> None: + try: + self.lock.extend(config.DEDUPLICATION_SET_LAST_ACTION_TIMEOUT, True) + except LockNotOwnedError as e: + raise self.LockNotOwnedException from e + + def release(self) -> None: + try: + self.lock.release() + except LockNotOwnedError as e: + raise self.LockNotOwnedException from e + + @classmethod + def for_deduplication_set( + cls: type[Self], deduplication_set: DeduplicationSet + ) -> Self: + return cls(f"lock:{deduplication_set.pk}") + + @classmethod + def from_string(cls: type[Self], serialized: str) -> Self: + name_bytes, token_bytes = map(b64decode, serialized.split(DELIMITER)) + return cls(name_bytes.decode(), token_bytes) diff --git a/src/hope_dedup_engine/apps/api/deduplication/process.py b/src/hope_dedup_engine/apps/api/deduplication/process.py new file mode 100644 index 00000000..c6a69737 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/deduplication/process.py @@ -0,0 +1,103 @@ +from celery import shared_task +from constance import config + +from hope_dedup_engine.apps.api.deduplication.lock import DeduplicationSetLock +from hope_dedup_engine.apps.api.deduplication.registry import ( + DuplicateFinder, + DuplicateKeyPair, + get_finders, +) +from hope_dedup_engine.apps.api.models import DeduplicationSet, Duplicate + + +def _sort_keys(pair: DuplicateKeyPair) -> DuplicateKeyPair: + first, second, score = pair + return *sorted((first, second)), score + + +def _save_duplicates( + finder: DuplicateFinder, + deduplication_set: DeduplicationSet, + lock_enabled: bool, + lock: DeduplicationSetLock, +) -> None: + reference_pk_to_filename_mapping = dict( + deduplication_set.image_set.values_list("reference_pk", "filename") + ) + ignored_filename_pairs = frozenset( + map( + tuple, + map( + sorted, + deduplication_set.ignoredfilenamepair_set.values_list( + "first", "second" + ), + ), + ) + ) + + ignored_reference_pk_pairs = frozenset( + deduplication_set.ignoredreferencepkpair_set.values_list("first", "second") + ) + + for first, second, score in map(_sort_keys, finder.run()): + first_filename, second_filename = sorted( + ( + reference_pk_to_filename_mapping[first], + reference_pk_to_filename_mapping[second], + ) + ) + ignored = (first, second) in ignored_reference_pk_pairs or ( + first_filename, + second_filename, + ) in ignored_filename_pairs + if not ignored: + duplicate, _ = Duplicate.objects.get_or_create( + deduplication_set=deduplication_set, + first_reference_pk=first, + second_reference_pk=second, + ) + duplicate.score += score * finder.weight + duplicate.save() + if lock_enabled: + lock.refresh() + + +HOUR = 60 * 60 + + +@shared_task(soft_time_limit=0.5 * HOUR, time_limit=1 * HOUR) +def find_duplicates(deduplication_set_id: str, serialized_lock: str) -> None: + deduplication_set = DeduplicationSet.objects.get(pk=deduplication_set_id) + try: + lock_enabled = config.DEDUPLICATION_SET_LOCK_ENABLED + lock = ( + DeduplicationSetLock.from_string(serialized_lock) if lock_enabled else None + ) + + if lock_enabled: + # refresh lock in case we spent much time waiting in queue + lock.refresh() + + # clean results + Duplicate.objects.filter(deduplication_set=deduplication_set).delete() + + weight_total = 0 + for finder in get_finders(deduplication_set): + _save_duplicates(finder, deduplication_set, lock_enabled, lock) + weight_total += finder.weight + + for duplicate in deduplication_set.duplicate_set.all(): + duplicate.score /= weight_total + duplicate.save() + + deduplication_set.state = deduplication_set.State.CLEAN + deduplication_set.save() + + if lock_enabled: + lock.release() + + except Exception: + deduplication_set.state = DeduplicationSet.State.ERROR + deduplication_set.save() + raise diff --git a/src/hope_dedup_engine/apps/api/deduplication/registry.py b/src/hope_dedup_engine/apps/api/deduplication/registry.py new file mode 100644 index 00000000..9886ab49 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/deduplication/registry.py @@ -0,0 +1,18 @@ +from collections.abc import Generator, Iterable +from typing import Protocol + +from hope_dedup_engine.apps.api.models import DeduplicationSet + +DuplicateKeyPair = tuple[str, str, float] + + +class DuplicateFinder(Protocol): + weight: int + + def run(self) -> Generator[DuplicateKeyPair, None, None]: ... + + +def get_finders(deduplication_set: DeduplicationSet) -> Iterable[DuplicateFinder]: + from hope_dedup_engine.apps.api.deduplication.adapters import DuplicateFaceFinder + + return (DuplicateFaceFinder(deduplication_set),) diff --git a/src/hope_dedup_engine/apps/api/migrations/0002_remove_duplicate_first_filename_and_more.py b/src/hope_dedup_engine/apps/api/migrations/0002_remove_duplicate_first_filename_and_more.py new file mode 100644 index 00000000..8578ca0c --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0002_remove_duplicate_first_filename_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 5.0.6 on 2024-07-25 09:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="duplicate", + name="first_filename", + ), + migrations.RemoveField( + model_name="duplicate", + name="second_filename", + ), + migrations.AlterField( + model_name="duplicate", + name="score", + field=models.FloatField(default=0), + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/0003_remove_deduplicationset_name.py b/src/hope_dedup_engine/apps/api/migrations/0003_remove_deduplicationset_name.py new file mode 100644 index 00000000..72c01f68 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0003_remove_deduplicationset_name.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.6 on 2024-08-07 14:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0002_remove_duplicate_first_filename_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="deduplicationset", + name="name", + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/0004_remove_deduplicationset_error_and_more.py b/src/hope_dedup_engine/apps/api/migrations/0004_remove_deduplicationset_error_and_more.py new file mode 100644 index 00000000..4d1cb7ea --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0004_remove_deduplicationset_error_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.7 on 2024-09-24 08:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0003_remove_deduplicationset_name"), + ] + + operations = [ + migrations.RemoveField( + model_name="deduplicationset", + name="error", + ), + migrations.AddField( + model_name="deduplicationset", + name="description", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="deduplicationset", + name="name", + field=models.CharField( + blank=True, db_index=True, max_length=128, null=True, unique=True + ), + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/0005_config_deduplicationset_config.py b/src/hope_dedup_engine/apps/api/migrations/0005_config_deduplicationset_config.py new file mode 100644 index 00000000..ddd414cd --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0005_config_deduplicationset_config.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0.7 on 2024-09-24 09:05 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0004_remove_deduplicationset_error_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="Config", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("face_distance_threshold", models.FloatField(null=True)), + ], + ), + migrations.AddField( + model_name="deduplicationset", + name="config", + field=models.OneToOneField( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="api.config" + ), + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/0006_alter_deduplicationset_state_and_more.py b/src/hope_dedup_engine/apps/api/migrations/0006_alter_deduplicationset_state_and_more.py new file mode 100644 index 00000000..01d7f3d9 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0006_alter_deduplicationset_state_and_more.py @@ -0,0 +1,27 @@ +# Generated by Django 5.0.7 on 2024-09-25 06:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0005_config_deduplicationset_config"), + ] + + operations = [ + migrations.AlterField( + model_name="deduplicationset", + name="state", + field=models.IntegerField( + choices=[(0, "Clean"), (1, "Dirty"), (2, "Processing"), (3, "Error")], + db_column="state", + default=0, + ), + ), + migrations.RenameField( + model_name="deduplicationset", + old_name="state", + new_name="state_value", + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more.py b/src/hope_dedup_engine/apps/api/migrations/0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more.py new file mode 100644 index 00000000..908b7ce1 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 5.0.7 on 2024-09-25 10:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0006_alter_deduplicationset_state_and_more"), + ] + + operations = [ + migrations.RenameModel( + old_name="IgnoredKeyPair", + new_name="IgnoredReferencePkPair", + ), + migrations.RenameField( + model_name="ignoredreferencepkpair", + old_name="first_reference_pk", + new_name="first", + ), + migrations.RenameField( + model_name="ignoredreferencepkpair", + old_name="second_reference_pk", + new_name="second", + ), + migrations.AlterUniqueTogether( + name="ignoredreferencepkpair", + unique_together={("deduplication_set", "first", "second")}, + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/0008_ignoredfilenamepair.py b/src/hope_dedup_engine/apps/api/migrations/0008_ignoredfilenamepair.py new file mode 100644 index 00000000..052cd42a --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0008_ignoredfilenamepair.py @@ -0,0 +1,40 @@ +# Generated by Django 5.0.7 on 2024-09-25 11:24 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="IgnoredFilenamePair", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("first", models.CharField(max_length=100)), + ("second", models.CharField(max_length=100)), + ( + "deduplication_set", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.deduplicationset", + ), + ), + ], + options={ + "unique_together": {("deduplication_set", "first", "second")}, + }, + ), + ] diff --git a/src/hope_dedup_engine/apps/api/models/auth.py b/src/hope_dedup_engine/apps/api/models/auth.py index 050a852b..901ebd99 100644 --- a/src/hope_dedup_engine/apps/api/models/auth.py +++ b/src/hope_dedup_engine/apps/api/models/auth.py @@ -5,6 +5,10 @@ class HDEToken(Token): + """ + Token model for user to integrate with HOPE + """ + user = models.ForeignKey( settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE ) diff --git a/src/hope_dedup_engine/apps/api/models/deduplication.py b/src/hope_dedup_engine/apps/api/models/deduplication.py index bbeb8cbd..f42d2047 100644 --- a/src/hope_dedup_engine/apps/api/models/deduplication.py +++ b/src/hope_dedup_engine/apps/api/models/deduplication.py @@ -1,15 +1,24 @@ -from typing import Any, override +from typing import Any, Final, override from uuid import uuid4 from django.conf import settings from django.db import models +from hope_dedup_engine.apps.api.utils.notification import send_notification from hope_dedup_engine.apps.security.models import ExternalSystem -REFERENCE_PK_LENGTH = 100 +REFERENCE_PK_LENGTH: Final[int] = 100 + + +class Config(models.Model): + face_distance_threshold = models.FloatField(null=True) class DeduplicationSet(models.Model): + """ + Bucket for entries we want to deduplicate + """ + class State(models.IntegerChoices): CLEAN = 0, "Clean" # Deduplication set is created or already processed DIRTY = ( @@ -20,15 +29,18 @@ class State(models.IntegerChoices): ERROR = 3, "Error" # Error occurred id = models.UUIDField(primary_key=True, default=uuid4) - name = models.CharField(max_length=100) - reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) - state = models.IntegerField( + name = models.CharField( + max_length=128, unique=True, null=True, blank=True, db_index=True + ) + description = models.TextField(null=True, blank=True) + reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) # source_id + state_value = models.IntegerField( choices=State.choices, default=State.CLEAN, + db_column="state", ) deleted = models.BooleanField(null=False, blank=False, default=False) external_system = models.ForeignKey(ExternalSystem, on_delete=models.CASCADE) - error = models.CharField(max_length=255, null=True, blank=True) created_by = models.ForeignKey( settings.AUTH_USER_MODEL, on_delete=models.CASCADE, @@ -46,9 +58,27 @@ class State(models.IntegerChoices): ) updated_at = models.DateTimeField(auto_now=True) notification_url = models.CharField(max_length=255, null=True, blank=True) + config = models.OneToOneField(Config, null=True, on_delete=models.SET_NULL) + + @property + def state(self) -> State: + return self.State(self.state_value) + + @state.setter + def state(self, value: State) -> None: + if value != self.state_value or value == self.State.CLEAN: + self.state_value = value + send_notification(self.notification_url) + + def __str__(self) -> str: + return f"ID: {self.pk}" if not self.name else f"{self.name}" class Image(models.Model): + """ + # TODO: rename to Entity/Entry + """ + id = models.UUIDField(primary_key=True, default=uuid4) deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE) reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) @@ -64,29 +94,46 @@ class Image(models.Model): class Duplicate(models.Model): + """ + Couple of similar entities + """ + deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE) - first_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) - first_filename = models.CharField(max_length=255) - second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) - second_filename = models.CharField(max_length=255) - score = models.FloatField() + first_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) # from hope + second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) # from hope + score = models.FloatField(default=0) -class IgnoredKeyPair(models.Model): +class IgnoredPair(models.Model): deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE) - first_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) - second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) class Meta: - unique_together = ( - "deduplication_set", - "first_reference_pk", - "second_reference_pk", - ) + abstract = True @override def save(self, **kwargs: Any) -> None: - self.first_reference_pk, self.second_reference_pk = sorted( - (self.first_reference_pk, self.second_reference_pk) - ) + self.first, self.second = sorted((self.first, self.second)) super().save(**kwargs) + + +UNIQUE_FOR_IGNORED_PAIR = ( + "deduplication_set", + "first", + "second", +) + + +class IgnoredReferencePkPair(IgnoredPair): + first = models.CharField(max_length=REFERENCE_PK_LENGTH) + second = models.CharField(max_length=REFERENCE_PK_LENGTH) + + class Meta: + unique_together = UNIQUE_FOR_IGNORED_PAIR + + +class IgnoredFilenamePair(IgnoredPair): + first = models.CharField(max_length=REFERENCE_PK_LENGTH) + second = models.CharField(max_length=REFERENCE_PK_LENGTH) + + class Meta: + unique_together = UNIQUE_FOR_IGNORED_PAIR diff --git a/src/hope_dedup_engine/apps/api/serializers.py b/src/hope_dedup_engine/apps/api/serializers.py index fc9495c9..c178c7f1 100644 --- a/src/hope_dedup_engine/apps/api/serializers.py +++ b/src/hope_dedup_engine/apps/api/serializers.py @@ -4,18 +4,29 @@ from hope_dedup_engine.apps.api.models import DeduplicationSet from hope_dedup_engine.apps.api.models.deduplication import ( + Config, Duplicate, - IgnoredKeyPair, + IgnoredFilenamePair, + IgnoredReferencePkPair, Image, ) +CONFIG = "config" + + +class ConfigSerializer(serializers.ModelSerializer): + class Meta: + model = Config + exclude = ("id",) + class DeduplicationSetSerializer(serializers.ModelSerializer): - state = serializers.CharField(source="get_state_display", read_only=True) + state = serializers.CharField(source="get_state_value_display", read_only=True) + config = ConfigSerializer(required=False) class Meta: model = DeduplicationSet - exclude = ("deleted",) + exclude = ("deleted", "state_value") read_only_fields = ( "external_system", "created_at", @@ -25,17 +36,49 @@ class Meta: "updated_by", ) + def create(self, validated_data) -> DeduplicationSet: + config_data = validated_data.get(CONFIG) and validated_data.pop(CONFIG) + config = Config.objects.create(**config_data) if config_data else None + return DeduplicationSet.objects.create(config=config, **validated_data) + + +class CreateConfigSerializer(ConfigSerializer): + pass + + +class CreateDeduplicationSetSerializer(serializers.ModelSerializer): + config = CreateConfigSerializer(required=False) + + class Meta: + model = DeduplicationSet + fields = ("config", "reference_pk", "notification_url") + class ImageSerializer(serializers.ModelSerializer): class Meta: model = Image - fields = "__all__" + fields = ( + "id", + "deduplication_set", + "reference_pk", + "filename", + "created_by", + "created_at", + ) read_only_fields = "created_by", "created_at" +class CreateImageSerializer(serializers.ModelSerializer): + class Meta: + model = Image + fields = ( + "reference_pk", + "filename", + ) + + class EntrySerializer(serializers.Serializer): reference_pk = serializers.SerializerMethodField() - filename = serializers.SerializerMethodField() def __init__(self, prefix: str, *args: Any, **kwargs: Any) -> None: self._prefix = prefix @@ -44,16 +87,43 @@ def __init__(self, prefix: str, *args: Any, **kwargs: Any) -> None: def get_reference_pk(self, duplicate: Duplicate) -> int: return getattr(duplicate, f"{self._prefix}_reference_pk") - def get_filename(self, duplicate: Duplicate) -> str: - return getattr(duplicate, f"{self._prefix}_filename") - -class DuplicateSerializer(serializers.Serializer): +class DuplicateSerializer(serializers.ModelSerializer): first = EntrySerializer(prefix="first", source="*") second = EntrySerializer(prefix="second", source="*") + class Meta: + model = Duplicate + fields = "first", "second", "score" + + +CREATE_PAIR_FIELDS = "first", "second" +PAIR_FIELDS = ("id", "deduplication_set") + CREATE_PAIR_FIELDS + + +class IgnoredReferencePkPairSerializer(serializers.ModelSerializer): + class Meta: + model = IgnoredReferencePkPair + fields = PAIR_FIELDS + -class IgnoredKeyPairSerializer(serializers.ModelSerializer): +class CreateIgnoredReferencePkPairSerializer(serializers.ModelSerializer): class Meta: - model = IgnoredKeyPair - fields = "__all__" + model = IgnoredReferencePkPair + fields = CREATE_PAIR_FIELDS + + +class IgnoredFilenamePairSerializer(serializers.ModelSerializer): + class Meta: + model = IgnoredFilenamePair + fields = PAIR_FIELDS + + +class CreateIgnoredFilenamePairSerializer(serializers.ModelSerializer): + class Meta: + model = IgnoredFilenamePair + fields = CREATE_PAIR_FIELDS + + +class EmptySerializer(serializers.Serializer): + pass diff --git a/src/hope_dedup_engine/apps/api/urls.py b/src/hope_dedup_engine/apps/api/urls.py index 8bda942a..5414a839 100644 --- a/src/hope_dedup_engine/apps/api/urls.py +++ b/src/hope_dedup_engine/apps/api/urls.py @@ -1,5 +1,10 @@ -from django.urls import include, path +from django.urls import include, path, re_path +from drf_spectacular.views import ( + SpectacularAPIView, + SpectacularRedocView, + SpectacularSwaggerView, +) from rest_framework import routers from rest_framework_nested import routers as nested_routers @@ -8,14 +13,16 @@ DEDUPLICATION_SET, DEDUPLICATION_SET_LIST, DUPLICATE_LIST, - IGNORED_KEYS_LIST, + IGNORED_FILENAME_LIST, + IGNORED_REFERENCE_PK_LIST, IMAGE_LIST, ) from hope_dedup_engine.apps.api.views import ( BulkImageViewSet, DeduplicationSetViewSet, DuplicateViewSet, - IgnoredKeyPairViewSet, + IgnoredFilenamePairViewSet, + IgnoredReferencePkPairViewSet, ImageViewSet, ) @@ -35,10 +42,26 @@ DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST ) deduplication_sets_router.register( - IGNORED_KEYS_LIST, IgnoredKeyPairViewSet, basename=IGNORED_KEYS_LIST + IGNORED_FILENAME_LIST, IgnoredFilenamePairViewSet, basename=IGNORED_FILENAME_LIST +) +deduplication_sets_router.register( + IGNORED_REFERENCE_PK_LIST, + IgnoredReferencePkPairViewSet, + basename=IGNORED_REFERENCE_PK_LIST, ) urlpatterns = [ path("", include(router.urls)), path("", include(deduplication_sets_router.urls)), + path("api/rest/", SpectacularAPIView.as_view(), name="schema"), + re_path( + "^api/rest/swagger/$", + SpectacularSwaggerView.as_view(url_name="schema"), + name="swagger-ui", + ), + re_path( + "^api/rest/redoc/$", + SpectacularRedocView.as_view(url_name="schema"), + name="redoc", + ), ] diff --git a/src/hope_dedup_engine/apps/api/utils.py b/src/hope_dedup_engine/apps/api/utils.py deleted file mode 100644 index af45893e..00000000 --- a/src/hope_dedup_engine/apps/api/utils.py +++ /dev/null @@ -1,19 +0,0 @@ -import requests - -from hope_dedup_engine.apps.api.models import DeduplicationSet - - -def start_processing(_: DeduplicationSet) -> None: - pass - - -def delete_model_data(_: DeduplicationSet) -> None: - pass - - -REQUEST_TIMEOUT = 5 - - -def send_notification(deduplication_set: DeduplicationSet) -> None: - if url := deduplication_set.notification_url: - requests.get(url, timeout=REQUEST_TIMEOUT) diff --git a/src/hope_dedup_engine/apps/api/utils/__init__.py b/src/hope_dedup_engine/apps/api/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hope_dedup_engine/apps/api/utils/notification.py b/src/hope_dedup_engine/apps/api/utils/notification.py new file mode 100644 index 00000000..b8c948e8 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/utils/notification.py @@ -0,0 +1,15 @@ +from typing import Final + +import requests +import sentry_sdk + +REQUEST_TIMEOUT: Final[int] = 5 + + +def send_notification(url: str | None) -> None: + try: + if url: + with requests.get(url, timeout=REQUEST_TIMEOUT) as response: + response.raise_for_status() + except requests.RequestException as e: + sentry_sdk.capture_exception(e) diff --git a/src/hope_dedup_engine/apps/api/utils/process.py b/src/hope_dedup_engine/apps/api/utils/process.py new file mode 100644 index 00000000..fe721931 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/utils/process.py @@ -0,0 +1,34 @@ +from constance import config +from rest_framework import status +from rest_framework.exceptions import APIException + +from hope_dedup_engine.apps.api.deduplication.lock import LOCK_IS_NOT_ENABLED +from hope_dedup_engine.apps.api.models import DeduplicationSet + + +class AlreadyProcessingError(APIException): + status_code = status.HTTP_409_CONFLICT + default_detail = "Deduplication set is being processed already, try again later." + default_code = "already_processing" + + +def start_processing(deduplication_set: DeduplicationSet) -> None: + from hope_dedup_engine.apps.api.deduplication.lock import DeduplicationSetLock + from hope_dedup_engine.apps.api.deduplication.process import find_duplicates + + try: + lock = ( + DeduplicationSetLock.for_deduplication_set(deduplication_set) + if config.DEDUPLICATION_SET_LOCK_ENABLED + else LOCK_IS_NOT_ENABLED + ) + deduplication_set.state = DeduplicationSet.State.PROCESSING + deduplication_set.save() + find_duplicates.delay(str(deduplication_set.pk), str(lock)) + except DeduplicationSetLock.LockNotOwnedException as e: + raise AlreadyProcessingError from e + + +def delete_model_data(_: DeduplicationSet) -> None: + # TODO + pass diff --git a/src/hope_dedup_engine/apps/api/views.py b/src/hope_dedup_engine/apps/api/views.py index 705ea10c..0667deea 100644 --- a/src/hope_dedup_engine/apps/api/views.py +++ b/src/hope_dedup_engine/apps/api/views.py @@ -3,8 +3,10 @@ from typing import Any from uuid import UUID -from django.db.models import QuerySet +from django.db.models import Q, QuerySet +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated @@ -25,24 +27,27 @@ from hope_dedup_engine.apps.api.models import DeduplicationSet from hope_dedup_engine.apps.api.models.deduplication import ( Duplicate, - IgnoredKeyPair, + IgnoredFilenamePair, + IgnoredReferencePkPair, Image, ) from hope_dedup_engine.apps.api.serializers import ( + CreateDeduplicationSetSerializer, + CreateIgnoredFilenamePairSerializer, + CreateIgnoredReferencePkPairSerializer, + CreateImageSerializer, DeduplicationSetSerializer, DuplicateSerializer, - IgnoredKeyPairSerializer, + EmptySerializer, + IgnoredFilenamePairSerializer, + IgnoredReferencePkPairSerializer, ImageSerializer, ) -from hope_dedup_engine.apps.api.utils import delete_model_data, start_processing - -MESSAGE = "message" -STARTED = "started" -RETRYING = "retrying" -ALREADY_PROCESSING = "already processing" +from hope_dedup_engine.apps.api.utils.process import delete_model_data, start_processing class DeduplicationSetViewSet( + mixins.RetrieveModelMixin, mixins.ListModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, @@ -73,25 +78,34 @@ def perform_destroy(self, instance: DeduplicationSet) -> None: instance.save() delete_model_data(instance) - @staticmethod - def _start_processing(deduplication_set: DeduplicationSet) -> None: - Duplicate.objects.filter(deduplication_set=deduplication_set).delete() - start_processing(deduplication_set) - + @extend_schema( + request=EmptySerializer, + responses=EmptySerializer, + description="Run duplicate search process for the deduplication set", + ) @action(detail=True, methods=(HTTPMethod.POST,)) def process(self, request: Request, pk: UUID | None = None) -> Response: - deduplication_set = DeduplicationSet.objects.get(pk=pk) - match deduplication_set.state: - case DeduplicationSet.State.CLEAN | DeduplicationSet.State.ERROR: - self._start_processing(deduplication_set) - return Response({MESSAGE: RETRYING}) - case DeduplicationSet.State.DIRTY: - self._start_processing(deduplication_set) - return Response({MESSAGE: STARTED}) - case DeduplicationSet.State.PROCESSING: - return Response( - {MESSAGE: ALREADY_PROCESSING}, status=status.HTTP_400_BAD_REQUEST - ) + start_processing(DeduplicationSet.objects.get(pk=pk)) + return Response({"message": "started"}) + + @extend_schema(description="List all deduplication sets available to the user") + def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().list(request, *args, **kwargs) + + @extend_schema( + request=CreateDeduplicationSetSerializer, + description="Create new deduplication set", + ) + def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().create(request, *args, **kwargs) + + @extend_schema(description="Retrieve specific deduplication set") + def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().retrieve(request, *args, **kwargs) + + @extend_schema(description="Delete specific deduplication set") + def destroy(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().destroy(request, *args, **kwargs) class ImageViewSet( @@ -127,6 +141,20 @@ def perform_destroy(self, instance: Image) -> None: deduplication_set.updated_by = self.request.user deduplication_set.save() + @extend_schema(description="List all images for the deduplication set") + def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().list(request, *args, **kwargs) + + @extend_schema( + request=CreateImageSerializer, description="Add image to the deduplication set" + ) + def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().create(request, *args, **kwargs) + + @extend_schema(description="Delete image from the deduplication set") + def destroy(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().destroy(request, *args, **kwargs) + @dataclass class ListDataWrapper: @@ -187,6 +215,7 @@ def perform_create(self, serializer: Serializer) -> None: deduplication_set.updated_by = self.request.user deduplication_set.save() + @extend_schema(description="Delete all images from deduplication set") @action(detail=False, methods=(HTTPMethod.DELETE,)) def clear(self, request: Request, deduplication_set_pk: str) -> Response: deduplication_set = DeduplicationSet.objects.get(pk=deduplication_set_pk) @@ -195,6 +224,16 @@ def clear(self, request: Request, deduplication_set_pk: str) -> Response: deduplication_set.save() return Response(status=status.HTTP_204_NO_CONTENT) + @extend_schema( + request=CreateImageSerializer(many=True), + description="Add multiple images to the deduplication set", + ) + def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().create(request, *args, **kwargs) + + +REFERENCE_PK = "reference_pk" + class DuplicateViewSet( nested_viewsets.NestedViewSetMixin[Duplicate], @@ -213,9 +252,31 @@ class DuplicateViewSet( DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER, } + def get_queryset(self) -> QuerySet[Duplicate]: + queryset = super().get_queryset() + if reference_pk := self.request.query_params.get(REFERENCE_PK): + return queryset.filter( + Q(first_reference_pk=reference_pk) | Q(second_reference_pk=reference_pk) + ) + return queryset + + @extend_schema( + description="List all duplicates found in the deduplication set", + parameters=[ + OpenApiParameter( + REFERENCE_PK, + OpenApiTypes.STR, + OpenApiParameter.QUERY, + description="Filters results by reference pk", + ) + ], + ) + def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().list(request, *args, **kwargs) + -class IgnoredKeyPairViewSet( - nested_viewsets.NestedViewSetMixin[IgnoredKeyPair], +class IgnoredPairViewSet[T]( + nested_viewsets.NestedViewSetMixin[T], mixins.ListModelMixin, mixins.CreateModelMixin, viewsets.GenericViewSet, @@ -226,8 +287,6 @@ class IgnoredKeyPairViewSet( AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem, ) - serializer_class = IgnoredKeyPairSerializer - queryset = IgnoredKeyPair.objects.all() parent_lookup_kwargs = { DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER, } @@ -238,3 +297,39 @@ def perform_create(self, serializer: Serializer) -> None: deduplication_set.state = DeduplicationSet.State.DIRTY deduplication_set.updated_by = self.request.user deduplication_set.save() + + +class IgnoredFilenamePairViewSet(IgnoredPairViewSet[IgnoredFilenamePair]): + serializer_class = IgnoredFilenamePairSerializer + queryset = IgnoredFilenamePair.objects.all() + + @extend_schema( + description="List all ignored filename pairs for the deduplication set" + ) + def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().list(request, *args, **kwargs) + + @extend_schema( + request=CreateIgnoredFilenamePairSerializer, + description="Add ignored filename pair for the deduplication set", + ) + def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().create(request, *args, **kwargs) + + +class IgnoredReferencePkPairViewSet(IgnoredPairViewSet[IgnoredReferencePkPair]): + serializer_class = IgnoredReferencePkPairSerializer + queryset = IgnoredReferencePkPair.objects.all() + + @extend_schema( + description="List all ignored reference pk pairs for the deduplication set" + ) + def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().list(request, *args, **kwargs) + + @extend_schema( + request=CreateIgnoredReferencePkPairSerializer, + description="Add ignored reference pk pair for the deduplication set", + ) + def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().create(request, *args, **kwargs) diff --git a/src/hope_dedup_engine/apps/core/apps.py b/src/hope_dedup_engine/apps/core/apps.py index 8b725e06..c7a275c4 100644 --- a/src/hope_dedup_engine/apps/core/apps.py +++ b/src/hope_dedup_engine/apps/core/apps.py @@ -8,3 +8,5 @@ class Config(AppConfig): def ready(self) -> None: super().ready() from hope_dedup_engine.utils import flags # noqa + + from . import checks # noqa diff --git a/src/hope_dedup_engine/apps/core/checks.py b/src/hope_dedup_engine/apps/core/checks.py new file mode 100644 index 00000000..a73348f8 --- /dev/null +++ b/src/hope_dedup_engine/apps/core/checks.py @@ -0,0 +1,128 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from django.conf import settings +from django.core.checks import Error, register + +from storages.backends.azure_storage import AzureStorage + +from hope_dedup_engine.config import env + + +@dataclass(frozen=True, slots=True) +class ErrorCode: + id: str + message: str + hint: str + + +class StorageErrorCodes: # pragma: no cover + ENVIRONMENT_NOT_CONFIGURED = ErrorCode( + id="hde.storage.E001", + message="The environment variable '{storage}' is missing or improperly defined.", + hint="Set the environment variable '{storage}'.\n\tExample: {storage}=storages.backends.azure_storage.AzureStorage?account_name=&account_key=&azure_container=&overwrite_files=True", # noqa: E501 + ) + STORAGE_CHECK_FAILED = ErrorCode( + id="hde.storage.E002", + message="Failed to access Azure storage due to incorrect data in the '{storage_name}' environment variable.", + hint="Verify the '{storage_name}' variable and ensure that the provided parameters are accurate.\n\tExample: {storage_name}=storages.backends.azure_storage.AzureStorage?account_name=&account_key=&azure_container=&overwrite_files=True", # noqa: E501 + ) + FILE_NOT_FOUND = ErrorCode( + id="hde.storage.E003", + message="The file '{filename}' could not be found in the Azure storage specified by the environment variable '{storage_name}'.", # noqa: E501 + hint="Ensure that the file '{filename}' exists in the storage. For details, refer to the documentation.", + ) + + +@register() +def example_check(app_configs, **kwargs: Any): # pragma: no cover + errors = [] + for t in settings.TEMPLATES: + for d in t["DIRS"]: + if not Path(d).is_dir(): + errors.append( + Error( + f"'{d}' is not a directory", + hint="Remove this directory from settings.TEMPLATES.", + obj=settings, + id="hde.E001", + ) + ) + return errors + + +@register(deploy=True) +def storages_check(app_configs: Any, **kwargs: Any) -> list[Error]: # pragma: no cover + """ + Checks if the necessary environment variables for Azure storage are configured + and verifies the presence of required files in the specified Azure storage containers. + + Args: + app_configs: Not used, but required by the checks framework. + kwargs: Additional arguments passed by the checks framework. + + Returns: + list[Error]: A list of Django Error objects, reporting missing environment variables, + missing files, or errors while accessing Azure storage containers. + """ + storages = ( + "FILE_STORAGE_DNN", + "FILE_STORAGE_HOPE", + "FILE_STORAGE_STATIC", + "FILE_STORAGE_MEDIA", + ) + + errors = [ + Error( + StorageErrorCodes.ENVIRONMENT_NOT_CONFIGURED.message.format( + storage=storage + ), + hint=StorageErrorCodes.ENVIRONMENT_NOT_CONFIGURED.hint.format( + storage=storage + ), + obj=storage, + id=StorageErrorCodes.ENVIRONMENT_NOT_CONFIGURED.id, + ) + for storage in storages + if not env.storage(storage).get("OPTIONS") + ] + + for storage_name in storages: + options = env.storage(storage_name).get("OPTIONS") + if options: + try: + storage = AzureStorage(**options) + storage.client.exists() + if storage_name == "FILE_STORAGE_DNN": + _, files = storage.listdir() + for _, info in settings.DNN_FILES.items(): + filename = info.get("filename") + if filename not in files: + errors.append( + Error( + StorageErrorCodes.FILE_NOT_FOUND.message.format( + filename=filename, storage_name=storage_name + ), + hint=StorageErrorCodes.FILE_NOT_FOUND.hint.format( + filename=filename + ), + obj=filename, + id=StorageErrorCodes.FILE_NOT_FOUND.id, + ) + ) + except Exception: + errors.append( + Error( + StorageErrorCodes.STORAGE_CHECK_FAILED.message.format( + storage_name=storage_name + ), + hint=StorageErrorCodes.STORAGE_CHECK_FAILED.hint.format( + storage_name=storage_name + ), + obj=storage_name, + id=StorageErrorCodes.STORAGE_CHECK_FAILED.id, + ) + ) + + return errors diff --git a/src/hope_dedup_engine/apps/core/exceptions.py b/src/hope_dedup_engine/apps/core/exceptions.py new file mode 100644 index 00000000..365ffc75 --- /dev/null +++ b/src/hope_dedup_engine/apps/core/exceptions.py @@ -0,0 +1,18 @@ +class StorageKeyError(Exception): + """ + Exception raised when the storage key does not exist. + """ + + def __init__(self, key: str) -> None: + self.key = key + super().__init__(f"Storage key '{key}' does not exist.") + + +class DownloaderKeyError(Exception): + """ + Exception raised when the downloader key does not exist. + """ + + def __init__(self, key: str) -> None: + self.key = key + super().__init__(f"Downloader key '{key}' does not exist.") diff --git a/src/hope_dedup_engine/apps/core/management/commands/createsystem.py b/src/hope_dedup_engine/apps/core/management/commands/createsystem.py index f9dafbac..eb436ab1 100644 --- a/src/hope_dedup_engine/apps/core/management/commands/createsystem.py +++ b/src/hope_dedup_engine/apps/core/management/commands/createsystem.py @@ -3,7 +3,7 @@ from hope_dedup_engine.apps.core.models import ExternalSystem -class Command(BaseCommand): +class Command(BaseCommand): # pragma: no cover help = "Creates external system" def add_arguments(self, parser): diff --git a/src/hope_dedup_engine/apps/core/management/commands/demo.py b/src/hope_dedup_engine/apps/core/management/commands/demo.py new file mode 100644 index 00000000..1a1f1b67 --- /dev/null +++ b/src/hope_dedup_engine/apps/core/management/commands/demo.py @@ -0,0 +1,131 @@ +import logging +import sys +from argparse import ArgumentParser +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Final + +from django.core.management import BaseCommand +from django.core.management.base import CommandError, SystemCheckError + +from hope_dedup_engine.config import env + +from .utils.azurite_manager import AzuriteManager + +logger = logging.getLogger(__name__) + + +BASE_PATH: Final[Path] = ( + Path(__file__).resolve().parents[6] / "tests" / "extras" / "demoapp" +) +DEFAULT_DEMO_IMAGES: Final[Path] = BASE_PATH / env("DEMO_IMAGES_PATH") +DEFAULT_DNN_FILES: Final[Path] = BASE_PATH / env("DNN_FILES_PATH") + +MESSAGES: Final[dict[str, str]] = { + "upload": "Starting upload of files...", + "not_exist": "Directory '%s' does not exist.", + "container_success": "Container for storage '%s' created successfully.", + "storage_success": "Files uploaded to storage '%s' successfully.", + "success": "Finished uploading files to storage.", + "failed": "Failed to upload files to storage '%s': %s", + "unexpected": "An unexpected error occurred while uploading files to storage '%s': %s", + "halted": "\n\n***\nSYSTEM HALTED", +} + + +@dataclass(frozen=True) +class Storage: + name: str + src: Path | None = field(default=None) + options: dict[str, str] = field(default_factory=dict) + + +class Command(BaseCommand): + help = "Create demo app" + + def add_arguments(self, parser: ArgumentParser) -> None: + """ + Define the command-line arguments that this command accepts. + + Args: + parser (ArgumentParser): The parser for command-line arguments. + """ + parser.add_argument( + "--demo-images", + type=str, + default=str(DEFAULT_DEMO_IMAGES), + help="Path to the demo images directory", + ) + parser.add_argument( + "--dnn-files", + type=str, + default=str(DEFAULT_DNN_FILES), + help="Path to the DNN files directory", + ) + + def handle(self, *args: Any, **options: dict[str, Any]) -> None: + """ + Main logic for handling the command to create containers and upload files to Azurite Storage. + + Args: + *args (Any): Positional arguments passed to the command. + **options (dict[str, Any]): Keyword arguments including command-line options. + + Raises: + FileNotFoundError: If a specified directory does not exist. + CommandError: If a Django management command error occurs. + SystemCheckError: If a Django system check error occurs. + Exception: For any other unexpected errors that may arise during the execution of the command. + + """ + storages = ( + Storage(name="hope", src=Path(options["demo_images"])), + Storage(name="dnn", src=Path(options["dnn_files"])), + Storage(name="media"), + Storage(name="staticfiles", options={"public_access": "blob"}), + ) + self.stdout.write(self.style.WARNING(MESSAGES["upload"])) + logger.info(MESSAGES["upload"]) + + for storage in storages: + try: + am = AzuriteManager(storage.name, storage.options) + self.stdout.write(MESSAGES["container_success"] % storage.name) + if storage.src is None: + continue + if storage.src.exists(): + am.upload_files(storage.src) + else: + self.stdout.write( + self.style.ERROR(MESSAGES["not_exist"] % storage.src) + ) + logger.error(MESSAGES["not_exist"] % storage.src) + self.halt(FileNotFoundError(MESSAGES["not_exist"] % storage.src)) + self.stdout.write(MESSAGES["storage_success"] % storage.name) + logger.info(MESSAGES["storage_success"] % storage.name) + except (CommandError, SystemCheckError) as e: + self.stdout.write( + self.style.ERROR(MESSAGES["failed"] % (storage.name, e)) + ) + logger.error(MESSAGES["failed"] % (storage.name, e)) + self.halt(e) + except Exception as e: + self.stdout.write( + self.style.ERROR(MESSAGES["unexpected"] % (storage.name, e)) + ) + logger.exception(MESSAGES["unexpected"] % (storage.name, e)) + self.halt(e) + + self.stdout.write(self.style.SUCCESS(MESSAGES["success"])) + + def halt(self, e: Exception) -> None: + """ + Handle an exception by logging the error and exiting the program. + + Args: + e (Exception): The exception that occurred. + """ + logger.exception(e) + self.stdout.write(str(e), style_func=self.style.ERROR) + self.stdout.write(MESSAGES["halted"], style_func=self.style.ERROR) + sys.exit(1) diff --git a/src/hope_dedup_engine/apps/core/management/commands/dnnsetup.py b/src/hope_dedup_engine/apps/core/management/commands/dnnsetup.py new file mode 100644 index 00000000..8c28bf38 --- /dev/null +++ b/src/hope_dedup_engine/apps/core/management/commands/dnnsetup.py @@ -0,0 +1,166 @@ +import logging +import sys +from argparse import ArgumentParser +from typing import Any, Final + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.core.management import BaseCommand +from django.core.management.base import CommandError, SystemCheckError + +import requests +from storages.backends.azure_storage import AzureStorage + +logger = logging.getLogger(__name__) + + +MESSAGES: Final[dict[str, str]] = { + "start": "Starting DNN setup...", + "already": "File '%s' already exists in 'FILE_STORAGE_DNN' storage.", + "process": "Downloading file from '%s' to '%s' in 'FILE_STORAGE_DNN' storage...", + "empty": "File at '%s' is empty (size is 0 bytes).", + "completed": "DNN setup completed successfully.", + "halted": "\n\n***\nSYSTEM HALTED\nUnable to start without DNN files...", +} + + +class Command(BaseCommand): + help = "Synchronizes DNN files from the git to azure storage" + dnn_files = None + + def add_arguments(self, parser: ArgumentParser) -> None: + """ + Adds custom command-line arguments to the management command. + + Args: + parser (ArgumentParser): The argument parser instance to which the arguments should be added. + + Adds the following arguments: + --force: A boolean flag that, when provided, forces the re-download of files even if they already exist + in Azure storage. Defaults to False. + --deployfile-url (str): The URL from which the deploy (prototxt) file is downloaded. + Defaults to the value set in the project settings. + --caffemodelfile-url (str): The URL from which the pre-trained model weights (caffemodel) are downloaded. + Defaults to the value set in the project settings. + --download-timeout (int): The maximum time allowed for downloading files, in seconds. + Defaults to 3 minutes (180 seconds). + --chunk-size (int): The size of each chunk to download in bytes. Defaults to 256 KB. + """ + parser.add_argument( + "--force", + action="store_true", + default=False, + help="Force the re-download of files even if they already exist", + ) + parser.add_argument( + "--deployfile-url", + type=str, + default=settings.DNN_FILES.get("prototxt", {}) + .get("sources", {}) + .get("github"), + help="The URL of the model architecture (deploy) file", + ) + parser.add_argument( + "--caffemodelfile-url", + type=str, + default=settings.DNN_FILES.get("caffemodel", {}) + .get("sources", {}) + .get("github"), + help="The URL of the pre-trained model weights (caffemodel) file", + ) + parser.add_argument( + "--download-timeout", + type=int, + default=3 * 60, # 3 minutes + help="The timeout for downloading files", + ) + parser.add_argument( + "--chunk-size", + type=int, + default=256 * 1024, # 256 KB + help="The size of each chunk to download in bytes", + ) + + def get_options(self, options: dict[str, Any]) -> None: + self.verbosity = options["verbosity"] + self.force = options["force"] + self.dnn_files = ( + { + "url": options["deployfile_url"], + "filename": settings.DNN_FILES.get("prototxt", {}) + .get("sources", {}) + .get("azure"), + }, + { + "url": options["caffemodelfile_url"], + "filename": settings.DNN_FILES.get("caffemodel", {}) + .get("sources", {}) + .get("azure"), + }, + ) + self.download_timeout = options["download_timeout"] + self.chunk_size = options["chunk_size"] + + def handle(self, *args: Any, **options: Any) -> None: + """ + Executes the command to download and store DNN files from a given source to Azure Blob Storage. + + Args: + *args (Any): Positional arguments passed to the command. + **options (dict[str, Any]): Keyword arguments passed to the command, including: + - force (bool): If True, forces the re-download of files even if they already exist in storage. + - deployfile_url (str): The URL of the DNN model architecture file to download. + - caffemodelfile_url (str): The URL of the pre-trained model weights to download. + - download_timeout (int): Timeout for downloading each file, in seconds. + - chunk_size (int): The size of chunks for streaming downloads, in bytes. + + Raises: + FileNotFoundError: If the downloaded file is empty (size is 0 bytes). + ValidationError: If any arguments are invalid or improperly configured. + CommandError: If an issue occurs with the Django command execution. + SystemCheckError: If a system check error is encountered during execution. + Exception: For any other errors that occur during the download or storage process. + """ + self.get_options(options) + if self.verbosity >= 1: + echo = self.stdout.write + else: + echo = lambda *a, **kw: None # noqa: E731 + echo(self.style.WARNING(MESSAGES["start"])) + + try: + dnn_storage = AzureStorage(**settings.STORAGES.get("dnn").get("OPTIONS")) + _, files = dnn_storage.listdir("") + for file in self.dnn_files: + if self.force or not file.get("filename") in files: + echo(MESSAGES["process"] % (file.get("url"), file.get("filename"))) + with requests.get( + file.get("url"), stream=True, timeout=self.download_timeout + ) as r: + r.raise_for_status() + if int(r.headers.get("Content-Length", 1)) == 0: + raise FileNotFoundError(MESSAGES["empty"] % file.get("url")) + with dnn_storage.open(file.get("filename"), "wb") as f: + for chunk in r.iter_content(chunk_size=self.chunk_size): + f.write(chunk) + else: + echo(MESSAGES["already"] % file.get("filename")) + echo(self.style.SUCCESS(MESSAGES["completed"])) + except ValidationError as e: + self.halt(Exception("\n- ".join(["Wrong argument(s):", *e.messages]))) + except (CommandError, FileNotFoundError, SystemCheckError) as e: + self.halt(e) + except Exception as e: + self.halt(e) + + def halt(self, e: Exception) -> None: + """ + Handle an exception by logging the error and exiting the program. + + Args: + e (Exception): The exception that occurred. + """ + logger.exception(e) + self.stdout.write(self.style.ERROR(str(e))) + self.stdout.write(self.style.ERROR(MESSAGES["halted"])) + sys.exit(1) diff --git a/src/hope_dedup_engine/apps/core/management/commands/env.py b/src/hope_dedup_engine/apps/core/management/commands/env.py deleted file mode 100644 index a1bfff67..00000000 --- a/src/hope_dedup_engine/apps/core/management/commands/env.py +++ /dev/null @@ -1,89 +0,0 @@ -import shlex -from typing import TYPE_CHECKING - -from django.core.management import BaseCommand, CommandError, CommandParser - -if TYPE_CHECKING: - from typing import Any - -DEVELOP = { - "DEBUG": True, - "SECRET_KEY": "only-development-secret-key", -} - - -def clean(value): - if isinstance(value, (list, tuple)): - ret = ",".join(value) - else: - ret = str(value) - return shlex.quote(ret) - - -class Command(BaseCommand): - requires_migrations_checks = False - requires_system_checks = [] - - def add_arguments(self, parser: "CommandParser") -> None: - - parser.add_argument( - "--pattern", - action="store", - dest="pattern", - default="export {key}={value}", - help="Check env for variable availability (default: 'export {key}=\"{value}\"')", - ) - parser.add_argument( - "--develop", action="store_true", help="Display development values" - ) - parser.add_argument( - "--config", action="store_true", help="Only list changed values" - ) - parser.add_argument("--diff", action="store_true", help="Mark changed values") - parser.add_argument( - "--check", - action="store_true", - dest="check", - default=False, - help="Check env for variable availability", - ) - parser.add_argument( - "--ignore-errors", - action="store_true", - dest="ignore_errors", - default=False, - help="Do not fail", - ) - - def handle(self, *args: "Any", **options: "Any") -> None: - from hope_dedup_engine.config import CONFIG, EXPLICIT_SET, env - - check_failure = False - pattern = options["pattern"] - - for k, __ in sorted(CONFIG.items()): - help: str = env.get_help(k) - default = env.get_default(k) - if options["check"]: - if k in EXPLICIT_SET and k not in env.ENVIRON: - self.stderr.write(self.style.ERROR(f"- Missing env variable: {k}")) - check_failure = True - else: - if options["develop"]: - value: Any = env.for_develop(k) - else: - value: Any = env.get_value(k) - - line: str = pattern.format( - key=k, value=clean(value), help=help, default=default - ) - if options["diff"]: - if value != default: - line = self.style.SUCCESS(line) - elif options["config"]: - if value == default and k not in EXPLICIT_SET: - continue - self.stdout.write(line) - - if check_failure and not options["ignore_errors"]: - raise CommandError("Env check command failure!") diff --git a/src/hope_dedup_engine/apps/core/management/commands/upgrade.py b/src/hope_dedup_engine/apps/core/management/commands/upgrade.py index 513e2f01..c997605d 100644 --- a/src/hope_dedup_engine/apps/core/management/commands/upgrade.py +++ b/src/hope_dedup_engine/apps/core/management/commands/upgrade.py @@ -26,7 +26,7 @@ def add_arguments(self, parser: "ArgumentParser") -> None: "--with-check", action="store_true", dest="check", - default=False, + default=True, help="Run checks", ) parser.add_argument( @@ -64,7 +64,13 @@ def add_arguments(self, parser: "ArgumentParser") -> None: default=True, help="Do not run collectstatic", ) - + parser.add_argument( + "--with-dnn-setup", + action="store_true", + dest="dnn_setup", + default=False, + help="Run DNN setup for celery worker", + ) parser.add_argument( "--admin-email", action="store", @@ -86,6 +92,7 @@ def get_options(self, options: dict[str, Any]) -> None: self.prompt = not options["prompt"] self.static = options["static"] self.migrate = options["migrate"] + self.dnn_setup = options["dnn_setup"] self.debug = options["debug"] self.admin_email = str(options["admin_email"] or env("ADMIN_EMAIL", "")) @@ -119,10 +126,14 @@ def handle(self, *args: Any, **options: Any) -> None: # noqa: C901 "stdout": self.stdout, } echo("Running upgrade", style_func=self.style.WARNING) + call_command("env", check=True) if self.run_check: call_command("check", deploy=True, verbosity=self.verbosity - 1) + if self.dnn_setup: + echo("Run DNN setup for celery worker.") + call_command("dnnsetup", verbosity=self.verbosity - 1) if self.static: static_root = Path(env("STATIC_ROOT")) echo( @@ -168,7 +179,7 @@ def handle(self, *args: Any, **options: Any) -> None: # noqa: C901 else: admin = User.objects.filter(is_superuser=True).first() if not admin: - raise CommandError("Create an admin user") + raise CommandError("Failure: Error when creating an admin user!") from hope_dedup_engine.apps.security.constants import DEFAULT_GROUP_NAME diff --git a/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py b/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py new file mode 100644 index 00000000..1b995c40 --- /dev/null +++ b/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py @@ -0,0 +1,179 @@ +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +from django.conf import settings + +from azure.storage.blob import BlobServiceClient, ContainerClient + +logger = logging.getLogger(__name__) + + +class AzuriteManager: # pragma: no cover + def __init__( + self, storage_name: str, container_options: dict | None = None + ) -> None: + """ + Initializes the AzuriteManager with the specified storage configuration. + + Args: + storage_name (str): + The name of the storage configuration as defined in settings.STORAGES. + container_options (dict, optional): + Additional options to configure the Azure Blob Storage container. Defaults to an empty dictionary. + """ + storage = settings.STORAGES.get(storage_name).get("OPTIONS", {}) + self.container_client: ContainerClient = ( + BlobServiceClient.from_connection_string( + storage.get("connection_string") + ).get_container_client(storage.get("azure_container")) + ) + self._create_container(container_options) + + def _create_container(self, options: dict | None = None) -> None: + """ + Creates a container if it does not already exist. + + Args: + options (dict, optional): + Additional options to configure the container creation. Defaults to an empty dictionary. + + Raises: + Exception: If the container creation fails for any reason. + """ + options = options or {} + try: + if not self.container_client.exists(): + self.container_client.create_container(**options) + logger.info( + "Container '%s' created successfully.", + self.container_client.container_name, + ) + except Exception: + logger.exception("Failed to create container.") + raise + + def list_files(self) -> list[str]: + """ + Lists all files in the Azure Blob Storage container. + + Returns: + list[str]: A list of blob names in the container. + """ + try: + blob_list = self.container_client.list_blobs() + return [blob.name for blob in blob_list] + except Exception: + logger.exception("Failed to list files.") + raise + + def _upload_file(self, file: Path) -> str: + """ + Uploads a single file to the Azure Blob Storage container. + + Args: + file (Path): The local path of the file to upload. + images_src_path (Path | None): The local directory path where files are stored, or None if not specified. + + Returns: + str: A message indicating the result of the upload. + """ + if not file.exists(): + message = "File %s does not exist." + logger.warning(message, file) + return message % file + try: + blob_client = self.container_client.get_blob_client(file.name) + with file.open("rb") as f: + blob_client.upload_blob(f, overwrite=True) + message = "File uploaded successfully! URL: %s" + logger.debug(message, blob_client.url) + return message % blob_client.url + except Exception: + logger.exception("Failed to upload file %s.", file) + raise + + def upload_files( + self, images_src_path: Path | None = None, batch_size: int = 250 + ) -> list[str]: + """ + Uploads all files from the local directory to the Azure Blob Storage container. + + Args: + batch_size (int, optional): The maximum number of concurrent uploads. Defaults to 250. + + Returns: + list[str]: A list of messages indicating the result of each upload. + """ + if images_src_path is None or not images_src_path.is_dir(): + message = "No valid directory provided for container '%s'." + logger.warning(message, self.container_client.container_name) + return [message % self.container_client.container_name] + + files = [f for f in images_src_path.glob("*.*") if f.is_file()] + results: list[str] = [] + with ThreadPoolExecutor(max_workers=batch_size) as executor: + futures = {executor.submit(self._upload_file, f): f for f in files} + for future in as_completed(futures): + try: + results.append(future.result()) + except Exception: + file = futures[future] + results.append(f"Exception while processing {file}") + logger.exception("Exception while processing %s", file) + raise + + return results + + def delete_files(self) -> str: + """ + Deletes all files in the Azure Blob Storage container. + + Returns: + str: A message indicating the result of the deletion. + """ + try: + blob_names = self.list_files() + failed_deletions = [] + + for blob_name in blob_names: + try: + self.container_client.delete_blob(blob_name) + logger.debug("Deleted blob: %s", blob_name) + except Exception as e: + failed_deletions.append(blob_name) + logger.error( + "Failed to delete blob: %s. Error: %s", blob_name, str(e) + ) + + if failed_deletions: + message = f"Failed to delete the following blobs: {', '.join(failed_deletions)}" + else: + message = ( + "All files deleted successfully!" + if blob_names + else "No files to delete." + ) + + logger.info(message) + return message + + except Exception: + logger.exception("Failed to delete files.") + raise + + def delete_container(self) -> str: + """ + Deletes the Azure Blob Storage container. + + Returns: + str: A message indicating the result of the container deletion. + """ + try: + self.container_client.delete_container() + message = "Container has been deleted successfully!" + logger.info(message) + return message + except Exception: + logger.exception("Failed to delete container.") + raise diff --git a/src/hope_dedup_engine/apps/core/storage.py b/src/hope_dedup_engine/apps/core/storage.py deleted file mode 100644 index 13f14eb1..00000000 --- a/src/hope_dedup_engine/apps/core/storage.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import Any - -from django.conf import settings -from django.core.files.storage import FileSystemStorage - -from storages.backends.azure_storage import AzureStorage - - -class UniqueStorageMixin: - def get_available_name(self, name: str, max_length: int | None = None) -> str: - if self.exists(name): - self.delete(name) - return name - - -class CV2DNNStorage(UniqueStorageMixin, FileSystemStorage): - pass - - -class HDEAzureStorage(UniqueStorageMixin, AzureStorage): - def __init__(self, *args: Any, **kwargs: Any) -> None: - self.account_name = settings.AZURE_ACCOUNT_NAME - self.account_key = settings.AZURE_ACCOUNT_KEY - self.custom_domain = settings.AZURE_CUSTOM_DOMAIN - self.connection_string = settings.AZURE_CONNECTION_STRING - super().__init__(*args, **kwargs) - self.azure_container = settings.AZURE_CONTAINER_HDE - - -class HOPEAzureStorage(HDEAzureStorage): - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.azure_container = settings.AZURE_CONTAINER_HOPE - - def delete(self, name: str) -> None: - raise RuntimeError("This storage cannot delete files") - - def open(self, name: str, mode: str = "rb") -> Any: - if "w" in mode: - raise RuntimeError("This storage cannot open files in write mode") - return super().open(name, mode="rb") - - def save(self, name: str, content: Any, max_length: int | None = None) -> None: - raise RuntimeError("This storage cannot save files") - - def listdir(self, path: str = "") -> tuple[list[str], list[str]]: - return ([], []) diff --git a/src/hope_dedup_engine/apps/faces/admin.py b/src/hope_dedup_engine/apps/faces/admin.py new file mode 100644 index 00000000..6fd74cff --- /dev/null +++ b/src/hope_dedup_engine/apps/faces/admin.py @@ -0,0 +1,65 @@ +from django.contrib import admin + +from admin_extra_buttons.decorators import button +from admin_extra_buttons.mixins import ExtraButtonsMixin +from celery import group +from constance import config + +from hope_dedup_engine.apps.faces.celery_tasks import sync_dnn_files +from hope_dedup_engine.apps.faces.models import DummyModel +from hope_dedup_engine.config.celery import app as celery_app + + +@admin.register(DummyModel) +class DummyModelAdmin(ExtraButtonsMixin, admin.ModelAdmin): + + change_list_template = "admin/faces/dummymodel/change_list.html" + + def get_queryset(self, request): + return DummyModel.objects.none() + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + def changelist_view(self, request, extra_context=None): + extra_context = extra_context or {} + extra_context["title"] = ( + f"Force syncronize DNN files from {config.DNN_FILES_SOURCE} to local storage." + ) + return super().changelist_view(request, extra_context=extra_context) + + @button(label="Run sync") + def sync_dnn_files(self, request): + active_workers = celery_app.control.inspect().active_queues() + if not active_workers: + self.message_user( + request, + "No active workers found. Please start the Celery worker processes.", + ) + else: + worker_count = len(active_workers) + if worker_count > 1: + print(f"{worker_count=}") + job = group(sync_dnn_files.s(force=True) for _ in range(worker_count)) + result = job.apply_async() + self.message_user( + request, + f"The DNN files synchronization group task `{result.id}` has been initiated across " + f"`{worker_count}` workers. " + f"The files will be forcibly synchronized with `{config.DNN_FILES_SOURCE}`.", + ) + else: + task = sync_dnn_files.delay(force=True) + self.message_user( + request, + f"The DNN files sync task `{task.id}` has started. " + f"The files will be forcibly synchronized with `{config.DNN_FILES_SOURCE}`.", + ) + + return None diff --git a/src/hope_dedup_engine/apps/faces/celery_tasks.py b/src/hope_dedup_engine/apps/faces/celery_tasks.py index 5ef321d6..26b34755 100644 --- a/src/hope_dedup_engine/apps/faces/celery_tasks.py +++ b/src/hope_dedup_engine/apps/faces/celery_tasks.py @@ -1,10 +1,12 @@ import traceback +from django.conf import settings + from celery import Task, shared_task, states +from constance import config -from hope_dedup_engine.apps.faces.services.duplication_detector import ( - DuplicationDetector, -) +from hope_dedup_engine.apps.faces.managers import FileSyncManager +from hope_dedup_engine.apps.faces.services import DuplicationDetector from hope_dedup_engine.apps.faces.utils.celery_utils import task_lifecycle @@ -29,7 +31,44 @@ def deduplicate( """ try: dd = DuplicationDetector(filenames, ignore_pairs) - return dd.find_duplicates() + return list(dd.find_duplicates()) + except Exception as e: + self.update_state( + state=states.FAILURE, + meta={"exc_message": str(e), "traceback": traceback.format_exc()}, + ) + raise e + + +@shared_task(bind=True) +def sync_dnn_files(self: Task, force: bool = False) -> bool: + """ + A Celery task that synchronizes DNN files from the specified source to local storage. + + Args: + self (Task): The bound Celery task instance. + force (bool): If True, forces the re-download of files even if they already exist locally. Defaults to False. + + Returns: + bool: True if all files were successfully synchronized, False otherwise. + + Raises: + Exception: If any error occurs during the synchronization process. The task state is updated to FAILURE, + and the exception is re-raised with the associated traceback. + """ + + try: + downloader = FileSyncManager(config.DNN_FILES_SOURCE).downloader + return all( + ( + downloader.sync( + info.get("filename"), + info.get("sources").get(config.DNN_FILES_SOURCE), + force=force, + ) + ) + for _, info in settings.DNN_FILES.items() + ) except Exception as e: self.update_state( state=states.FAILURE, diff --git a/src/hope_dedup_engine/apps/faces/exceptions.py b/src/hope_dedup_engine/apps/faces/exceptions.py deleted file mode 100644 index ff8a42f4..00000000 --- a/src/hope_dedup_engine/apps/faces/exceptions.py +++ /dev/null @@ -1,8 +0,0 @@ -class StorageKeyError(Exception): - """ - Exception raised when the storage key does not exist. - """ - - def __init__(self, key: str) -> None: - self.key = key - super().__init__(f"Storage key '{key}' does not exist.") diff --git a/src/hope_dedup_engine/apps/faces/forms.py b/src/hope_dedup_engine/apps/faces/forms.py index c61799d2..e764833a 100644 --- a/src/hope_dedup_engine/apps/faces/forms.py +++ b/src/hope_dedup_engine/apps/faces/forms.py @@ -2,7 +2,7 @@ class MeanValuesTupleField(CharField): - def to_python(self, value: str) -> tuple[float, float, float]: + def to_python(self, value: str) -> tuple[float, ...]: try: values = tuple(map(float, value.split(", "))) if len(values) != 3: diff --git a/src/hope_dedup_engine/apps/faces/management/__init__.py b/src/hope_dedup_engine/apps/faces/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hope_dedup_engine/apps/faces/management/commands/syncdnn.py b/src/hope_dedup_engine/apps/faces/management/commands/syncdnn.py new file mode 100644 index 00000000..5780db44 --- /dev/null +++ b/src/hope_dedup_engine/apps/faces/management/commands/syncdnn.py @@ -0,0 +1,124 @@ +import logging +import sys +from typing import Any, Final + +from django.conf import settings +from django.core.management import BaseCommand +from django.core.management.base import CommandError, SystemCheckError + +from constance import config + +from hope_dedup_engine.apps.faces.managers.file_sync import FileSyncManager + +logger = logging.getLogger(__name__) + + +MESSAGES: Final[dict[str, str]] = { + "sync": "Starting synchronization of DNN files from %s ...", + "success": "Finished synchronizing DNN files successfully.", + "failed": "Failed to synchronize DNN files.", + "halted": "\n\n***\nSYSTEM HALTED\nUnable to start without DNN files...", +} + + +class Command(BaseCommand): + help = "Synchronizes DNN files from the specified source to local storage" + + def add_arguments(self, parser): + """ + Adds custom command-line arguments to the management command. + + Args: + parser (argparse.ArgumentParser): The argument parser instance to which the arguments should be added. + + Adds the following arguments: + --force: A boolean flag that, when provided, forces the re-download of files even if they + already exist locally. Defaults to False. + --source (str): Specifies the source from which to download the DNN files. The available choices + are dynamically retrieved from the CONSTANCE_ADDITIONAL_FIELDS configuration. + Defaults to the value of config.DNN_FILES_SOURCE. + """ + parser.add_argument( + "--force", + action="store_true", + default=False, + help="Force the re-download of files even if they already exist locally", + ) + parser.add_argument( + "--source", + type=str, + default=config.DNN_FILES_SOURCE, + choices=tuple( + ch[0] + for ch in settings.CONSTANCE_ADDITIONAL_FIELDS.get("dnn_files_source")[ + 1 + ].get("choices") + ), + help="The source from which to download the DNN files", + ) + + def handle(self, *args: Any, **options: dict[str, Any]) -> None: + """ + Executes the command to synchronize DNN files from a specified source to local storage. + + Args: + *args (Any): Positional arguments passed to the command. + **options (dict[str, Any]): Keyword arguments passed to the command, including: + - force (bool): If True, forces the re-download of files even if they already exist locally. + - source (str): The source from which to download the DNN files. + + Raises: + CommandError: If there is a problem executing the command. + SystemCheckError: If there is a system check error. + Exception: For any other errors that occur during execution. + """ + + def on_progress(filename: str, percent: int, is_complete: bool = False) -> None: + """ + Callback function to report the progress of a file download. + + Args: + filename (str): The name of the file being downloaded. + percent (int): The current download progress as a percentage (0-100). + is_complete (bool): If True, indicates that the download is complete. Defaults to False. + + Returns: + None + """ + self.stdout.write(f"\rDownloading file {filename}: {percent}%", ending="") + if is_complete: + self.stdout.write("\n") + + self.stdout.write(self.style.WARNING(MESSAGES["sync"]) % options.get("source")) + logger.info(MESSAGES["sync"]) + + try: + downloader = FileSyncManager(options.get("source")).downloader + for _, info in settings.DNN_FILES.items(): + downloader.sync( + info.get("filename"), + info.get("sources").get(options.get("source")), + force=options.get("force"), + on_progress=on_progress, + ) + on_progress(info.get("filename"), 100, is_complete=True) + except (CommandError, SystemCheckError) as e: + self.halt(e) + except Exception as e: + self.stdout.write(self.style.ERROR(MESSAGES["failed"])) + logger.error(MESSAGES["failed"]) + self.halt(e) + + self.stdout.write(self.style.SUCCESS(MESSAGES["success"])) + + def halt(self, e: Exception) -> None: + """ + Handle an exception by logging the error and exiting the program. + + Args: + e (Exception): The exception that occurred. + """ + logger.exception(e) + self.stdout.write(self.style.ERROR(str(e))) + self.stdout.write(self.style.ERROR(MESSAGES["halted"])) + sys.exit(1) diff --git a/src/hope_dedup_engine/apps/faces/managers/__init__.py b/src/hope_dedup_engine/apps/faces/managers/__init__.py index e69de29b..b11c4b03 100644 --- a/src/hope_dedup_engine/apps/faces/managers/__init__.py +++ b/src/hope_dedup_engine/apps/faces/managers/__init__.py @@ -0,0 +1,3 @@ +from .file_sync import FileSyncManager # noqa: F401 +from .net import DNNInferenceManager # noqa: F401 +from .storage import StorageManager # noqa: F401 diff --git a/src/hope_dedup_engine/apps/faces/managers/file_sync.py b/src/hope_dedup_engine/apps/faces/managers/file_sync.py new file mode 100644 index 00000000..c19df1c4 --- /dev/null +++ b/src/hope_dedup_engine/apps/faces/managers/file_sync.py @@ -0,0 +1,253 @@ +from pathlib import Path +from typing import Callable + +from django.conf import settings +from django.core.files.storage import FileSystemStorage + +import requests +from storages.backends.azure_storage import AzureStorage + +from hope_dedup_engine.apps.core.exceptions import DownloaderKeyError + + +class FileDownloader: + """ + Base class for downloading files from different sources. + """ + + def __init__(self) -> None: + """ + Initializes the FileDownloader with a local storage backend. + """ + self.local_storage = FileSystemStorage( + **settings.STORAGES.get("default").get("OPTIONS") + ) + + def sync( + self, + filename: str, + source: str, + force: bool = False, + on_progress: Callable[[str, int], None] = None, + *args, + **kwargs, + ) -> bool: + """ + Synchronize a file from the specified source to the local storage. + + Args: + filename (str): The name of the file to be synchronized. + source (str): The source from which the file should be downloaded. + force (bool): If True, the file will be re-downloaded even if it already exists locally. + on_progress (Callable[[str, int], None], optional): A callback function to report the download + progress. The function should accept two arguments: the filename and the download progress + as a percentage. Defaults to None. + *args: Additional positional arguments for extended functionality in subclasses. + **kwargs: Additional keyword arguments for extended functionality in subclasses. + + Returns: + bool: True if the file was successfully synchronized or already exists locally, + False otherwise. + + Raises: + NotImplementedError: This method should be overridden by subclasses to provide + specific synchronization logic. + """ + raise NotImplementedError("This method should be overridden by subclasses.") + + def _prepare_local_filepath(self, filename: str, force: bool) -> Path | None: + """ + Prepares the local file path for the file to be downloaded. + + Args: + filename (str): The name of the file. + force (bool): If True, the file will be re-downloaded even if it exists locally. + + Returns: + Path | None: The local file path if the file should be downloaded, + None if the file exists and `force` is False. + """ + local_filepath = Path(self.local_storage.path(filename)) + if not force and local_filepath.exists(): + return None + local_filepath.parent.mkdir(parents=True, exist_ok=True) + return local_filepath + + def _report_progress( + self, + filename: str, + downloaded: int, + total: int, + on_progress: Callable[[str, int], None] = None, + ) -> None: + """ + Reports the download progress of a file. + + Args: + filename (str): The name of the file being downloaded. + downloaded (int): The number of bytes that have been downloaded so far. + total (int): The total size of the file in bytes. + on_progress (Callable[[str, int], None], optional): A callback function that is called with the filename + and the download percentage. Defaults to None. + + Returns: + None + """ + if on_progress and total > 0: + on_progress(filename, int((downloaded / total) * 100)) + + +class GithubFileDownloader(FileDownloader): + """ + Downloader class for downloading files from GitHub. + + Inherits from FileDownloader and implements the sync method to download files from a given GitHub URL. + """ + + def sync( + self, + filename: str, + url: str, + force: bool = False, + on_progress: Callable[[str, int], None] = None, + timeout: int = 3 * 60, + chunk_size: int = 128 * 1024, + ) -> bool: + """ + Downloads a file from the specified URL and saves it to local storage. + + Args: + filename (str): The name of the file to be downloaded. + url (str): The URL of the file to download. + force (bool): If True, the file will be re-downloaded even if it exists locally. Defaults to False. + on_progress (Callable[[str, int], None], optional): A callback function that reports the download progress + as a percentage. Defaults to None. + timeout (int): The timeout for the download request in seconds. Defaults to 3 minutes. + chunk_size (int): The size of each chunk to download in bytes. Defaults to 128 KB. + + Returns: + bool: True if the file was downloaded successfully or already exists, False otherwise. + + Raises: + requests.exceptions.HTTPError: If the HTTP request returned an unsuccessful status code. + FileNotFoundError: If the file at the specified URL is empty (size is 0 bytes). + """ + local_filepath = self._prepare_local_filepath(filename, force) + if local_filepath is None: + return True + + with requests.get(url, stream=True, timeout=timeout) as r: + r.raise_for_status() + total, downloaded = int(r.headers.get("Content-Length", 1)), 0 + + if total == 0: + raise FileNotFoundError( + f"File {filename} at {url} is empty (size is 0 bytes)." + ) + + with local_filepath.open("wb") as f: + for chunk in r.iter_content(chunk_size=chunk_size): + f.write(chunk) + downloaded += len(chunk) + self._report_progress(filename, downloaded, total, on_progress) + + return True + + +class AzureFileDownloader(FileDownloader): + """ + Downloader class for downloading files from Azure Blob Storage. + + Inherits from FileDownloader and implements the sync method to download files from a given Azure Blob Storage. + """ + + def __init__(self) -> None: + """ + Initializes the AzureFileDownloader with a remote storage backend. + """ + super().__init__() + self.remote_storage = AzureStorage( + **settings.STORAGES.get("dnn").get("OPTIONS") + ) + + def sync( + self, + filename: str, + blob_name: str, + force: bool = False, + on_progress: Callable[[str, int], None] = None, + chunk_size: int = 128 * 1024, + ) -> bool: + """ + Downloads a file from Azure Blob Storage and saves it to local storage. + + Args: + filename (str): The name of the file to be saved locally. + blob_name (str): The name of the blob in Azure Blob Storage. + force (bool): If True, the file will be re-downloaded even if it exists locally. Defaults to False. + on_progress (Callable[[str, int], None], optional): A callback function that reports the download progress + as a percentage. Defaults to None. + chunk_size (int): The size of each chunk to download in bytes. Defaults to 128 KB. + + Returns: + bool: True if the file was downloaded successfully or already exists, False otherwise. + + Raises: + FileNotFoundError: If the specified blob does not exist in Azure Blob Storage + or if the blob has a size of 0 bytes. + """ + local_filepath = self._prepare_local_filepath(filename, force) + if local_filepath is None: + return True + + _, files = self.remote_storage.listdir("") + if blob_name not in files: + raise FileNotFoundError( + f"File {blob_name} does not exist in remote storage" + ) + + blob_size, downloaded = self.remote_storage.size(blob_name) or 1, 0 + if blob_size == 0: + raise FileNotFoundError(f"File {blob_name} is empty (size is 0 bytes).") + + with self.remote_storage.open(blob_name, "rb") as remote_file: + with local_filepath.open("wb") as local_file: + for chunk in remote_file.chunks(chunk_size=chunk_size): + local_file.write(chunk) + downloaded += len(chunk) + self._report_progress(filename, downloaded, blob_size, on_progress) + + return True + + +class FileSyncManager: + def __init__(self, source: str) -> None: + """ + Initialize the FileSyncManager with the specified source. + + Args: + source (str): The source for downloading files. + """ + self.downloader = self._create_downloader(source) + + def _create_downloader(self, source: str) -> FileDownloader: + """ + Create an instance of the appropriate downloader based on the source. + + Args: + source (str): The source for downloading files (e.g., 'github' or 'azure'). + + Returns: + FileDownloader: An instance of the appropriate downloader. + + Raises: + DownloaderKeyError: If the source is not recognized. + """ + downloader_classes = { + "github": GithubFileDownloader, + "azure": AzureFileDownloader, + } + try: + return downloader_classes[source]() + except KeyError: + raise DownloaderKeyError(source) diff --git a/src/hope_dedup_engine/apps/faces/managers/net.py b/src/hope_dedup_engine/apps/faces/managers/net.py index a1d4532d..3664d941 100644 --- a/src/hope_dedup_engine/apps/faces/managers/net.py +++ b/src/hope_dedup_engine/apps/faces/managers/net.py @@ -1,10 +1,9 @@ from django.conf import settings +from django.core.files.storage import FileSystemStorage from constance import config from cv2 import dnn, dnn_Net -from hope_dedup_engine.apps.core.storage import CV2DNNStorage - class DNNInferenceManager: """ @@ -14,16 +13,16 @@ class DNNInferenceManager: specified storage and configure the model with preferred backend and target settings. """ - def __init__(self, storage: CV2DNNStorage) -> None: + def __init__(self, storage: FileSystemStorage) -> None: """ Loads and configures the neural network model using the specified storage. Args: - storage (CV2DNNStorage): The storage object from which to load the neural network model. + storage (FileSystemStorage): The storage object from which to load the neural network model. """ self.net = dnn.readNetFromCaffe( - storage.path(settings.PROTOTXT_FILE), - storage.path(settings.CAFFEMODEL_FILE), + storage.path(settings.DNN_FILES.get("prototxt").get("filename")), + storage.path(settings.DNN_FILES.get("caffemodel").get("filename")), ) self.net.setPreferableBackend(int(config.DNN_BACKEND)) self.net.setPreferableTarget(int(config.DNN_TARGET)) diff --git a/src/hope_dedup_engine/apps/faces/managers/storage.py b/src/hope_dedup_engine/apps/faces/managers/storage.py index 22318669..b1d6fba4 100644 --- a/src/hope_dedup_engine/apps/faces/managers/storage.py +++ b/src/hope_dedup_engine/apps/faces/managers/storage.py @@ -1,11 +1,9 @@ from django.conf import settings +from django.core.files.storage import FileSystemStorage -from hope_dedup_engine.apps.core.storage import ( - CV2DNNStorage, - HDEAzureStorage, - HOPEAzureStorage, -) -from hope_dedup_engine.apps.faces.exceptions import StorageKeyError +from storages.backends.azure_storage import AzureStorage + +from hope_dedup_engine.apps.core.exceptions import StorageKeyError class StorageManager: @@ -20,18 +18,22 @@ def __init__(self) -> None: Raises: FileNotFoundError: If any of the required DNN model files do not exist in the storage. """ - self.storages: dict[str, HOPEAzureStorage | CV2DNNStorage | HDEAzureStorage] = { - "images": HOPEAzureStorage(), - "cv2dnn": CV2DNNStorage(settings.CV2DNN_PATH), - "encoded": HDEAzureStorage(), + self.storages: dict[str, AzureStorage | FileSystemStorage] = { + "cv2": FileSystemStorage(**settings.STORAGES.get("default").get("OPTIONS")), + "encoded": FileSystemStorage( + **settings.STORAGES.get("default").get("OPTIONS") + ), + "images": AzureStorage(**settings.STORAGES.get("hope").get("OPTIONS")), } - for file in (settings.PROTOTXT_FILE, settings.CAFFEMODEL_FILE): - if not self.storages.get("cv2dnn").exists(file): + + for file in ( + settings.DNN_FILES.get("prototxt").get("filename"), + settings.DNN_FILES.get("caffemodel").get("filename"), + ): + if not self.storages.get("cv2").exists(file): raise FileNotFoundError(f"File {file} does not exist in storage.") - def get_storage( - self, key: str - ) -> HOPEAzureStorage | CV2DNNStorage | HDEAzureStorage: + def get_storage(self, key: str) -> AzureStorage | FileSystemStorage: """ Get the storage object for the given key. @@ -39,7 +41,7 @@ def get_storage( key (str): The key associated with the desired storage backend. Returns: - HOPEAzureStorage | CV2DNNStorage | HDEAzureStorage: The storage object associated with the given key. + AzureStorage | FileSystemStorage: The storage object associated with the given key. Raises: StorageKeyError: If the given key does not exist in the storages dictionary. diff --git a/src/hope_dedup_engine/apps/faces/models.py b/src/hope_dedup_engine/apps/faces/models.py new file mode 100644 index 00000000..dbaaeccf --- /dev/null +++ b/src/hope_dedup_engine/apps/faces/models.py @@ -0,0 +1,8 @@ +from django.db import models + + +class DummyModel(models.Model): + class Meta: + managed = False + verbose_name = "DNN file" + verbose_name_plural = "DNN files" diff --git a/src/hope_dedup_engine/apps/faces/services/__init__.py b/src/hope_dedup_engine/apps/faces/services/__init__.py index e69de29b..fc02e5c3 100644 --- a/src/hope_dedup_engine/apps/faces/services/__init__.py +++ b/src/hope_dedup_engine/apps/faces/services/__init__.py @@ -0,0 +1 @@ +from .duplication_detector import DuplicationDetector # noqa: F401 diff --git a/src/hope_dedup_engine/apps/faces/services/duplication_detector.py b/src/hope_dedup_engine/apps/faces/services/duplication_detector.py index a25db7e3..d7b60a35 100644 --- a/src/hope_dedup_engine/apps/faces/services/duplication_detector.py +++ b/src/hope_dedup_engine/apps/faces/services/duplication_detector.py @@ -1,15 +1,13 @@ import logging import os -from typing import Any +from itertools import combinations +from typing import Any, Generator import face_recognition import numpy as np -from hope_dedup_engine.apps.faces.managers.storage import StorageManager +from hope_dedup_engine.apps.faces.managers import StorageManager from hope_dedup_engine.apps.faces.services.image_processor import ImageProcessor -from hope_dedup_engine.apps.faces.utils.duplicate_groups_builder import ( - DuplicateGroupsBuilder, -) from hope_dedup_engine.apps.faces.validators import IgnorePairsValidator @@ -21,7 +19,10 @@ class DuplicationDetector: logger: logging.Logger = logging.getLogger(__name__) def __init__( - self, filenames: tuple[str], ignore_pairs: tuple[tuple[str, str], ...] = tuple() + self, + filenames: tuple[str], + face_distance_threshold: float, + ignore_pairs: tuple[tuple[str, str], ...] = (), ) -> None: """ Initialize the DuplicationDetector with the given filenames and ignore pairs. @@ -32,9 +33,10 @@ def __init__( The pairs of filenames to ignore. Defaults to an empty tuple. """ self.filenames = filenames + self.face_distance_threshold = face_distance_threshold self.ignore_set = IgnorePairsValidator.validate(ignore_pairs) self.storages = StorageManager() - self.image_processor = ImageProcessor() + self.image_processor = ImageProcessor(face_distance_threshold) def _encodings_filename(self, filename: str) -> str: """ @@ -73,7 +75,8 @@ def _load_encodings_all(self) -> dict[str, list[np.ndarray[np.float32, Any]]]: try: _, files = self.storages.get_storage("encoded").listdir("") for file in files: - if self._has_encodings(filename := os.path.splitext(file)[0]): + filename = os.path.splitext(file)[0] + if file == self._encodings_filename(filename): with self.storages.get_storage("encoded").open(file, "rb") as f: data[filename] = np.load(f, allow_pickle=False) except Exception as e: @@ -81,47 +84,63 @@ def _load_encodings_all(self) -> dict[str, list[np.ndarray[np.float32, Any]]]: raise e return data - def find_duplicates(self) -> list[list[str]]: + def _existed_images_name(self) -> list[str]: """ - Find and return a list of duplicate images based on face encodings. + Return filenames from `self.filenames` that exist in the image storage, ensuring they have encodings. Returns: - list[list[str]]: A list of lists, where each inner list contains - the filenames of duplicate images. + list[str]: List of existing filenames with encodings. """ - try: - for filename in self.filenames: + filenames: list = [] + _, files = self.storages.get_storage("images").listdir("") + for filename in self.filenames: + if filename not in files: + self.logger.warning( + "Image %s not found in the image storage.", filename + ) + else: + filenames.append(filename) if not self._has_encodings(filename): self.image_processor.encode_face( filename, self._encodings_filename(filename) ) + return filenames + + def find_duplicates(self) -> Generator[tuple[str, str, float], None, None]: + """ + Finds duplicate images based on facial encodings and yields pairs of image paths with their minimum distance. + + Yields: + Generator[tuple[str, str, float], None, None]: A generator yielding tuples containing: + - The first image path (str) + - The second image path (str) + - The minimum facial distance between the images, rounded to five decimal places (float) + + Raises: + Exception: If an error occurs during processing, it logs the exception and re-raises it. + """ + try: + + existed_images_name = self._existed_images_name() encodings_all = self._load_encodings_all() - checked = set() - for path1, encodings1 in encodings_all.items(): - for path2, encodings2 in encodings_all.items(): - if all( - ( - path1 < path2, - not any( - p in self.ignore_set - for p in ((path1, path2), (path2, path1)) - ), + for path1, path2 in combinations(existed_images_name, 2): + min_distance = self.face_distance_threshold + encodings1 = encodings_all.get(path1) + encodings2 = encodings_all.get(path2) + if encodings1 is None or encodings2 is None: + continue + + for encoding1 in encodings1: + if ( + current_min := min( + face_recognition.face_distance(encodings2, encoding1) ) - ): - min_distance = float("inf") - for encoding1 in encodings1: - if ( - current_min := min( - face_recognition.face_distance( - encodings2, encoding1 - ) - ) - ) < min_distance: - min_distance = current_min - checked.add((path1, path2, min_distance)) - - return DuplicateGroupsBuilder.build(checked) + ) < min_distance: + min_distance = current_min + + if min_distance < self.face_distance_threshold: + yield (path1, path2, round(min_distance, 5)) except Exception as e: self.logger.exception( "Error finding duplicates for images %s", self.filenames diff --git a/src/hope_dedup_engine/apps/faces/services/image_processor.py b/src/hope_dedup_engine/apps/faces/services/image_processor.py index 26c4ab5e..7e3a0bad 100644 --- a/src/hope_dedup_engine/apps/faces/services/image_processor.py +++ b/src/hope_dedup_engine/apps/faces/services/image_processor.py @@ -11,8 +11,7 @@ import numpy as np from constance import config -from hope_dedup_engine.apps.faces.managers.net import DNNInferenceManager -from hope_dedup_engine.apps.faces.managers.storage import StorageManager +from hope_dedup_engine.apps.faces.managers import DNNInferenceManager, StorageManager @dataclass(frozen=True, slots=True) @@ -26,6 +25,7 @@ class BlobFromImageConfig: shape: dict[str, int] = field(init=False) scale_factor: float mean_values: tuple[float, float, float] + prototxt_path: str def __post_init__(self) -> None: object.__setattr__(self, "shape", self._get_shape()) @@ -36,7 +36,7 @@ def __post_init__(self) -> None: def _get_shape(self) -> dict[str, int]: pattern = r"input_shape\s*\{\s*dim:\s*(\d+)\s*dim:\s*(\d+)\s*dim:\s*(\d+)\s*dim:\s*(\d+)\s*\}" - with open(settings.PROTOTXT_FILE, "r") as file: + with open(self.prototxt_path, "r") as file: if match := re.search(pattern, file.read()): return { "batch_size": int(match.group(1)), @@ -56,23 +56,26 @@ class ImageProcessor: logger: logging.Logger = logging.getLogger(__name__) - def __init__(self) -> None: + def __init__(self, face_distance_threshold: float) -> None: """ Initialize the ImageProcessor with the required configurations. """ self.storages = StorageManager() - self.net = DNNInferenceManager(self.storages.get_storage("cv2dnn")).get_model() + self.net = DNNInferenceManager(self.storages.get_storage("cv2")).get_model() self.blob_from_image_cfg = BlobFromImageConfig( scale_factor=config.BLOB_FROM_IMAGE_SCALE_FACTOR, mean_values=config.BLOB_FROM_IMAGE_MEAN_VALUES, + prototxt_path=self.storages.get_storage("cv2").path( + settings.DNN_FILES.get("prototxt").get("filename") + ), ) self.face_encodings_cfg = FaceEncodingsConfig( num_jitters=config.FACE_ENCODINGS_NUM_JITTERS, model=config.FACE_ENCODINGS_MODEL, ) self.face_detection_confidence: float = config.FACE_DETECTION_CONFIDENCE - self.distance_threshold: float = config.FACE_DISTANCE_THRESHOLD + self.distance_threshold: float = face_distance_threshold self.nms_threshold: float = config.NMS_THRESHOLD def _get_face_detections_dnn( @@ -158,7 +161,7 @@ def encode_face(self, filename: str, encodings_filename: str) -> None: encodings: list[np.ndarray[np.float32, Any]] = [] face_regions = self._get_face_detections_dnn(filename) if not face_regions: - self.logger.error("No face regions detected in image %s", filename) + self.logger.warning("No face regions detected in image %s", filename) else: for region in face_regions: if isinstance(region, (list, tuple)) and len(region) == 4: diff --git a/src/hope_dedup_engine/apps/faces/utils/celery_utils.py b/src/hope_dedup_engine/apps/faces/utils/celery_utils.py index 9e12d864..72791e95 100644 --- a/src/hope_dedup_engine/apps/faces/utils/celery_utils.py +++ b/src/hope_dedup_engine/apps/faces/utils/celery_utils.py @@ -3,18 +3,27 @@ from functools import wraps from typing import Any -from django.conf import settings - -import redis +from django.core.cache import cache from hope_dedup_engine.apps.faces.services.duplication_detector import ( DuplicationDetector, ) -redis_client = redis.Redis.from_url(settings.CELERY_BROKER_URL) +redis_client = cache._cache.get_client() def task_lifecycle(name: str, ttl: int) -> callable: + """ + Decorator to manage the lifecycle of a task with logging and distributed locking. + + Args: + name (str): The name of the task for logging purposes. + ttl (int): The time-to-live (TTL) for the distributed lock in seconds. + + Returns: + Callable: The decorated function with task lifecycle management. + """ + def decorator(func: callable) -> callable: @wraps(func) def wrapper(self: DuplicationDetector, *args: Any, **kwargs: Any) -> Any: @@ -47,14 +56,40 @@ def wrapper(self: DuplicationDetector, *args: Any, **kwargs: Any) -> Any: def _acquire_lock(lock_name: str, ttl: int = 1 * 60 * 60) -> bool | None: + """ + Acquire a distributed lock using Redis. + + Args: + lock_name (str): The name of the lock to set. + ttl (int): The time-to-live for the lock in seconds. Default is 1 hour (3600 seconds). + + Returns: + bool | None: True if the lock was successfully acquired, None if the lock is already set. + """ return redis_client.set(lock_name, "true", nx=True, ex=ttl) def _release_lock(lock_name: str) -> None: + """ + Release a distributed lock using Redis. + + Args: + lock_name (str): The name of the lock to delete. + """ redis_client.delete(lock_name) def _get_hash(filenames: tuple[str], ignore_pairs: tuple[tuple[str, str]]) -> str: + """ + Generate a SHA-256 hash based on filenames and ignore pairs. + + Args: + filenames (tuple[str]): A tuple of filenames. + ignore_pairs (tuple[tuple[str, str]]): A tuple of pairs of filenames to ignore. + + Returns: + str: A SHA-256 hash string representing the combination of filenames and ignore pairs. + """ fn_str: str = ",".join(sorted(filenames)) ip_sorted = sorted( (min(item1, item2), max(item1, item2)) for item1, item2 in ignore_pairs diff --git a/src/hope_dedup_engine/apps/faces/utils/duplicate_groups_builder.py b/src/hope_dedup_engine/apps/faces/utils/duplicate_groups_builder.py deleted file mode 100644 index 3b61eeeb..00000000 --- a/src/hope_dedup_engine/apps/faces/utils/duplicate_groups_builder.py +++ /dev/null @@ -1,44 +0,0 @@ -from collections import defaultdict - -from constance import config - - -class DuplicateGroupsBuilder: - @staticmethod - def build(checked: set[tuple[str, str, float]]) -> list[list[str]]: - """ - Transform a set of tuples with distances into a tuple of grouped duplicate paths. - - Args: - checked (set[tuple[str, str, float]]): A set of tuples containing the paths and their distances. - - Returns: - list[list[str]]: A list of grouped duplicate paths. - """ - # Dictionary to store connections between paths where distances are less than the threshold - groups = [] - connections = defaultdict(set) - for path1, path2, dist in checked: - if dist < config.FACE_DISTANCE_THRESHOLD: - connections[path1].add(path2) - connections[path2].add(path1) - # Iterate over each path and form groups - for path, neighbors in connections.items(): - # Check if the path has already been included in any group - if not any(path in group for group in groups): - new_group = {path} - queue = list(neighbors) - # Try to expand the group ensuring each new path is duplicated to all in the group - while queue: - neighbor = queue.pop(0) - if neighbor not in new_group and all( - neighbor in connections[member] for member in new_group - ): - new_group.add(neighbor) - # Add neighbors of the current neighbor, excluding those already in the group - queue.extend( - [n for n in connections[neighbor] if n not in new_group] - ) - # Add the newly formed group to the list of groups - groups.append(new_group) - return list(map(list, groups)) diff --git a/src/hope_dedup_engine/apps/security/admin.py b/src/hope_dedup_engine/apps/security/admin.py index 6b58abc1..a86436a6 100644 --- a/src/hope_dedup_engine/apps/security/admin.py +++ b/src/hope_dedup_engine/apps/security/admin.py @@ -3,14 +3,19 @@ # from unicef_security.admin import UserAdminPlus from django.contrib.auth.admin import UserAdmin as BaseUserAdmin -from hope_dedup_engine.apps.security.models import User, UserRole +from hope_dedup_engine.apps.security.models import ExternalSystem, User, UserRole @admin.register(User) class UserAdmin(BaseUserAdmin): - pass + fieldsets = BaseUserAdmin.fieldsets + ((None, {"fields": ("external_system",)}),) @admin.register(UserRole) class UserRoleAdmin(admin.ModelAdmin): list_display = ("user", "system", "group") + + +@admin.register(ExternalSystem) +class ExternalSystemAdmin(admin.ModelAdmin): + pass diff --git a/src/hope_dedup_engine/apps/security/backends.py b/src/hope_dedup_engine/apps/security/backends.py index e69de29b..6b0a477c 100644 --- a/src/hope_dedup_engine/apps/security/backends.py +++ b/src/hope_dedup_engine/apps/security/backends.py @@ -0,0 +1,26 @@ +from typing import TYPE_CHECKING, Any, Optional + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.auth.backends import ModelBackend + +if TYPE_CHECKING: + from django.contrib.auth.models import AbstractBaseUser + from django.http import HttpRequest + + +class AnyUserAuthBackend(ModelBackend): + def authenticate( + self, + request: "Optional[HttpRequest]", + username: str | None = None, + password: str | None = None, + **kwargs: Any, + ) -> "AbstractBaseUser | None": + if settings.DEBUG: + user, __ = get_user_model().objects.update_or_create( + username=username, + defaults=dict(is_staff=True, is_active=True, is_superuser=True), + ) + return user + return None diff --git a/src/hope_dedup_engine/apps/security/migrations/0002_alter_system_id_alter_user_id_alter_userrole_id.py b/src/hope_dedup_engine/apps/security/migrations/0002_alter_system_id_alter_user_id_alter_userrole_id.py new file mode 100644 index 00000000..66cee70d --- /dev/null +++ b/src/hope_dedup_engine/apps/security/migrations/0002_alter_system_id_alter_user_id_alter_userrole_id.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.6 on 2024-07-16 13:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("security", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="system", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + migrations.AlterField( + model_name="user", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + migrations.AlterField( + model_name="userrole", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ] diff --git a/src/hope_dedup_engine/config/__init__.py b/src/hope_dedup_engine/config/__init__.py index ccd74314..5f915371 100644 --- a/src/hope_dedup_engine/config/__init__.py +++ b/src/hope_dedup_engine/config/__init__.py @@ -1,93 +1,144 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Dict, Tuple, TypeAlias, Union -from urllib import parse -from environ import Env +from smart_env import SmartEnv if TYPE_CHECKING: ConfigItem: TypeAlias = Union[ Tuple[type, Any, str, Any], Tuple[type, Any, str], Tuple[type, Any] ] - -DJANGO_HELP_BASE = "https://docs.djangoproject.com/en/5.0/ref/settings" +DJANGO_HELP_BASE = "https://docs.djangoproject.com/en/5.1/ref/settings" def setting(anchor: str) -> str: return f"@see {DJANGO_HELP_BASE}#{anchor}" +def celery_doc(anchor: str) -> str: + return ( + f"@see https://docs.celeryq.dev/en/stable/" + f"userguide/configuration.html#{anchor}" + ) + + class Group(Enum): DJANGO = 1 -NOT_SET = "<- not set ->" -EXPLICIT_SET = [ - "DATABASE_URL", - "SECRET_KEY", - "CACHE_URL", - "CELERY_BROKER_URL", - "MEDIA_ROOT", - "STATIC_ROOT", -] - CONFIG: "Dict[str, ConfigItem]" = { - "ADMIN_EMAIL": (str, "", "Initial user created at first deploy"), - "ADMIN_PASSWORD": (str, "", "Password for initial user created at first deploy"), - "ALLOWED_HOSTS": (list, ["127.0.0.1", "localhost"], setting("allowed-hosts")), - "AUTHENTICATION_BACKENDS": (list, [], setting("authentication-backends")), - "CACHE_URL": (str, "redis://localhost:6379/0"), - "CATCH_ALL_EMAIL": (str, "If set all the emails will be sent to this address"), + "ADMIN_EMAIL": ( + str, + SmartEnv.NOTSET, + "admin", + True, + "Initial user created at first deploy", + ), + "ADMIN_PASSWORD": ( + str, + "", + "", + True, + "Password for initial user created at first deploy", + ), + "ALLOWED_HOSTS": ( + list, + [], + ["127.0.0.1", "localhost"], + True, + setting("allowed-hosts"), + ), + "AUTHENTICATION_BACKENDS": ( + list, + [], + [], + False, + setting("authentication-backends"), + ), + "AZURE_CLIENT_SECRET": (str, ""), + "AZURE_TENANT_ID": (str, ""), + "AZURE_CLIENT_KEY": (str, ""), + "CACHE_URL": ( + str, + SmartEnv.NOTSET, + "redis://localhost:6379/0", + True, + setting("cache-url"), + ), + "CATCH_ALL_EMAIL": ( + str, + "", + "", + False, + "If set all the emails will be sent to this address", + ), "CELERY_BROKER_URL": ( str, - NOT_SET, + "", + "", + True, "https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html", ), "CELERY_TASK_ALWAYS_EAGER": ( bool, False, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_always_eager", True, + False, + f"{celery_doc}#std-setting-task_always_eager", ), "CELERY_TASK_EAGER_PROPAGATES": ( bool, True, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates", + True, + False, + f"{celery_doc}#task-eager-propagates", ), "CELERY_VISIBILITY_TIMEOUT": ( int, 1800, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#broker-transport-options", + 1800, + False, + f"{celery_doc}#broker-transport-options", ), - "CSRF_COOKIE_SECURE": (bool, True, setting("csrf-cookie-secure")), + "CSRF_COOKIE_SECURE": (bool, True, False, setting("csrf-cookie-secure")), "DATABASE_URL": ( str, - "postgres://127.0.0.1:5432/dedupe", + SmartEnv.NOTSET, + SmartEnv.NOTSET, + True, "https://django-environ.readthedocs.io/en/latest/types.html#environ-env-db-url", - "postgres://127.0.0.1:5432/dedupe", ), - "DEBUG": (bool, False, setting("debug"), True), - "EMAIL_BACKEND": ( + "DEBUG": (bool, False, True, False, setting("debug")), + "DEFAULT_ROOT": ( str, - "django.core.mail.backends.smtp.EmailBackend", - setting("email-backend"), + "/var/default/", + "/tmp/default", # nosec True, + "Default root for stored locally files", ), - "EMAIL_HOST": (str, "localhost", setting("email-host"), True), - "EMAIL_HOST_USER": (str, "", setting("email-host-user"), True), - "EMAIL_HOST_PASSWORD": (str, "", setting("email-host-password"), True), - "EMAIL_PORT": (int, "25", setting("email-port"), True), + "DEMO_IMAGES_PATH": (str, "demo_images"), + "DNN_FILES_PATH": (str, "dnn_files"), + # "EMAIL_BACKEND": ( + # str, + # "django.core.mail.backends.smtp.EmailBackend", + # setting("email-backend"), + # True, + # ), + "EMAIL_HOST": (str, "", "", False, setting("email-host")), + "EMAIL_HOST_USER": (str, "", "", False, setting("email-host-user")), + "EMAIL_HOST_PASSWORD": (str, "", "", False, setting("email-host-password")), + "EMAIL_PORT": (int, "25", "25", False, setting("email-port")), "EMAIL_SUBJECT_PREFIX": ( str, "[Hope-dedupe]", + "[Hope-dedupe-dev]", + False, setting("email-subject-prefix"), - True, ), - "EMAIL_USE_LOCALTIME": (bool, False, setting("email-use-localtime"), True), - "EMAIL_USE_TLS": (bool, False, setting("email-use-tls"), True), - "EMAIL_USE_SSL": (bool, False, setting("email-use-ssl"), True), - "EMAIL_TIMEOUT": (str, None, setting("email-timeout"), True), - "LOGGING_LEVEL": (str, "CRITICAL", setting("logging-level")), + "EMAIL_USE_LOCALTIME": (bool, False, False, False, setting("email-use-localtime")), + "EMAIL_USE_TLS": (bool, False, False, False, setting("email-use-tls")), + "EMAIL_USE_SSL": (bool, False, False, False, setting("email-use-ssl")), + "EMAIL_TIMEOUT": (str, None, None, False, setting("email-timeout")), "FILE_STORAGE_DEFAULT": ( str, "django.core.files.storage.FileSystemStorage", @@ -105,98 +156,79 @@ class Group(Enum): ), "FILE_STORAGE_HOPE": ( str, - "django.core.files.storage.FileSystemStorage", + "storages.backends.azure_storage.AzureStorage", setting("storages"), ), - "MEDIA_ROOT": (str, None, setting("media-root")), - "MEDIA_URL": (str, "/media/", setting("media-url")), - "ROOT_TOKEN": (str, "", ""), - "SECRET_KEY": (str, NOT_SET, setting("secret-key")), - "SECURE_HSTS_PRELOAD": (bool, True, setting("secure-hsts-preload"), False), - "SECURE_HSTS_SECONDS": (int, 60, setting("secure-hsts-seconds")), - "SECURE_SSL_REDIRECT": (bool, True, setting("secure-ssl-redirect"), False), - "SENTRY_DSN": (str, "", "Sentry DSN"), - "SENTRY_ENVIRONMENT": (str, "production", "Sentry Environment"), - "SENTRY_URL": (str, "", "Sentry server url"), - "SESSION_COOKIE_DOMAIN": ( + "FILE_STORAGE_DNN": ( + str, + "storages.backends.azure_storage.AzureStorage", + setting("storages"), + ), + "LOG_LEVEL": (str, "CRITICAL", "DEBUG", False, setting("logging-level")), + "MEDIA_ROOT": ( + str, + "/var/media/", + "/tmp/media", # nosec + True, + setting("media-root"), + ), + "MEDIA_URL": (str, "/media/", "/media", False, setting("media-root")), # nosec + "ROOT_TOKEN_HEADER": (str, "x-root-token", "x-root-token"), + "ROOT_TOKEN": (str, ""), + "SECRET_KEY": ( str, "", - setting("std-setting-SESSION_COOKIE_DOMAIN"), + "super_sensitive_key_just_for_testing", + True, + setting("secret-key"), + ), + "SECURE_HSTS_PRELOAD": (bool, True, False, False, setting("secure-hsts-preload")), + "SECURE_HSTS_SECONDS": (int, 60, 0, False, setting("secure-hsts-seconds")), + "SECURE_SSL_REDIRECT": (bool, True, False, False, setting("secure-ssl-redirect")), + "SENTRY_DSN": (str, "", "", False, "Sentry DSN"), + "SENTRY_ENVIRONMENT": (str, "production", "develop", False, "Sentry Environment"), + "SENTRY_URL": (str, "", "", False, "Sentry server url"), + "SESSION_COOKIE_DOMAIN": ( + str, + SmartEnv.NOTSET, "localhost", + False, + setting("std-setting-SESSION_COOKIE_DOMAIN"), + ), + "SESSION_COOKIE_HTTPONLY": ( + bool, + True, + False, + False, + setting("session-cookie-httponly"), ), - "SESSION_COOKIE_HTTPONLY": (bool, True, setting("session-cookie-httponly"), False), "SESSION_COOKIE_NAME": (str, "dedupe_session", setting("session-cookie-name")), "SESSION_COOKIE_PATH": (str, "/", setting("session-cookie-path")), - "SESSION_COOKIE_SECURE": (bool, True, setting("session-cookie-secure"), False), + "SESSION_COOKIE_SECURE": ( + bool, + True, + False, + False, + setting("session-cookie-secure"), + ), "SIGNING_BACKEND": ( str, "django.core.signing.TimestampSigner", setting("signing-backend"), ), - "SOCIAL_AUTH_LOGIN_URL": (str, "/login/", "", ""), - "SOCIAL_AUTH_RAISE_EXCEPTIONS": (bool, False, "", True), - "SOCIAL_AUTH_REDIRECT_IS_HTTPS": (bool, True, "", False), - "STATIC_FILE_STORAGE": ( + "SOCIAL_AUTH_LOGIN_URL": (str, "/login/", "", False, ""), + "SOCIAL_AUTH_RAISE_EXCEPTIONS": (bool, False, True, False), + "SOCIAL_AUTH_REDIRECT_IS_HTTPS": (bool, True, False, False, ""), + "STATIC_ROOT": ( str, - "django.core.files.storage.FileSystemStorage", - setting("storages"), - ), - "STATIC_ROOT": (str, None, setting("static-root")), - "STATIC_URL": (str, "/static/", setting("static-url")), - "TIME_ZONE": (str, "UTC", setting("std-setting-TIME_ZONE")), - "AZURE_ACCOUNT_NAME": (str, ""), - "AZURE_ACCOUNT_KEY": (str, ""), - "AZURE_CUSTOM_DOMAIN": (str, ""), - "AZURE_CONNECTION_STRING": (str, ""), - "CV2DNN_PATH": (str, ""), + "/var/static", + "/tmp/static", + True, + setting("static-root"), + ), # nosec + "STATIC_URL": (str, "/static/", "/static/", False, setting("static-url")), # nosec + "TIME_ZONE": (str, "UTC", "UTC", False, setting("std-setting-TIME_ZONE")), } -class SmartEnv(Env): - def __init__(self, **scheme): # type: ignore[no-untyped-def] - self.raw = scheme - values = {k: v[:2] for k, v in scheme.items()} - super().__init__(**values) - - def get_help(self, key: str) -> str: - entry: "ConfigItem" = self.raw.get(key, "") - if len(entry) > 2: - return entry[2] - return "" - - def for_develop(self, key: str) -> Any: - entry: ConfigItem = self.raw.get(key, "") - if len(entry) > 3: - value = entry[3] - else: - value = self.get_value(key) - return value - - def storage(self, value: str) -> dict[str, str | dict[str, Any]] | None: - raw_value = self.get_value(value, str) - if not raw_value: - return None - options = {} - if "?" in raw_value: - value, options = raw_value.split("?", 1) - options = dict(parse.parse_qsl(options)) - else: - value = raw_value - - return {"BACKEND": value, "OPTIONS": options} - - def get_default(self, var: str) -> Any: - var_name = f"{self.prefix}{var}" - value = "" - if var_name in self.scheme: - var_info = self.scheme[var_name] - value = var_info[1] - try: - cast = var_info[0] - return cast(value) - except TypeError as e: - raise TypeError(f"Can't cast {var} to {cast}") from e - return value - - env = SmartEnv(**CONFIG) # type: ignore[no-untyped-call] diff --git a/src/hope_dedup_engine/config/fragments/constance.py b/src/hope_dedup_engine/config/fragments/constance.py index 9f110e0e..32fe0b38 100644 --- a/src/hope_dedup_engine/config/fragments/constance.py +++ b/src/hope_dedup_engine/config/fragments/constance.py @@ -5,11 +5,10 @@ CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend" CONSTANCE_CONFIG = { - "NEW_USER_IS_STAFF": (False, "Set any new user as staff", bool), - "NEW_USER_DEFAULT_GROUP": ( - DEFAULT_GROUP_NAME, - "Group to assign to any new user", - str, + "DNN_FILES_SOURCE": ( + "azure", + "Specifies the source from which to download the DNN model files.", + "dnn_files_source", ), "DNN_BACKEND": ( cv2.dnn.DNN_BACKEND_OPENCV, @@ -77,7 +76,7 @@ "face_encodings_model", ), "FACE_DISTANCE_THRESHOLD": ( - 0.5, + 0.4, """ Specifies the maximum allowable distance between two face embeddings for them to be considered a match. It helps determine if two faces belong to the same person by setting a threshold for similarity. Lower values result in @@ -85,16 +84,29 @@ """, float, ), + "DEDUPLICATION_SET_LOCK_ENABLED": ( + True, + "Enable or disable the lock mechanism for deduplication sets", + bool, + ), + "DEDUPLICATION_SET_LAST_ACTION_TIMEOUT": ( + 120, + "Timeout in seconds for the last action on a deduplication set", + int, + ), + "NEW_USER_IS_STAFF": (False, "Set any new user as staff", bool), + "NEW_USER_DEFAULT_GROUP": ( + DEFAULT_GROUP_NAME, + "Group to assign to any new user", + str, + ), } CONSTANCE_CONFIG_FIELDSETS = { - "User settings": { - "fields": ("NEW_USER_IS_STAFF", "NEW_USER_DEFAULT_GROUP"), - "collapse": False, - }, "Face recognition settings": { "fields": ( + "DNN_FILES_SOURCE", "DNN_BACKEND", "DNN_TARGET", "BLOB_FROM_IMAGE_SCALE_FACTOR", @@ -107,6 +119,17 @@ ), "collapse": False, }, + "Task lock settings": { + "fields": ( + "DEDUPLICATION_SET_LOCK_ENABLED", + "DEDUPLICATION_SET_LAST_ACTION_TIMEOUT", + ), + "collapse": False, + }, + "User settings": { + "fields": ("NEW_USER_IS_STAFF", "NEW_USER_DEFAULT_GROUP"), + "collapse": False, + }, } CONSTANCE_ADDITIONAL_FIELDS = { @@ -114,6 +137,13 @@ "django.forms.EmailField", {}, ], + "dnn_files_source": [ + "django.forms.ChoiceField", + { + # "choices": (("github", "GITHUB"), ("azure", "AZURE")), + "choices": (("azure", "AZURE"),), + }, + ], "dnn_backend": [ "django.forms.ChoiceField", { diff --git a/src/hope_dedup_engine/config/fragments/social_auth.py b/src/hope_dedup_engine/config/fragments/social_auth.py index 66c53170..4b40bae9 100644 --- a/src/hope_dedup_engine/config/fragments/social_auth.py +++ b/src/hope_dedup_engine/config/fragments/social_auth.py @@ -1,5 +1,9 @@ from ..settings import env # type: ignore[attr-defined] +SOCIAL_AUTH_SECRET = env.str("AZURE_CLIENT_SECRET", default="") +SOCIAL_AUTH_TENANT_ID = env("AZURE_TENANT_ID", default="") +SOCIAL_AUTH_KEY = env.str("AZURE_CLIENT_KEY", default="") + SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = [ "username", "first_name", diff --git a/src/hope_dedup_engine/config/fragments/spectacular.py b/src/hope_dedup_engine/config/fragments/spectacular.py index 732d2e24..033aa35e 100644 --- a/src/hope_dedup_engine/config/fragments/spectacular.py +++ b/src/hope_dedup_engine/config/fragments/spectacular.py @@ -1,6 +1,6 @@ SPECTACULAR_SETTINGS = { - "TITLE": "Payment Gateway API", - "DESCRIPTION": "Payment Gateway to integrate HOPE with FSP", + "TITLE": "Deduplication Engine API", + "DESCRIPTION": "Deduplication Engine to run deduplication for different datasets", "VERSION": "1.0.0", "SERVE_INCLUDE_SCHEMA": True, "SWAGGER_UI_DIST": "SIDECAR", diff --git a/src/hope_dedup_engine/config/fragments/storages.py b/src/hope_dedup_engine/config/fragments/storages.py index 11a5b2ea..76349948 100644 --- a/src/hope_dedup_engine/config/fragments/storages.py +++ b/src/hope_dedup_engine/config/fragments/storages.py @@ -1,13 +1,18 @@ -from hope_dedup_engine.config import env +from typing import Final -AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME") -AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY") -AZURE_CUSTOM_DOMAIN = env("AZURE_CUSTOM_DOMAIN") -AZURE_CONNECTION_STRING = env("AZURE_CONNECTION_STRING") - -AZURE_CONTAINER_HDE = "hde" -AZURE_CONTAINER_HOPE = "hope" - -CV2DNN_PATH = env("CV2DNN_PATH") -PROTOTXT_FILE = f"{CV2DNN_PATH}deploy.prototxt" -CAFFEMODEL_FILE = f"{CV2DNN_PATH}res10_300x300_ssd_iter_140000.caffemodel" +DNN_FILES: Final[dict[str, dict[str, str]]] = { + "prototxt": { + "filename": "deploy.prototxt.txt", + "sources": { + "github": "https://raw.githubusercontent.com/sr6033/face-detection-with-OpenCV-and-DNN/master/deploy.prototxt.txt", # noqa: E501 + "azure": "deploy.prototxt.txt", + }, + }, + "caffemodel": { + "filename": "res10_300x300_ssd_iter_140000.caffemodel", + "sources": { + "github": "https://raw.githubusercontent.com/sr6033/face-detection-with-OpenCV-and-DNN/master/res10_300x300_ssd_iter_140000.caffemodel", # noqa: E501 + "azure": "res10_300x300_ssd_iter_140000.caffemodel", + }, + }, +} diff --git a/src/hope_dedup_engine/config/settings.py b/src/hope_dedup_engine/config/settings.py index f9e4b333..ce1adf73 100644 --- a/src/hope_dedup_engine/config/settings.py +++ b/src/hope_dedup_engine/config/settings.py @@ -6,7 +6,6 @@ # BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) SETTINGS_DIR = Path(__file__).parent PACKAGE_DIR = SETTINGS_DIR.parent -DEVELOPMENT_DIR = PACKAGE_DIR.parent.parent DEBUG = env.bool("DEBUG") @@ -18,7 +17,7 @@ "hope_dedup_engine.web", "hope_dedup_engine.apps.core.apps.Config", "hope_dedup_engine.apps.security.apps.Config", - # "unicef_security", + "unicef_security", "django.contrib.contenttypes", "django.contrib.auth", "django.contrib.humanize", @@ -44,6 +43,7 @@ "hope_dedup_engine.apps.api", "hope_dedup_engine.apps.faces", "storages", + "smart_env", ) MIDDLEWARE = ( @@ -63,7 +63,6 @@ *env("AUTHENTICATION_BACKENDS"), ) - # path MEDIA_ROOT = env("MEDIA_ROOT") MEDIA_URL = env("MEDIA_URL") @@ -73,16 +72,24 @@ # # # # STATICFILES_DIRS = [] STATICFILES_FINDERS = [ - # "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] STORAGES = { + # Local filesystem + # FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage?location=/var/hope_dedupe_engine/default # noqa "default": env.storage("FILE_STORAGE_DEFAULT"), "staticfiles": env.storage("FILE_STORAGE_STATIC"), "media": env.storage("FILE_STORAGE_MEDIA"), + # Azure BLOB (readonly HOPE images). Example in case use Azurite: + # FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; # noqa "hope": env.storage("FILE_STORAGE_HOPE"), + # Azure BLOB. Example in case use Azurite: + # FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; # noqa + "dnn": env.storage("FILE_STORAGE_DNN"), } +DEFAULT_ROOT = env("DEFAULT_ROOT") +STORAGES["default"].get("OPTIONS", {}).update({"location": DEFAULT_ROOT}) SECRET_KEY = env("SECRET_KEY") ALLOWED_HOSTS = env("ALLOWED_HOSTS") @@ -94,8 +101,6 @@ TIME_ZONE = env("TIME_ZONE") -# Language code for this installation. All choices can be found here: -# http://www.i18nguy.com/unicode/language-identifiers.html LANGUAGE_CODE = "en-us" ugettext: callable = lambda s: s # noqa LANGUAGES = ( @@ -105,7 +110,7 @@ ("ar", ugettext("Arabic")), ) -DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" SITE_ID = 1 INTERNAL_IPS = ["127.0.0.1", "localhost"] @@ -129,7 +134,7 @@ TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [str(PACKAGE_DIR / "templates")], + "DIRS": [str(PACKAGE_DIR / "web/templates")], "APP_DIRS": False, "OPTIONS": { "loaders": [ @@ -164,9 +169,14 @@ }, }, "loggers": { + "environ": { + "handlers": ["console"], + "level": "CRITICAL", + "propagate": True, + }, "": { "handlers": ["console"], - "level": "DEBUG", + "level": env("LOG_LEVEL"), "propagate": True, }, }, diff --git a/src/hope_dedup_engine/state.py b/src/hope_dedup_engine/state.py index 28253cbf..76fe3c76 100644 --- a/src/hope_dedup_engine/state.py +++ b/src/hope_dedup_engine/state.py @@ -67,6 +67,7 @@ def configure(self, **kwargs: "Dict[str,Any]") -> "Iterator[None]": @contextlib.contextmanager def set(self, **kwargs: "Dict[str,Any]") -> "Iterator[None]": + pre = {} for k, v in kwargs.items(): if hasattr(self, k): diff --git a/src/hope_dedup_engine/web/templates/admin/api/deduplicationset/change_form.html b/src/hope_dedup_engine/web/templates/admin/api/deduplicationset/change_form.html new file mode 100644 index 00000000..acd63c38 --- /dev/null +++ b/src/hope_dedup_engine/web/templates/admin/api/deduplicationset/change_form.html @@ -0,0 +1,8 @@ +{% extends "admin/change_form.html" %} + +{% block object-tools-items %} + {{ block.super }} + {% include "admin_extra_buttons/includes/change_form_buttons.html" %} +{% endblock %} + +{% block pagination %}{% endblock %} diff --git a/src/hope_dedup_engine/web/templates/admin/base.html b/src/hope_dedup_engine/web/templates/admin/base.html deleted file mode 100644 index acfee3d0..00000000 --- a/src/hope_dedup_engine/web/templates/admin/base.html +++ /dev/null @@ -1,3 +0,0 @@ -{% extends "admin/base.html" %} - -{% block extrahead %}{% include "_header.html" %}{% endblock %} diff --git a/src/hope_dedup_engine/web/templates/admin/faces/dummymodel/change_list.html b/src/hope_dedup_engine/web/templates/admin/faces/dummymodel/change_list.html new file mode 100644 index 00000000..dc2e6403 --- /dev/null +++ b/src/hope_dedup_engine/web/templates/admin/faces/dummymodel/change_list.html @@ -0,0 +1,8 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} + {{ block.super }} + {% include "admin_extra_buttons/includes/change_list_buttons.html" %} +{% endblock %} + +{% block pagination %}{% endblock %} diff --git a/src/hope_dedup_engine/web/urls.py b/src/hope_dedup_engine/web/urls.py index 1c92dbf9..6fe653e9 100644 --- a/src/hope_dedup_engine/web/urls.py +++ b/src/hope_dedup_engine/web/urls.py @@ -4,6 +4,7 @@ urlpatterns = [ path("", views.index, name="home"), + path("health", views.healthcheck, name="healthcheck"), path("healthcheck", views.healthcheck, name="healthcheck"), path("healthcheck/", views.healthcheck, name="healthcheck"), ] diff --git a/tests/admin/test_admin_smoke.py b/tests/admin/test_admin_smoke.py index 2a6278df..5782b0d7 100644 --- a/tests/admin/test_admin_smoke.py +++ b/tests/admin/test_admin_smoke.py @@ -23,6 +23,7 @@ def extend(self, __iterable) -> None: [ r"django_celery_beat\.ClockedSchedule", r"contenttypes\.ContentType", + r"faces\.DummyModel", "authtoken", "social_django", "depot", @@ -95,12 +96,16 @@ def record(db, request): modeladmin = request.getfixturevalue("modeladmin") instance = modeladmin.model.objects.first() if not instance: - full_name = f"{modeladmin.model._meta.app_label}.{modeladmin.model._meta.object_name}" + full_name = ( + f"{modeladmin.model._meta.app_label}.{modeladmin.model._meta.object_name}" + ) factory = get_factory_for_model(modeladmin.model) try: instance = factory(**KWARGS.get(full_name, {})) except Exception as e: - raise Exception(f"Error creating fixture for {factory} using {KWARGS}") from e + raise Exception( + f"Error creating fixture for {factory} using {KWARGS}" + ) from e return instance diff --git a/tests/api/api_const.py b/tests/api/api_const.py index 1c2cbb9e..2accb963 100644 --- a/tests/api/api_const.py +++ b/tests/api/api_const.py @@ -2,7 +2,8 @@ BULK_IMAGE_LIST, DEDUPLICATION_SET_LIST, DUPLICATE_LIST, - IGNORED_KEYS_LIST, + IGNORED_FILENAME_LIST, + IGNORED_REFERENCE_PK_LIST, IMAGE_LIST, ) @@ -17,4 +18,5 @@ BULK_IMAGE_LIST_VIEW = f"{BULK_IMAGE_LIST}-{LIST}" BULK_IMAGE_CLEAR_VIEW = f"{BULK_IMAGE_LIST}-clear" DUPLICATE_LIST_VIEW = f"{DUPLICATE_LIST}-{LIST}" -IGNORED_KEYS_LIST_VIEW = f"{IGNORED_KEYS_LIST}-{LIST}" +IGNORED_REFERENCE_PK_LIST_VIEW = f"{IGNORED_REFERENCE_PK_LIST}-{LIST}" +IGNORED_FILENAME_LIST_VIEW = f"{IGNORED_FILENAME_LIST}-{LIST}" diff --git a/tests/api/conftest.py b/tests/api/conftest.py index 8fec4a87..e546e269 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -5,24 +5,41 @@ from pytest_factoryboy import LazyFixture, register from pytest_mock import MockerFixture from rest_framework.test import APIClient +from testutils.duplicate_finders import ( + AllDuplicateFinder, + FailingDuplicateFinder, + NoDuplicateFinder, +) from testutils.factories.api import ( + ConfigFactory, DeduplicationSetFactory, DuplicateFactory, - IgnoredKeyPairFactory, + IgnoredFilenamePairFactory, + IgnoredReferencePkPairFactory, ImageFactory, TokenFactory, ) from testutils.factories.user import ExternalSystemFactory, UserFactory -from hope_dedup_engine.apps.api.models import HDEToken +from hope_dedup_engine.apps.api.deduplication.registry import DuplicateFinder +from hope_dedup_engine.apps.api.models import DeduplicationSet, HDEToken from hope_dedup_engine.apps.security.models import User register(ExternalSystemFactory) register(UserFactory) register(DeduplicationSetFactory, external_system=LazyFixture("external_system")) -register(ImageFactory, deduplication_Set=LazyFixture("deduplication_set")) +register(ImageFactory, deduplication_set=LazyFixture("deduplication_set")) +register( + ImageFactory, + _name="second_image", + deduplication_Set=LazyFixture("deduplication_set"), +) register(DuplicateFactory, deduplication_set=LazyFixture("deduplication_set")) -register(IgnoredKeyPairFactory, deduplication_set=LazyFixture("deduplication_set")) +register(IgnoredFilenamePairFactory, deduplication_set=LazyFixture("deduplication_set")) +register( + IgnoredReferencePkPairFactory, deduplication_set=LazyFixture("deduplication_set") +) +register(ConfigFactory) @fixture @@ -60,3 +77,40 @@ def delete_model_data(mocker: MockerFixture) -> MagicMock: @fixture def start_processing(mocker: MockerFixture) -> MagicMock: return mocker.patch("hope_dedup_engine.apps.api.views.start_processing") + + +@fixture(autouse=True) +def send_notification(mocker: MockerFixture) -> MagicMock: + return mocker.patch( + "hope_dedup_engine.apps.api.models.deduplication.send_notification" + ) + + +@fixture +def duplicate_finders(mocker: MockerFixture) -> list[DuplicateFinder]: + finders = [] + mock = mocker.patch("hope_dedup_engine.apps.api.deduplication.process.get_finders") + mock.return_value = finders + return finders + + +@fixture +def all_duplicates_finder( + deduplication_set: DeduplicationSet, duplicate_finders: list[DuplicateFinder] +) -> DuplicateFinder: + duplicate_finders.append(finder := AllDuplicateFinder(deduplication_set)) + return finder + + +@fixture +def no_duplicate_finder(duplicate_finders: list[DuplicateFinder]) -> DuplicateFinder: + duplicate_finders.append(finder := NoDuplicateFinder()) + return finder + + +@fixture +def failing_duplicate_finder( + duplicate_finders: list[DuplicateFinder], +) -> DuplicateFinder: + duplicate_finders.append(finder := FailingDuplicateFinder()) + return finder diff --git a/tests/api/test_adapters.py b/tests/api/test_adapters.py new file mode 100644 index 00000000..45f81309 --- /dev/null +++ b/tests/api/test_adapters.py @@ -0,0 +1,76 @@ +from random import random +from unittest.mock import MagicMock + +from constance.test.unittest import override_config +from pytest import fixture +from pytest_mock import MockerFixture + +from hope_dedup_engine.apps.api.deduplication.adapters import DuplicateFaceFinder +from hope_dedup_engine.apps.api.models import DeduplicationSet, Image + + +@fixture +def duplication_detector(mocker: MockerFixture) -> MagicMock: + yield mocker.patch( + "hope_dedup_engine.apps.api.deduplication.adapters.DuplicationDetector" + ) + + +def test_duplicate_face_finder_uses_duplication_detector( + deduplication_set: DeduplicationSet, + image: Image, + second_image: Image, + duplication_detector: MagicMock, +) -> None: + duplication_detector.return_value.find_duplicates.return_value = iter( + ( + ( + image.filename, + second_image.filename, + distance := 0.5, + ), + ) + ) + + finder = DuplicateFaceFinder(deduplication_set) + found_pairs = tuple(finder.run()) + + duplication_detector.assert_called_once_with( + (image.filename, second_image.filename), + deduplication_set.config.face_distance_threshold, + ) + duplication_detector.return_value.find_duplicates.assert_called_once() + assert len(found_pairs) == 1 + assert found_pairs[0] == ( + image.reference_pk, + second_image.reference_pk, + 1 - distance, + ) + + +def _run_duplicate_face_finder(deduplication_set: DeduplicationSet) -> None: + finder = DuplicateFaceFinder(deduplication_set) + tuple(finder.run()) # tuple is used to make generator finish execution + + +def test_duplication_detector_is_initiated_with_correct_face_distance_threshold_value( + deduplication_set: DeduplicationSet, + duplication_detector: MagicMock, +) -> None: + # deduplication set face_distance_threshold config value is used + _run_duplicate_face_finder(deduplication_set) + duplication_detector.assert_called_once_with( + (), deduplication_set.config.face_distance_threshold + ) + face_distance_threshold = random() + with override_config(FACE_DISTANCE_THRESHOLD=face_distance_threshold): + # value from global config is used when face_distance_threshold is not set in deduplication set config + duplication_detector.reset_mock() + deduplication_set.config.face_distance_threshold = None + _run_duplicate_face_finder(deduplication_set) + duplication_detector.assert_called_once_with((), face_distance_threshold) + # value from global config is used when deduplication set has no config + duplication_detector.reset_mock() + deduplication_set.config = None + _run_duplicate_face_finder(deduplication_set) + duplication_detector.assert_called_once_with((), face_distance_threshold) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 75636dad..1d716cf7 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -7,7 +7,7 @@ BULK_IMAGE_LIST_VIEW, DEDUPLICATION_SET_DETAIL_VIEW, DEDUPLICATION_SET_LIST_VIEW, - IGNORED_KEYS_LIST_VIEW, + IGNORED_REFERENCE_PK_LIST_VIEW, IMAGE_DETAIL_VIEW, IMAGE_LIST_VIEW, JSON, @@ -33,14 +33,17 @@ (BULK_IMAGE_LIST_VIEW, HTTPMethod.POST, (PK,)), (IMAGE_DETAIL_VIEW, HTTPMethod.DELETE, (PK, PK)), (BULK_IMAGE_CLEAR_VIEW, HTTPMethod.DELETE, (PK,)), - (IGNORED_KEYS_LIST_VIEW, HTTPMethod.GET, (PK,)), - (IGNORED_KEYS_LIST_VIEW, HTTPMethod.POST, (PK,)), + (IGNORED_REFERENCE_PK_LIST_VIEW, HTTPMethod.GET, (PK,)), + (IGNORED_REFERENCE_PK_LIST_VIEW, HTTPMethod.POST, (PK,)), ) @mark.parametrize(("view_name", "method", "args"), REQUESTS) def test_anonymous_cannot_access( - anonymous_api_client: APIClient, view_name: str, method: HTTPMethod, args: tuple[Any, ...] + anonymous_api_client: APIClient, + view_name: str, + method: HTTPMethod, + args: tuple[Any, ...], ) -> None: response = getattr(anonymous_api_client, method.lower())(reverse(view_name, args)) assert response.status_code == status.HTTP_401_UNAUTHORIZED @@ -50,7 +53,9 @@ def test_anonymous_cannot_access( def test_authenticated_can_access( api_client: APIClient, view_name: str, method: HTTPMethod, args: tuple[Any, ...] ) -> None: - response = getattr(api_client, method.lower())(reverse(view_name, args), format=JSON) + response = getattr(api_client, method.lower())( + reverse(view_name, args), format=JSON + ) assert response.status_code != status.HTTP_401_UNAUTHORIZED diff --git a/tests/api/test_business_logic.py b/tests/api/test_business_logic.py deleted file mode 100644 index 7f83ee37..00000000 --- a/tests/api/test_business_logic.py +++ /dev/null @@ -1,86 +0,0 @@ -from unittest.mock import MagicMock - -from api_const import ( - DEDUPLICATION_SET_DETAIL_VIEW, - DEDUPLICATION_SET_LIST_VIEW, - DEDUPLICATION_SET_PROCESS_VIEW, - IMAGE_DETAIL_VIEW, - IMAGE_LIST_VIEW, - JSON, -) -from pytest import mark -from rest_framework import status -from rest_framework.reverse import reverse -from rest_framework.test import APIClient -from testutils.factories.api import DeduplicationSetFactory, ImageFactory - -from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image -from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer, ImageSerializer - - -def test_new_deduplication_set_status_is_clean(api_client: APIClient) -> None: - data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data - - response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) - assert response.status_code == status.HTTP_201_CREATED - deduplication_set = response.json() - assert deduplication_set["state"] == DeduplicationSet.State.CLEAN.label - - -@mark.parametrize( - "deduplication_set__state", - (DeduplicationSet.State.CLEAN, DeduplicationSet.State.DIRTY, DeduplicationSet.State.ERROR), -) -def test_deduplication_set_processing_trigger( - api_client: APIClient, start_processing: MagicMock, deduplication_set: DeduplicationSet -) -> None: - response = api_client.post(reverse(DEDUPLICATION_SET_PROCESS_VIEW, (deduplication_set.pk,))) - assert response.status_code == status.HTTP_200_OK - start_processing.assert_called_once_with(deduplication_set) - - -def test_duplicates_are_removed_before_processing( - api_client: APIClient, deduplication_set: DeduplicationSet, duplicate: Duplicate -) -> None: - assert Duplicate.objects.count() - response = api_client.post(reverse(DEDUPLICATION_SET_PROCESS_VIEW, (deduplication_set.pk,))) - assert response.status_code == status.HTTP_200_OK - assert not Duplicate.objects.count() - - -def test_new_image_makes_deduplication_set_state_dirty( - api_client: APIClient, deduplication_set: DeduplicationSet -) -> None: - assert deduplication_set.state == DeduplicationSet.State.CLEAN - response = api_client.post( - reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=ImageSerializer(ImageFactory.build()).data, format=JSON - ) - assert response.status_code == status.HTTP_201_CREATED - deduplication_set.refresh_from_db() - assert deduplication_set.state == DeduplicationSet.State.DIRTY - - -def test_image_deletion_makes_deduplication_state_dirty( - api_client: APIClient, deduplication_set: DeduplicationSet, image: Image -) -> None: - response = api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) - assert response.status_code == status.HTTP_204_NO_CONTENT - deduplication_set.refresh_from_db() - assert deduplication_set.state == DeduplicationSet.State.DIRTY - - -def test_deletion_triggers_model_data_deletion( - api_client: APIClient, deduplication_set: DeduplicationSet, delete_model_data: MagicMock -) -> None: - response = api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) - assert response.status_code == status.HTTP_204_NO_CONTENT - delete_model_data.assert_called_once_with(deduplication_set) - - -def test_unauthorized_deletion_does_not_trigger_model_data_deletion( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet, delete_model_data: MagicMock -) -> None: - response = another_system_api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) - assert response.status_code == status.HTTP_403_FORBIDDEN - delete_model_data.assert_not_called() diff --git a/tests/api/test_deduplication_set_create.py b/tests/api/test_deduplication_set_create.py index 62973d03..b36365b3 100644 --- a/tests/api/test_deduplication_set_create.py +++ b/tests/api/test_deduplication_set_create.py @@ -1,5 +1,3 @@ -from typing import Any - from api_const import DEDUPLICATION_SET_LIST_VIEW, JSON from pytest import mark from rest_framework import status @@ -8,53 +6,58 @@ from testutils.factories.api import DeduplicationSetFactory from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer +from hope_dedup_engine.apps.api.serializers import CreateDeduplicationSetSerializer def test_can_create_deduplication_set(api_client: APIClient) -> None: previous_amount = DeduplicationSet.objects.count() - data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data + data = CreateDeduplicationSetSerializer(DeduplicationSetFactory.build()).data + + response = api_client.post( + reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON + ) - response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) assert response.status_code == status.HTTP_201_CREATED assert DeduplicationSet.objects.count() == previous_amount + 1 + data = response.json() + assert data["state"] == DeduplicationSet.State.CLEAN.label + + +def test_missing_fields_handling(api_client: APIClient) -> None: + data = CreateDeduplicationSetSerializer(DeduplicationSetFactory.build()).data + del data["reference_pk"] + response = api_client.post( + reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON + ) -@mark.parametrize( - "omit", - ( - "name", - "reference_pk", - ("name", "reference_pk"), - ), -) -def test_missing_fields_handling(api_client: APIClient, omit: str | tuple[str, ...]) -> None: - data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data - missing_fields = (omit,) if isinstance(omit, str) else omit - for field in missing_fields: - del data[field] - - response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) assert response.status_code == status.HTTP_400_BAD_REQUEST errors = response.json() - assert len(errors) == len(missing_fields) - for field in missing_fields: - assert field in errors - - -@mark.parametrize( - ("field", "value"), - ( - ("name", ""), - ("name", None), - ("reference_pk", None), - ), -) -def test_invalid_values_handling(api_client: APIClient, field: str, value: Any) -> None: - data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data - data[field] = value - response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) + assert len(errors) == 1 + assert "reference_pk" in errors + + +@mark.parametrize("field", ("reference_pk", "config")) +def test_invalid_values_handling(field: str, api_client: APIClient) -> None: + data = CreateDeduplicationSetSerializer(DeduplicationSetFactory.build()).data + data[field] = None + + response = api_client.post( + reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST errors = response.json() assert len(errors) == 1 assert field in errors + + +def test_can_set_deduplication_set_without_config(api_client: APIClient) -> None: + data = CreateDeduplicationSetSerializer(DeduplicationSetFactory.build()).data + del data["config"] + + response = api_client.post( + reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON + ) + + assert response.status_code == status.HTTP_201_CREATED diff --git a/tests/api/test_deduplication_set_delete.py b/tests/api/test_deduplication_set_delete.py index 1c81fbab..4807d937 100644 --- a/tests/api/test_deduplication_set_delete.py +++ b/tests/api/test_deduplication_set_delete.py @@ -9,12 +9,19 @@ from hope_dedup_engine.apps.security.models import User -def test_can_delete_deduplication_set(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: +def test_can_delete_deduplication_set( + api_client: APIClient, + user: User, + deduplication_set: DeduplicationSet, + delete_model_data: MagicMock, +) -> None: assert not deduplication_set.deleted assert deduplication_set.updated_by is None previous_amount = DeduplicationSet.objects.count() - response = api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) + response = api_client.delete( + reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,)) + ) assert response.status_code == status.HTTP_204_NO_CONTENT # object is only marked as deleted @@ -23,11 +30,29 @@ def test_can_delete_deduplication_set(api_client: APIClient, user: User, dedupli assert deduplication_set.deleted assert deduplication_set.updated_by == user + delete_model_data.assert_called_once_with(deduplication_set) + def test_cannot_delete_deduplication_set_between_systems( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet, delete_model_data: MagicMock + another_system_api_client: APIClient, + deduplication_set: DeduplicationSet, + delete_model_data: MagicMock, ) -> None: set_count = DeduplicationSet.objects.filter(deleted=False).count() - response = another_system_api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) + response = another_system_api_client.delete( + reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,)) + ) assert response.status_code == status.HTTP_403_FORBIDDEN assert DeduplicationSet.objects.filter(deleted=False).count() == set_count + + +def test_unauthorized_deletion_does_not_trigger_model_data_deletion( + another_system_api_client: APIClient, + deduplication_set: DeduplicationSet, + delete_model_data: MagicMock, +) -> None: + response = another_system_api_client.delete( + reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + delete_model_data.assert_not_called() diff --git a/tests/api/test_deduplication_set_list.py b/tests/api/test_deduplication_set_list.py index 09481bf6..6eb77860 100644 --- a/tests/api/test_deduplication_set_list.py +++ b/tests/api/test_deduplication_set_list.py @@ -6,7 +6,9 @@ from hope_dedup_engine.apps.api.models import DeduplicationSet -def test_can_list_deduplication_sets(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: +def test_can_list_deduplication_sets( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: response = api_client.get(reverse(DEDUPLICATION_SET_LIST_VIEW)) assert response.status_code == status.HTTP_200_OK data = response.json() diff --git a/tests/api/test_deduplication_set_lock.py b/tests/api/test_deduplication_set_lock.py new file mode 100644 index 00000000..1885860f --- /dev/null +++ b/tests/api/test_deduplication_set_lock.py @@ -0,0 +1,53 @@ +from time import sleep + +from constance.test.pytest import override_config +from pytest import fail, raises +from pytest_django.fixtures import SettingsWrapper + +from hope_dedup_engine.apps.api.deduplication.lock import DeduplicationSetLock +from hope_dedup_engine.apps.api.models import DeduplicationSet + + +def test_basic_usage(deduplication_set: DeduplicationSet) -> None: + try: + lock = DeduplicationSetLock.for_deduplication_set(deduplication_set) + lock.refresh() + lock.release() + except Exception as e: + fail(f"Unexpected exception raised: {e}") + + +def test_can_serialize_and_deserialize(deduplication_set: DeduplicationSet) -> None: + try: + DeduplicationSetLock.from_string( + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)) + ) + except Exception as e: + fail(f"Unexpected exception raised: {e}") + + +def test_cannot_acquire_second_lock_for_same_deduplication_set( + deduplication_set: DeduplicationSet, +) -> None: + DeduplicationSetLock.for_deduplication_set(deduplication_set) + with raises(DeduplicationSetLock.LockNotOwnedException): + DeduplicationSetLock.for_deduplication_set(deduplication_set) + + +def test_cannot_deserialize_released_lock(deduplication_set: DeduplicationSet) -> None: + lock = DeduplicationSetLock.for_deduplication_set(deduplication_set) + serialized_lock = str(lock) + lock.release() + with raises(DeduplicationSetLock.LockNotOwnedException): + DeduplicationSetLock.from_string(serialized_lock) + + +def test_lock_is_released_after_timeout( + deduplication_set: DeduplicationSet, settings: SettingsWrapper +) -> None: + timeout = 0.1 + with override_config(DEDUPLICATION_SET_LAST_ACTION_TIMEOUT=timeout): + lock = DeduplicationSetLock.for_deduplication_set(deduplication_set) + sleep(2 * timeout) + with raises(DeduplicationSetLock.LockNotOwnedException): + lock.refresh() diff --git a/tests/api/test_deduplication_set_process.py b/tests/api/test_deduplication_set_process.py new file mode 100644 index 00000000..9227c69a --- /dev/null +++ b/tests/api/test_deduplication_set_process.py @@ -0,0 +1,43 @@ +from unittest.mock import MagicMock + +from api_const import DEDUPLICATION_SET_PROCESS_VIEW +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.utils.process import AlreadyProcessingError + + +@mark.parametrize( + "deduplication_set__state", + ( + DeduplicationSet.State.CLEAN, + DeduplicationSet.State.DIRTY, + DeduplicationSet.State.PROCESSING, + DeduplicationSet.State.ERROR, + ), +) +def test_can_trigger_deduplication_set_processing_in_any_state( + api_client: APIClient, + start_processing: MagicMock, + deduplication_set: DeduplicationSet, +) -> None: + response = api_client.post( + reverse(DEDUPLICATION_SET_PROCESS_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_200_OK + start_processing.assert_called_once_with(deduplication_set) + + +def test_cannot_trigger_deduplication_set_processing_when_already_processing( + api_client: APIClient, + start_processing: MagicMock, + deduplication_set: DeduplicationSet, +) -> None: + start_processing.side_effect = AlreadyProcessingError + response = api_client.post( + reverse(DEDUPLICATION_SET_PROCESS_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_409_CONFLICT diff --git a/tests/api/test_deduplication_set_retrieve.py b/tests/api/test_deduplication_set_retrieve.py new file mode 100644 index 00000000..397385ba --- /dev/null +++ b/tests/api/test_deduplication_set_retrieve.py @@ -0,0 +1,18 @@ +from api_const import DEDUPLICATION_SET_DETAIL_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer + + +def test_can_retrieve_deduplication_set( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + response = api_client.get( + reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data == DeduplicationSetSerializer(deduplication_set).data diff --git a/tests/api/test_duplicate_list.py b/tests/api/test_duplicate_list.py index eae8401a..a1ed1695 100644 --- a/tests/api/test_duplicate_list.py +++ b/tests/api/test_duplicate_list.py @@ -1,13 +1,22 @@ +from collections.abc import Callable +from operator import attrgetter +from urllib.parse import urlencode + from api_const import DUPLICATE_LIST_VIEW +from factory.fuzzy import FuzzyText +from pytest import mark from rest_framework import status from rest_framework.reverse import reverse from rest_framework.test import APIClient from hope_dedup_engine.apps.api.models import DeduplicationSet from hope_dedup_engine.apps.api.models.deduplication import Duplicate +from hope_dedup_engine.apps.api.views import REFERENCE_PK -def test_can_list_duplicates(api_client: APIClient, deduplication_set: DeduplicationSet, duplicate: Duplicate) -> None: +def test_can_list_duplicates( + api_client: APIClient, deduplication_set: DeduplicationSet, duplicate: Duplicate +) -> None: response = api_client.get(reverse(DUPLICATE_LIST_VIEW, (deduplication_set.pk,))) assert response.status_code == status.HTTP_200_OK data = response.json() @@ -15,8 +24,39 @@ def test_can_list_duplicates(api_client: APIClient, deduplication_set: Deduplica def test_cannot_list_duplicates_between_systems( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet, duplicate: Duplicate + another_system_api_client: APIClient, + deduplication_set: DeduplicationSet, + duplicate: Duplicate, ) -> None: assert DeduplicationSet.objects.count() - response = another_system_api_client.get(reverse(DUPLICATE_LIST_VIEW, (deduplication_set.pk,))) + response = another_system_api_client.get( + reverse(DUPLICATE_LIST_VIEW, (deduplication_set.pk,)) + ) assert response.status_code == status.HTTP_403_FORBIDDEN + + +@mark.parametrize( + ("filter_value_getter", "expected_amount"), + ( + # filter by first_reference_pk + (attrgetter("first_reference_pk"), 1), + # filter by second_reference_pk + (attrgetter("second_reference_pk"), 1), + # filter by random string + (lambda _: FuzzyText().fuzz(), 0), + ), +) +def test_can_filter_by_reference_pk( + api_client: APIClient, + deduplication_set: DeduplicationSet, + duplicate: Duplicate, + filter_value_getter: Callable[[Duplicate], str], + expected_amount: int, +) -> None: + url = f"{reverse(DUPLICATE_LIST_VIEW, (deduplication_set.pk, ))}?" + urlencode( + {REFERENCE_PK: filter_value_getter(duplicate)} + ) + response = api_client.get(url) + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert len(data) == expected_amount diff --git a/tests/api/test_find_duplicates.py b/tests/api/test_find_duplicates.py new file mode 100644 index 00000000..3a9a6593 --- /dev/null +++ b/tests/api/test_find_duplicates.py @@ -0,0 +1,117 @@ +from unittest.mock import MagicMock + +from pytest import raises + +from hope_dedup_engine.apps.api.deduplication.lock import DeduplicationSetLock +from hope_dedup_engine.apps.api.deduplication.process import find_duplicates +from hope_dedup_engine.apps.api.deduplication.registry import DuplicateFinder +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image + + +def test_previous_results_are_removed_before_processing( + deduplication_set: DeduplicationSet, + duplicate: Duplicate, + duplicate_finders: list[DuplicateFinder], +) -> None: + assert deduplication_set.duplicate_set.count() + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + assert not deduplication_set.duplicate_set.count() + + +def test_duplicates_are_stored( + deduplication_set: DeduplicationSet, + image: Image, + second_image: Image, + all_duplicates_finder: DuplicateFinder, +) -> None: + assert not deduplication_set.duplicate_set.count() + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + assert deduplication_set.duplicate_set.count() + + +def test_ignored_reference_pk_pairs( + deduplication_set: DeduplicationSet, + image: Image, + second_image: Image, + all_duplicates_finder: DuplicateFinder, +) -> None: + assert not deduplication_set.duplicate_set.count() + ignored_reference_pk_pair = deduplication_set.ignoredreferencepkpair_set.create( + first=image.reference_pk, + second=second_image.reference_pk, + ) + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + ignored_reference_pk_pair.delete() + assert not deduplication_set.duplicate_set.count() + + +def test_ignored_filename_pairs( + deduplication_set: DeduplicationSet, + image: Image, + second_image: Image, + all_duplicates_finder: DuplicateFinder, +) -> None: + assert not deduplication_set.duplicate_set.count() + ignored_filename_pair = deduplication_set.ignoredfilenamepair_set.create( + first=image.filename, + second=second_image.filename, + ) + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + ignored_filename_pair.delete() + assert not deduplication_set.duplicate_set.count() + + +def test_weight_is_taken_into_account( + deduplication_set: DeduplicationSet, + image: Image, + second_image: Image, + all_duplicates_finder: DuplicateFinder, + no_duplicate_finder: DuplicateFinder, +) -> None: + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + assert deduplication_set.duplicate_set.first().score == 0.5 + + +def test_notification_sent_on_successful_run( + deduplication_set: DeduplicationSet, + duplicate_finders: list[DuplicateFinder], + send_notification: MagicMock, +) -> None: + send_notification.reset_mock() # remove notification for CREATE state + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + send_notification.assert_called_once_with(deduplication_set.notification_url) + + +def test_notification_sent_on_failure( + deduplication_set: DeduplicationSet, + failing_duplicate_finder: DuplicateFinder, + send_notification: MagicMock, +) -> None: + send_notification.reset_mock() # remove notification for CREATE state + with raises(Exception): + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + deduplication_set.refresh_from_db() + assert deduplication_set.state == deduplication_set.State.ERROR + send_notification.assert_called_once_with(deduplication_set.notification_url) diff --git a/tests/api/test_finder_registry.py b/tests/api/test_finder_registry.py new file mode 100644 index 00000000..89327a5f --- /dev/null +++ b/tests/api/test_finder_registry.py @@ -0,0 +1,12 @@ +from hope_dedup_engine.apps.api.deduplication.adapters import DuplicateFaceFinder +from hope_dedup_engine.apps.api.deduplication.registry import get_finders +from hope_dedup_engine.apps.api.models import DeduplicationSet + + +def test_get_finders_returns_duplicate_face_finder( + deduplication_set: DeduplicationSet, +) -> None: + assert any( + isinstance(finder, DuplicateFaceFinder) + for finder in get_finders(deduplication_set) + ) diff --git a/tests/api/test_ignored_filename_create.py b/tests/api/test_ignored_filename_create.py new file mode 100644 index 00000000..80b3ee1f --- /dev/null +++ b/tests/api/test_ignored_filename_create.py @@ -0,0 +1,111 @@ +from api_const import IGNORED_FILENAME_LIST_VIEW, JSON +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import IgnoredFilenamePairFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredFilenamePair +from hope_dedup_engine.apps.api.serializers import IgnoredFilenamePairSerializer +from hope_dedup_engine.apps.security.models import User + + +def test_can_create_ignored_filename_pair( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + previous_amount = IgnoredFilenamePair.objects.filter( + deduplication_set=deduplication_set + ).count() + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_201_CREATED + assert ( + IgnoredFilenamePair.objects.filter(deduplication_set=deduplication_set).count() + == previous_amount + 1 + ) + + +def test_cannot_create_ignored_filename_pair_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + previous_amount = IgnoredFilenamePair.objects.filter( + deduplication_set=deduplication_set + ).count() + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + + response = another_system_api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert ( + IgnoredFilenamePair.objects.filter(deduplication_set=deduplication_set).count() + == previous_amount + ) + + +INVALID_FILENAME_VALUES = "", None + + +@mark.parametrize("first_filename", INVALID_FILENAME_VALUES) +@mark.parametrize("second_filename", INVALID_FILENAME_VALUES) +def test_invalid_values_handling( + api_client: APIClient, + deduplication_set: DeduplicationSet, + first_filename: str | None, + second_filename: str | None, +) -> None: + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + data["first"] = first_filename + data["second"] = second_filename + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert len(errors) == 2 + assert "first" in errors + assert "second" in errors + + +def test_missing_filename_handling( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + del data["first"], data["second"] + + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert "first" in errors + assert "second" in errors + + +def test_deduplication_set_is_updated( + api_client: APIClient, user: User, deduplication_set: DeduplicationSet +) -> None: + assert deduplication_set.updated_by is None + + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + + assert response.status_code == status.HTTP_201_CREATED + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_ignored_filename_list.py b/tests/api/test_ignored_filename_list.py new file mode 100644 index 00000000..f07ce386 --- /dev/null +++ b/tests/api/test_ignored_filename_list.py @@ -0,0 +1,35 @@ +from api_const import IGNORED_FILENAME_LIST_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredFilenamePair + + +def test_can_list_ignored_filename_pairs( + api_client: APIClient, + deduplication_set: DeduplicationSet, + ignored_filename_pair: IgnoredFilenamePair, +) -> None: + response = api_client.get( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_200_OK + ignored_filename_pairs = response.json() + assert len(ignored_filename_pairs) + assert ( + len(ignored_filename_pairs) + == IgnoredFilenamePair.objects.filter( + deduplication_set=deduplication_set + ).count() + ) + + +def test_cannot_list_ignored_filename_pairs_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + response = another_system_api_client.get( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/api/test_ignored_keys_create.py b/tests/api/test_ignored_keys_create.py deleted file mode 100644 index 317e39c0..00000000 --- a/tests/api/test_ignored_keys_create.py +++ /dev/null @@ -1,74 +0,0 @@ -from api_const import IGNORED_KEYS_LIST_VIEW, JSON -from pytest import mark -from rest_framework import status -from rest_framework.reverse import reverse -from rest_framework.test import APIClient -from testutils.factories.api import IgnoredKeyPairFactory - -from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair -from hope_dedup_engine.apps.api.serializers import IgnoredKeyPairSerializer -from hope_dedup_engine.apps.security.models import User - - -def test_can_create_ignored_key_pair(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: - previous_amount = IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data - - response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) - assert response.status_code == status.HTTP_201_CREATED - assert IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + 1 - - -def test_cannot_create_ignored_key_pair_between_systems( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet -) -> None: - previous_amount = IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data - - response = another_system_api_client.post( - reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON - ) - assert response.status_code == status.HTTP_403_FORBIDDEN - assert IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() == previous_amount - - -INVALID_PK_VALUES = "", None - - -@mark.parametrize("first_pk", INVALID_PK_VALUES) -@mark.parametrize("second_pk", INVALID_PK_VALUES) -def test_invalid_values_handling( - api_client: APIClient, deduplication_set: DeduplicationSet, first_pk: str | None, second_pk: str | None -) -> None: - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data - data["first_reference_pk"] = first_pk - data["second_reference_pk"] = second_pk - response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) - assert response.status_code == status.HTTP_400_BAD_REQUEST - errors = response.json() - assert len(errors) == 2 - assert "first_reference_pk" in errors - assert "second_reference_pk" in errors - - -def test_missing_pk_handling(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data - del data["first_reference_pk"], data["second_reference_pk"] - - response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) - assert response.status_code == status.HTTP_400_BAD_REQUEST - errors = response.json() - assert "first_reference_pk" in errors - assert "second_reference_pk" in errors - - -def test_deduplication_set_is_updated(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: - assert deduplication_set.updated_by is None - - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data - response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) - - assert response.status_code == status.HTTP_201_CREATED - deduplication_set.refresh_from_db() - assert deduplication_set.updated_by == user diff --git a/tests/api/test_ignored_keys_list.py b/tests/api/test_ignored_keys_list.py deleted file mode 100644 index 8affedf4..00000000 --- a/tests/api/test_ignored_keys_list.py +++ /dev/null @@ -1,24 +0,0 @@ -from api_const import IGNORED_KEYS_LIST_VIEW -from rest_framework import status -from rest_framework.reverse import reverse -from rest_framework.test import APIClient - -from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair - - -def test_can_list_ignored_key_pairs( - api_client: APIClient, deduplication_set: DeduplicationSet, ignored_key_pair: IgnoredKeyPair -) -> None: - response = api_client.get(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,))) - assert response.status_code == status.HTTP_200_OK - ignored_key_pairs = response.json() - assert len(ignored_key_pairs) - assert len(ignored_key_pairs) == IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() - - -def test_cannot_list_ignored_key_pairs_between_systems( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet -) -> None: - response = another_system_api_client.get(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,))) - assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/api/test_ignored_reference_pk_create.py b/tests/api/test_ignored_reference_pk_create.py new file mode 100644 index 00000000..08c43762 --- /dev/null +++ b/tests/api/test_ignored_reference_pk_create.py @@ -0,0 +1,118 @@ +from api_const import IGNORED_REFERENCE_PK_LIST_VIEW, JSON +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import IgnoredReferencePkPairFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredReferencePkPair +from hope_dedup_engine.apps.api.serializers import IgnoredReferencePkPairSerializer +from hope_dedup_engine.apps.security.models import User + + +def test_can_create_ignored_reference_pk_pair( + api_client: APIClient, + deduplication_set: DeduplicationSet, +) -> None: + previous_amount = IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data + + response = api_client.post( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_201_CREATED + assert ( + IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() + == previous_amount + 1 + ) + + +def test_cannot_create_ignored_reference_pk_pair_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + previous_amount = IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data + + response = another_system_api_client.post( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert ( + IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() + == previous_amount + ) + + +INVALID_PK_VALUES = "", None + + +@mark.parametrize("first_pk", INVALID_PK_VALUES) +@mark.parametrize("second_pk", INVALID_PK_VALUES) +def test_invalid_values_handling( + api_client: APIClient, + deduplication_set: DeduplicationSet, + first_pk: str | None, + second_pk: str | None, +) -> None: + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data + data["first"] = first_pk + data["second"] = second_pk + response = api_client.post( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert len(errors) == 2 + assert "first" in errors + assert "second" in errors + + +def test_missing_pk_handling( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data + del data["first"], data["second"] + + response = api_client.post( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert "first" in errors + assert "second" in errors + + +def test_deduplication_set_is_updated( + api_client: APIClient, + user: User, + deduplication_set: DeduplicationSet, +) -> None: + assert deduplication_set.updated_by is None + + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data + response = api_client.post( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + + assert response.status_code == status.HTTP_201_CREATED + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_ignored_reference_pk_list.py b/tests/api/test_ignored_reference_pk_list.py new file mode 100644 index 00000000..acf40291 --- /dev/null +++ b/tests/api/test_ignored_reference_pk_list.py @@ -0,0 +1,35 @@ +from api_const import IGNORED_REFERENCE_PK_LIST_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredReferencePkPair + + +def test_can_list_ignored_reference_pk_pairs( + api_client: APIClient, + deduplication_set: DeduplicationSet, + ignored_reference_pk_pair: IgnoredReferencePkPair, +) -> None: + response = api_client.get( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_200_OK + ignored_reference_pk_pairs = response.json() + assert len(ignored_reference_pk_pairs) + assert ( + len(ignored_reference_pk_pairs) + == IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() + ) + + +def test_cannot_list_ignored_reference_pk_pairs_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + response = another_system_api_client.get( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/api/test_image_bulk_create.py b/tests/api/test_image_bulk_create.py index 6b91dd8a..a001ab2f 100644 --- a/tests/api/test_image_bulk_create.py +++ b/tests/api/test_image_bulk_create.py @@ -9,9 +9,13 @@ from hope_dedup_engine.apps.security.models import User -def test_can_bulk_create_images(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: +def test_can_bulk_create_images( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: data = ImageSerializer(ImageFactory.build_batch(10), many=True).data - response = api_client.post(reverse(BULK_IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + response = api_client.post( + reverse(BULK_IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) assert response.status_code == status.HTTP_201_CREATED @@ -25,11 +29,15 @@ def test_cannot_bulk_create_images_between_systems( assert response.status_code == status.HTTP_403_FORBIDDEN -def test_deduplication_set_is_updated(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: +def test_deduplication_set_is_updated( + api_client: APIClient, user: User, deduplication_set: DeduplicationSet +) -> None: assert deduplication_set.updated_by is None data = ImageSerializer(ImageFactory.build_batch(10), many=True).data - response = api_client.post(reverse(BULK_IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + response = api_client.post( + reverse(BULK_IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) assert response.status_code == status.HTTP_201_CREATED deduplication_set.refresh_from_db() diff --git a/tests/api/test_image_bulk_delete.py b/tests/api/test_image_bulk_delete.py index a1898483..7b4ae469 100644 --- a/tests/api/test_image_bulk_delete.py +++ b/tests/api/test_image_bulk_delete.py @@ -8,27 +8,42 @@ from hope_dedup_engine.apps.security.models import User -def test_can_delete_all_images(api_client: APIClient, deduplication_set: DeduplicationSet, image: Image) -> None: +def test_can_delete_all_images( + api_client: APIClient, deduplication_set: DeduplicationSet, image: Image +) -> None: image_count = Image.objects.filter(deduplication_set=deduplication_set).count() - response = api_client.delete(reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,))) + response = api_client.delete( + reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,)) + ) assert response.status_code == status.HTTP_204_NO_CONTENT - assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count - 1 + assert ( + Image.objects.filter(deduplication_set=deduplication_set).count() + == image_count - 1 + ) def test_cannot_delete_images_between_systems( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet, image: Image + another_system_api_client: APIClient, + deduplication_set: DeduplicationSet, + image: Image, ) -> None: image_count = Image.objects.filter(deduplication_set=deduplication_set).count() - response = another_system_api_client.delete(reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,))) + response = another_system_api_client.delete( + reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,)) + ) assert response.status_code == status.HTTP_403_FORBIDDEN - assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count + assert ( + Image.objects.filter(deduplication_set=deduplication_set).count() == image_count + ) def test_deduplication_set_is_updated( api_client: APIClient, user: User, deduplication_set: DeduplicationSet, image: Image ) -> None: assert deduplication_set.updated_by is None - response = api_client.delete(reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,))) + response = api_client.delete( + reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,)) + ) assert response.status_code == status.HTTP_204_NO_CONTENT deduplication_set.refresh_from_db() assert deduplication_set.updated_by == user diff --git a/tests/api/test_image_create.py b/tests/api/test_image_create.py index fe87ca16..bff3582f 100644 --- a/tests/api/test_image_create.py +++ b/tests/api/test_image_create.py @@ -11,13 +11,25 @@ from hope_dedup_engine.apps.security.models import User -def test_can_create_image(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: +def test_can_create_image( + api_client: APIClient, + deduplication_set: DeduplicationSet, +) -> None: previous_amount = Image.objects.filter(deduplication_set=deduplication_set).count() data = ImageSerializer(ImageFactory.build()).data + assert deduplication_set.state == DeduplicationSet.State.CLEAN - response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + response = api_client.post( + reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) assert response.status_code == status.HTTP_201_CREATED - assert Image.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + 1 + assert ( + Image.objects.filter(deduplication_set=deduplication_set).count() + == previous_amount + 1 + ) + + deduplication_set.refresh_from_db() + assert deduplication_set.state == DeduplicationSet.State.DIRTY def test_cannot_create_image_between_systems( @@ -26,9 +38,14 @@ def test_cannot_create_image_between_systems( previous_amount = Image.objects.filter(deduplication_set=deduplication_set).count() data = ImageSerializer(ImageFactory.build()).data - response = another_system_api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + response = another_system_api_client.post( + reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) assert response.status_code == status.HTTP_403_FORBIDDEN - assert Image.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + assert ( + Image.objects.filter(deduplication_set=deduplication_set).count() + == previous_amount + ) @mark.parametrize( @@ -43,28 +60,40 @@ def test_invalid_values_handling( ) -> None: data = ImageSerializer(ImageFactory.build()).data data["filename"] = filename - response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + response = api_client.post( + reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) assert response.status_code == status.HTTP_400_BAD_REQUEST errors = response.json() assert len(errors) == 1 assert "filename" in errors -def test_missing_filename_handling(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: +def test_missing_filename_handling( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: data = ImageSerializer(ImageFactory.build()).data del data["filename"] - response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + response = api_client.post( + reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) assert response.status_code == status.HTTP_400_BAD_REQUEST errors = response.json() assert "filename" in errors -def test_deduplication_set_is_updated(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: +def test_deduplication_set_is_updated( + api_client: APIClient, + user: User, + deduplication_set: DeduplicationSet, +) -> None: assert deduplication_set.updated_by is None data = ImageSerializer(ImageFactory.build()).data - response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + response = api_client.post( + reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) assert response.status_code == status.HTTP_201_CREATED deduplication_set.refresh_from_db() diff --git a/tests/api/test_image_delete.py b/tests/api/test_image_delete.py index e0745847..e987e58c 100644 --- a/tests/api/test_image_delete.py +++ b/tests/api/test_image_delete.py @@ -7,27 +7,51 @@ from hope_dedup_engine.apps.security.models import User -def test_can_delete_image(api_client: APIClient, deduplication_set: DeduplicationSet, image: Image) -> None: +def test_can_delete_image( + api_client: APIClient, + deduplication_set: DeduplicationSet, + image: Image, +) -> None: image_count = Image.objects.filter(deduplication_set=deduplication_set).count() - response = api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) + assert deduplication_set.state == DeduplicationSet.State.CLEAN + response = api_client.delete( + reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk)) + ) assert response.status_code == status.HTTP_204_NO_CONTENT - assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count - 1 + assert ( + Image.objects.filter(deduplication_set=deduplication_set).count() + == image_count - 1 + ) + + deduplication_set.refresh_from_db() + assert deduplication_set.state == DeduplicationSet.State.DIRTY def test_cannot_delete_image_between_systems( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet, image: Image + another_system_api_client: APIClient, + deduplication_set: DeduplicationSet, + image: Image, ) -> None: image_count = Image.objects.filter(deduplication_set=deduplication_set).count() - response = another_system_api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) + response = another_system_api_client.delete( + reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk)) + ) assert response.status_code == status.HTTP_403_FORBIDDEN - assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count + assert ( + Image.objects.filter(deduplication_set=deduplication_set).count() == image_count + ) def test_deduplication_set_is_updated( - api_client: APIClient, user: User, deduplication_set: DeduplicationSet, image: Image + api_client: APIClient, + user: User, + deduplication_set: DeduplicationSet, + image: Image, ) -> None: assert deduplication_set.updated_by is None - response = api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) + response = api_client.delete( + reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk)) + ) assert response.status_code == status.HTTP_204_NO_CONTENT deduplication_set.refresh_from_db() assert deduplication_set.updated_by == user diff --git a/tests/api/test_image_list.py b/tests/api/test_image_list.py index e02e5f54..a2cb09d3 100644 --- a/tests/api/test_image_list.py +++ b/tests/api/test_image_list.py @@ -7,16 +7,22 @@ from hope_dedup_engine.apps.api.models.deduplication import Image -def test_can_list_images(api_client: APIClient, deduplication_set: DeduplicationSet, image: Image) -> None: +def test_can_list_images( + api_client: APIClient, deduplication_set: DeduplicationSet, image: Image +) -> None: response = api_client.get(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,))) assert response.status_code == status.HTTP_200_OK images = response.json() assert len(images) - assert len(images) == Image.objects.filter(deduplication_set=deduplication_set).count() + assert ( + len(images) == Image.objects.filter(deduplication_set=deduplication_set).count() + ) def test_cannot_list_images_between_systems( another_system_api_client: APIClient, deduplication_set: DeduplicationSet ) -> None: - response = another_system_api_client.get(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,))) + response = another_system_api_client.get( + reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)) + ) assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/api/test_utils.py b/tests/api/test_utils.py index 64385448..2b809da7 100644 --- a/tests/api/test_utils.py +++ b/tests/api/test_utils.py @@ -1,28 +1,46 @@ from unittest.mock import MagicMock from pytest import fixture, mark -from pytest_mock import MockerFixture +from pytest_mock import MockFixture +from requests import RequestException -from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.utils import REQUEST_TIMEOUT, send_notification +from hope_dedup_engine.apps.api.utils.notification import ( + REQUEST_TIMEOUT, + send_notification, +) @fixture -def requests_get_mock(mocker: MockerFixture) -> MagicMock: - return mocker.patch("hope_dedup_engine.apps.api.utils.requests.get") +def requests_get(mocker: MockFixture) -> MagicMock: + return mocker.patch("hope_dedup_engine.apps.api.utils.notification.requests.get") -@mark.parametrize("deduplication_set__notification_url", ("https://example.com",)) -def test_notification_is_sent_when_url_is_set( - requests_get_mock: MagicMock, deduplication_set: DeduplicationSet +@fixture +def sentry_sdk_capture_exception(mocker: MockFixture) -> MagicMock: + return mocker.patch( + "hope_dedup_engine.apps.api.utils.notification.sentry_sdk.capture_exception" + ) + + +@mark.parametrize( + ("url", "http_request_sent"), (("https://example.com", True), (None, False)) +) +def test_send_notification( + url: str | None, + http_request_sent: bool, + requests_get: MagicMock, ) -> None: - send_notification(deduplication_set) - requests_get_mock.assert_called_once_with(deduplication_set.notification_url, timeout=REQUEST_TIMEOUT) + send_notification(url) + if http_request_sent: + requests_get.assert_called_once_with(url, timeout=REQUEST_TIMEOUT) + else: + requests_get.assert_not_called() -@mark.parametrize("deduplication_set__notification_url", (None,)) -def test_notification_is_not_sent_when_url_is_not_set( - requests_get_mock: MagicMock, deduplication_set: DeduplicationSet +def test_exception_is_sent_to_sentry( + requests_get: MagicMock, sentry_sdk_capture_exception: MagicMock ) -> None: - send_notification(deduplication_set) - requests_get_mock.assert_not_called() + exception = RequestException() + requests_get.side_effect = exception + send_notification("https://example.com") + sentry_sdk_capture_exception.assert_called_once_with(exception) diff --git a/tests/conftest.py b/tests/conftest.py index df46bfbc..742cb05a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,15 +17,25 @@ def pytest_configure(config): os.environ.update(DJANGO_SETTINGS_MODULE="hope_dedup_engine.config.settings") os.environ.setdefault("MEDIA_ROOT", "/tmp/static/") os.environ.setdefault("STATIC_ROOT", "/tmp/media/") + os.environ.setdefault("DEFAULT_ROOT", "/tmp/default/") + os.environ.setdefault("TEST_EMAIL_SENDER", "sender@example.com") os.environ.setdefault("TEST_EMAIL_RECIPIENT", "recipient@example.com") os.environ["MAILJET_API_KEY"] = "11" os.environ["MAILJET_SECRET_KEY"] = "11" - os.environ["FILE_STORAGE_DEFAULT"] = "django.core.files.storage.FileSystemStorage?location=/tmp/hde/storage/" - os.environ["FILE_STORAGE_STATIC"] = "django.core.files.storage.FileSystemStorage?location=/tmp/hde/static/" - os.environ["FILE_STORAGE_MEDIA"] = "django.core.files.storage.FileSystemStorage?location=/tmp/hde/storage/" - os.environ["FILE_STORAGE_HOPE"] = "django.core.files.storage.FileSystemStorage?location=/tmp/hde/hope/" + os.environ["FILE_STORAGE_DEFAULT"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/storage/" + ) + os.environ["FILE_STORAGE_STATIC"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/static/" + ) + os.environ["FILE_STORAGE_MEDIA"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/storage/" + ) + os.environ["FILE_STORAGE_HOPE"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/hope/" + ) os.environ["SOCIAL_AUTH_REDIRECT_IS_HTTPS"] = "0" os.environ["CELERY_TASK_ALWAYS_EAGER"] = "0" os.environ["SECURE_HSTS_PRELOAD"] = "0" diff --git a/tests/extras/demoapp/compose.yml b/tests/extras/demoapp/compose.yml new file mode 100644 index 00000000..9c3dd2e6 --- /dev/null +++ b/tests/extras/demoapp/compose.yml @@ -0,0 +1,132 @@ +x-common: &common + image: unicef/hope-dedupe-engine:release-0.1 + platform: linux/amd64 + environment: + - ADMIN_EMAIL=adm@hde.org + - ADMIN_PASSWORD=123 + - ALLOWED_HOSTS=localhost,127.0.0.1 + - CACHE_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/9 + - CELERY_TASK_ALWAYS_EAGER=False + - CSRF_COOKIE_SECURE=False + - DATABASE_URL=postgres://hde:password@db:5432/hope_dedupe_engine + - DEFAULT_ROOT=/var/hope_dedupe_engine/default + - DJANGO_SETTINGS_MODULE=hope_dedup_engine.config.settings + - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage?location=/var/hope_dedupe_engine/default + - FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - MEDIA_ROOT=/var/hope_dedupe_engine/media + - PYTHONPATH=/code/__pypackages__/3.12/lib/ + - SECRET_KEY=very-secret-key + - SECURE_SSL_REDIRECT=False + - SESSION_COOKIE_DOMAIN= + - SESSION_COOKIE_SECURE=False + - SOCIAL_AUTH_REDIRECT_IS_HTTPS=False + - STATIC_ROOT=/var/hope_dedupe_engine/static + volumes: + - ./demo_images:/code/__pypackages__/3.12/tests/extras/demoapp/demo_images + - ./dnn_files:/code/__pypackages__/3.12/tests/extras/demoapp/dnn_files + restart: unless-stopped + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy +x-celery: &celery + <<: *common + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + backend: + condition: service_healthy + +services: + backend: + <<: *common + container_name: hde_app + ports: + - 8000:8000 + command: > + /bin/sh -c " + django-admin demo --skip-checks && + django-admin upgrade && + docker-entrypoint.sh run + " + + healthcheck: + test: ["CMD", "pidof", "uwsgi"] + interval: 10s + timeout: 5s + retries: 5 + + db: + image: postgres:16 + container_name: hde_db + environment: + - POSTGRES_USER=hde + - POSTGRES_PASSWORD=password + - POSTGRES_DB=hope_dedupe_engine + volumes: + - postgres_data:/var/lib/postgresql/data/ + ports: + - 5432:5432 + restart: always + healthcheck: + test: ["CMD", "pg_isready", "-U", "hde", "-d", "hope_dedupe_engine"] + start_period: 5s + start_interval: 1s + interval: 5s + timeout: 4s + retries: 5 + + redis: + image: redis:7.2 + container_name: hde_redis + ports: + - 6379:6379 + restart: always + healthcheck: + test: ["CMD", "redis-cli", "ping"] + start_period: 5s + start_interval: 1s + interval: 5s + timeout: 4s + retries: 5 + + azurite: + image: mcr.microsoft.com/azure-storage/azurite + container_name: hde_azurite + command: "azurite -l /workspace -d /workspace/debug.log --blobPort 10000 --blobHost 0.0.0.0 --loose --silent" + ports: + - "10000:10000" # Blob service + restart: always + volumes: + - azurite_data:/workspace + + celery_worker: + <<: *celery + container_name: hde_worker + command: worker + + celery_beat: + <<: *celery + container_name: hde_beat + command: beat + + celery-flower: + <<: *celery + ports: + - 5555:5555 + command: > + /bin/sh -c " + exec celery -A hope_dedup_engine.config.celery flower --address=0.0.0.0 + " + + +volumes: + azurite_data: + postgres_data: diff --git a/tests/extras/demoapp/demo_images/Aaron_Eckhart_0001.jpg b/tests/extras/demoapp/demo_images/Aaron_Eckhart_0001.jpg new file mode 100644 index 00000000..4d2fb8db Binary files /dev/null and b/tests/extras/demoapp/demo_images/Aaron_Eckhart_0001.jpg differ diff --git a/tests/extras/demoapp/demo_images/Aaron_Guiel_0001.jpg b/tests/extras/demoapp/demo_images/Aaron_Guiel_0001.jpg new file mode 100644 index 00000000..c2fb5d0f Binary files /dev/null and b/tests/extras/demoapp/demo_images/Aaron_Guiel_0001.jpg differ diff --git a/tests/extras/demoapp/demo_images/Aaron_Peirsol_0001.jpg b/tests/extras/demoapp/demo_images/Aaron_Peirsol_0001.jpg new file mode 100644 index 00000000..b1cc3287 Binary files /dev/null and b/tests/extras/demoapp/demo_images/Aaron_Peirsol_0001.jpg differ diff --git a/tests/extras/demoapp/demo_images/Aaron_Peirsol_0002.jpg b/tests/extras/demoapp/demo_images/Aaron_Peirsol_0002.jpg new file mode 100644 index 00000000..cf561ab4 Binary files /dev/null and b/tests/extras/demoapp/demo_images/Aaron_Peirsol_0002.jpg differ diff --git a/tests/extras/demoapp/demo_images/Cathy_Freeman_0001.jpg b/tests/extras/demoapp/demo_images/Cathy_Freeman_0001.jpg new file mode 100644 index 00000000..f2d6b5d8 Binary files /dev/null and b/tests/extras/demoapp/demo_images/Cathy_Freeman_0001.jpg differ diff --git a/tests/extras/demoapp/demo_images/Cathy_Freeman_0002.jpg b/tests/extras/demoapp/demo_images/Cathy_Freeman_0002.jpg new file mode 100644 index 00000000..e4f8f62c Binary files /dev/null and b/tests/extras/demoapp/demo_images/Cathy_Freeman_0002.jpg differ diff --git a/tests/extras/demoapp/demo_images/Ziwang_Xu_0001.jpg b/tests/extras/demoapp/demo_images/Ziwang_Xu_0001.jpg new file mode 100644 index 00000000..2665ebc0 Binary files /dev/null and b/tests/extras/demoapp/demo_images/Ziwang_Xu_0001.jpg differ diff --git a/tests/extras/demoapp/demo_images/Zoe_Ball_0001.jpg b/tests/extras/demoapp/demo_images/Zoe_Ball_0001.jpg new file mode 100644 index 00000000..f26223d2 Binary files /dev/null and b/tests/extras/demoapp/demo_images/Zoe_Ball_0001.jpg differ diff --git a/tests/extras/demoapp/demo_images/without_face.jpg b/tests/extras/demoapp/demo_images/without_face.jpg new file mode 100644 index 00000000..e3b70996 Binary files /dev/null and b/tests/extras/demoapp/demo_images/without_face.jpg differ diff --git a/tests/extras/demoapp/dnn_files/deploy.prototxt.txt b/tests/extras/demoapp/dnn_files/deploy.prototxt.txt new file mode 100644 index 00000000..905580ee --- /dev/null +++ b/tests/extras/demoapp/dnn_files/deploy.prototxt.txt @@ -0,0 +1,1789 @@ +input: "data" +input_shape { + dim: 1 + dim: 3 + dim: 300 + dim: 300 +} + +layer { + name: "data_bn" + type: "BatchNorm" + bottom: "data" + top: "data_bn" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "data_scale" + type: "Scale" + bottom: "data_bn" + top: "data_bn" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "conv1_h" + type: "Convolution" + bottom: "data_bn" + top: "conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 32 + pad: 3 + kernel_size: 7 + stride: 2 + weight_filler { + type: "msra" + variance_norm: FAN_OUT + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "conv1_bn_h" + type: "BatchNorm" + bottom: "conv1_h" + top: "conv1_h" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "conv1_scale_h" + type: "Scale" + bottom: "conv1_h" + top: "conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "conv1_relu" + type: "ReLU" + bottom: "conv1_h" + top: "conv1_h" +} +layer { + name: "conv1_pool" + type: "Pooling" + bottom: "conv1_h" + top: "conv1_pool" + pooling_param { + kernel_size: 3 + stride: 2 + } +} +layer { + name: "layer_64_1_conv1_h" + type: "Convolution" + bottom: "conv1_pool" + top: "layer_64_1_conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 32 + bias_term: false + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_64_1_bn2_h" + type: "BatchNorm" + bottom: "layer_64_1_conv1_h" + top: "layer_64_1_conv1_h" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "layer_64_1_scale2_h" + type: "Scale" + bottom: "layer_64_1_conv1_h" + top: "layer_64_1_conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "layer_64_1_relu2" + type: "ReLU" + bottom: "layer_64_1_conv1_h" + top: "layer_64_1_conv1_h" +} +layer { + name: "layer_64_1_conv2_h" + type: "Convolution" + bottom: "layer_64_1_conv1_h" + top: "layer_64_1_conv2_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 32 + bias_term: false + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_64_1_sum" + type: "Eltwise" + bottom: "layer_64_1_conv2_h" + bottom: "conv1_pool" + top: "layer_64_1_sum" +} +layer { + name: "layer_128_1_bn1_h" + type: "BatchNorm" + bottom: "layer_64_1_sum" + top: "layer_128_1_bn1_h" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "layer_128_1_scale1_h" + type: "Scale" + bottom: "layer_128_1_bn1_h" + top: "layer_128_1_bn1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "layer_128_1_relu1" + type: "ReLU" + bottom: "layer_128_1_bn1_h" + top: "layer_128_1_bn1_h" +} +layer { + name: "layer_128_1_conv1_h" + type: "Convolution" + bottom: "layer_128_1_bn1_h" + top: "layer_128_1_conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 128 + bias_term: false + pad: 1 + kernel_size: 3 + stride: 2 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_128_1_bn2" + type: "BatchNorm" + bottom: "layer_128_1_conv1_h" + top: "layer_128_1_conv1_h" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "layer_128_1_scale2" + type: "Scale" + bottom: "layer_128_1_conv1_h" + top: "layer_128_1_conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "layer_128_1_relu2" + type: "ReLU" + bottom: "layer_128_1_conv1_h" + top: "layer_128_1_conv1_h" +} +layer { + name: "layer_128_1_conv2" + type: "Convolution" + bottom: "layer_128_1_conv1_h" + top: "layer_128_1_conv2" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 128 + bias_term: false + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_128_1_conv_expand_h" + type: "Convolution" + bottom: "layer_128_1_bn1_h" + top: "layer_128_1_conv_expand_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 128 + bias_term: false + pad: 0 + kernel_size: 1 + stride: 2 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_128_1_sum" + type: "Eltwise" + bottom: "layer_128_1_conv2" + bottom: "layer_128_1_conv_expand_h" + top: "layer_128_1_sum" +} +layer { + name: "layer_256_1_bn1" + type: "BatchNorm" + bottom: "layer_128_1_sum" + top: "layer_256_1_bn1" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "layer_256_1_scale1" + type: "Scale" + bottom: "layer_256_1_bn1" + top: "layer_256_1_bn1" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "layer_256_1_relu1" + type: "ReLU" + bottom: "layer_256_1_bn1" + top: "layer_256_1_bn1" +} +layer { + name: "layer_256_1_conv1" + type: "Convolution" + bottom: "layer_256_1_bn1" + top: "layer_256_1_conv1" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 256 + bias_term: false + pad: 1 + kernel_size: 3 + stride: 2 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_256_1_bn2" + type: "BatchNorm" + bottom: "layer_256_1_conv1" + top: "layer_256_1_conv1" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "layer_256_1_scale2" + type: "Scale" + bottom: "layer_256_1_conv1" + top: "layer_256_1_conv1" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "layer_256_1_relu2" + type: "ReLU" + bottom: "layer_256_1_conv1" + top: "layer_256_1_conv1" +} +layer { + name: "layer_256_1_conv2" + type: "Convolution" + bottom: "layer_256_1_conv1" + top: "layer_256_1_conv2" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 256 + bias_term: false + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_256_1_conv_expand" + type: "Convolution" + bottom: "layer_256_1_bn1" + top: "layer_256_1_conv_expand" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 256 + bias_term: false + pad: 0 + kernel_size: 1 + stride: 2 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_256_1_sum" + type: "Eltwise" + bottom: "layer_256_1_conv2" + bottom: "layer_256_1_conv_expand" + top: "layer_256_1_sum" +} +layer { + name: "layer_512_1_bn1" + type: "BatchNorm" + bottom: "layer_256_1_sum" + top: "layer_512_1_bn1" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "layer_512_1_scale1" + type: "Scale" + bottom: "layer_512_1_bn1" + top: "layer_512_1_bn1" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "layer_512_1_relu1" + type: "ReLU" + bottom: "layer_512_1_bn1" + top: "layer_512_1_bn1" +} +layer { + name: "layer_512_1_conv1_h" + type: "Convolution" + bottom: "layer_512_1_bn1" + top: "layer_512_1_conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 128 + bias_term: false + pad: 1 + kernel_size: 3 + stride: 1 # 2 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_512_1_bn2_h" + type: "BatchNorm" + bottom: "layer_512_1_conv1_h" + top: "layer_512_1_conv1_h" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "layer_512_1_scale2_h" + type: "Scale" + bottom: "layer_512_1_conv1_h" + top: "layer_512_1_conv1_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "layer_512_1_relu2" + type: "ReLU" + bottom: "layer_512_1_conv1_h" + top: "layer_512_1_conv1_h" +} +layer { + name: "layer_512_1_conv2_h" + type: "Convolution" + bottom: "layer_512_1_conv1_h" + top: "layer_512_1_conv2_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 256 + bias_term: false + pad: 2 # 1 + kernel_size: 3 + stride: 1 + dilation: 2 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_512_1_conv_expand_h" + type: "Convolution" + bottom: "layer_512_1_bn1" + top: "layer_512_1_conv_expand_h" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + convolution_param { + num_output: 256 + bias_term: false + pad: 0 + kernel_size: 1 + stride: 1 # 2 + weight_filler { + type: "msra" + } + bias_filler { + type: "constant" + value: 0.0 + } + } +} +layer { + name: "layer_512_1_sum" + type: "Eltwise" + bottom: "layer_512_1_conv2_h" + bottom: "layer_512_1_conv_expand_h" + top: "layer_512_1_sum" +} +layer { + name: "last_bn_h" + type: "BatchNorm" + bottom: "layer_512_1_sum" + top: "layer_512_1_sum" + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } + param { + lr_mult: 0.0 + } +} +layer { + name: "last_scale_h" + type: "Scale" + bottom: "layer_512_1_sum" + top: "layer_512_1_sum" + param { + lr_mult: 1.0 + decay_mult: 1.0 + } + param { + lr_mult: 2.0 + decay_mult: 1.0 + } + scale_param { + bias_term: true + } +} +layer { + name: "last_relu" + type: "ReLU" + bottom: "layer_512_1_sum" + top: "fc7" +} + +layer { + name: "conv6_1_h" + type: "Convolution" + bottom: "fc7" + top: "conv6_1_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 0 + kernel_size: 1 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv6_1_relu" + type: "ReLU" + bottom: "conv6_1_h" + top: "conv6_1_h" +} +layer { + name: "conv6_2_h" + type: "Convolution" + bottom: "conv6_1_h" + top: "conv6_2_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 256 + pad: 1 + kernel_size: 3 + stride: 2 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv6_2_relu" + type: "ReLU" + bottom: "conv6_2_h" + top: "conv6_2_h" +} +layer { + name: "conv7_1_h" + type: "Convolution" + bottom: "conv6_2_h" + top: "conv7_1_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 64 + pad: 0 + kernel_size: 1 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv7_1_relu" + type: "ReLU" + bottom: "conv7_1_h" + top: "conv7_1_h" +} +layer { + name: "conv7_2_h" + type: "Convolution" + bottom: "conv7_1_h" + top: "conv7_2_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 1 + kernel_size: 3 + stride: 2 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv7_2_relu" + type: "ReLU" + bottom: "conv7_2_h" + top: "conv7_2_h" +} +layer { + name: "conv8_1_h" + type: "Convolution" + bottom: "conv7_2_h" + top: "conv8_1_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 64 + pad: 0 + kernel_size: 1 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv8_1_relu" + type: "ReLU" + bottom: "conv8_1_h" + top: "conv8_1_h" +} +layer { + name: "conv8_2_h" + type: "Convolution" + bottom: "conv8_1_h" + top: "conv8_2_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv8_2_relu" + type: "ReLU" + bottom: "conv8_2_h" + top: "conv8_2_h" +} +layer { + name: "conv9_1_h" + type: "Convolution" + bottom: "conv8_2_h" + top: "conv9_1_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 64 + pad: 0 + kernel_size: 1 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv9_1_relu" + type: "ReLU" + bottom: "conv9_1_h" + top: "conv9_1_h" +} +layer { + name: "conv9_2_h" + type: "Convolution" + bottom: "conv9_1_h" + top: "conv9_2_h" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv9_2_relu" + type: "ReLU" + bottom: "conv9_2_h" + top: "conv9_2_h" +} +layer { + name: "conv4_3_norm" + type: "Normalize" + bottom: "layer_256_1_bn1" + top: "conv4_3_norm" + norm_param { + across_spatial: false + scale_filler { + type: "constant" + value: 20 + } + channel_shared: false + } +} +layer { + name: "conv4_3_norm_mbox_loc" + type: "Convolution" + bottom: "conv4_3_norm" + top: "conv4_3_norm_mbox_loc" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 16 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv4_3_norm_mbox_loc_perm" + type: "Permute" + bottom: "conv4_3_norm_mbox_loc" + top: "conv4_3_norm_mbox_loc_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv4_3_norm_mbox_loc_flat" + type: "Flatten" + bottom: "conv4_3_norm_mbox_loc_perm" + top: "conv4_3_norm_mbox_loc_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv4_3_norm_mbox_conf" + type: "Convolution" + bottom: "conv4_3_norm" + top: "conv4_3_norm_mbox_conf" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 8 # 84 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv4_3_norm_mbox_conf_perm" + type: "Permute" + bottom: "conv4_3_norm_mbox_conf" + top: "conv4_3_norm_mbox_conf_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv4_3_norm_mbox_conf_flat" + type: "Flatten" + bottom: "conv4_3_norm_mbox_conf_perm" + top: "conv4_3_norm_mbox_conf_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv4_3_norm_mbox_priorbox" + type: "PriorBox" + bottom: "conv4_3_norm" + bottom: "data" + top: "conv4_3_norm_mbox_priorbox" + prior_box_param { + min_size: 30.0 + max_size: 60.0 + aspect_ratio: 2 + flip: true + clip: false + variance: 0.1 + variance: 0.1 + variance: 0.2 + variance: 0.2 + step: 8 + offset: 0.5 + } +} +layer { + name: "fc7_mbox_loc" + type: "Convolution" + bottom: "fc7" + top: "fc7_mbox_loc" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 24 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "fc7_mbox_loc_perm" + type: "Permute" + bottom: "fc7_mbox_loc" + top: "fc7_mbox_loc_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "fc7_mbox_loc_flat" + type: "Flatten" + bottom: "fc7_mbox_loc_perm" + top: "fc7_mbox_loc_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "fc7_mbox_conf" + type: "Convolution" + bottom: "fc7" + top: "fc7_mbox_conf" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 12 # 126 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "fc7_mbox_conf_perm" + type: "Permute" + bottom: "fc7_mbox_conf" + top: "fc7_mbox_conf_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "fc7_mbox_conf_flat" + type: "Flatten" + bottom: "fc7_mbox_conf_perm" + top: "fc7_mbox_conf_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "fc7_mbox_priorbox" + type: "PriorBox" + bottom: "fc7" + bottom: "data" + top: "fc7_mbox_priorbox" + prior_box_param { + min_size: 60.0 + max_size: 111.0 + aspect_ratio: 2 + aspect_ratio: 3 + flip: true + clip: false + variance: 0.1 + variance: 0.1 + variance: 0.2 + variance: 0.2 + step: 16 + offset: 0.5 + } +} +layer { + name: "conv6_2_mbox_loc" + type: "Convolution" + bottom: "conv6_2_h" + top: "conv6_2_mbox_loc" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 24 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv6_2_mbox_loc_perm" + type: "Permute" + bottom: "conv6_2_mbox_loc" + top: "conv6_2_mbox_loc_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv6_2_mbox_loc_flat" + type: "Flatten" + bottom: "conv6_2_mbox_loc_perm" + top: "conv6_2_mbox_loc_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv6_2_mbox_conf" + type: "Convolution" + bottom: "conv6_2_h" + top: "conv6_2_mbox_conf" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 12 # 126 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv6_2_mbox_conf_perm" + type: "Permute" + bottom: "conv6_2_mbox_conf" + top: "conv6_2_mbox_conf_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv6_2_mbox_conf_flat" + type: "Flatten" + bottom: "conv6_2_mbox_conf_perm" + top: "conv6_2_mbox_conf_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv6_2_mbox_priorbox" + type: "PriorBox" + bottom: "conv6_2_h" + bottom: "data" + top: "conv6_2_mbox_priorbox" + prior_box_param { + min_size: 111.0 + max_size: 162.0 + aspect_ratio: 2 + aspect_ratio: 3 + flip: true + clip: false + variance: 0.1 + variance: 0.1 + variance: 0.2 + variance: 0.2 + step: 32 + offset: 0.5 + } +} +layer { + name: "conv7_2_mbox_loc" + type: "Convolution" + bottom: "conv7_2_h" + top: "conv7_2_mbox_loc" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 24 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv7_2_mbox_loc_perm" + type: "Permute" + bottom: "conv7_2_mbox_loc" + top: "conv7_2_mbox_loc_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv7_2_mbox_loc_flat" + type: "Flatten" + bottom: "conv7_2_mbox_loc_perm" + top: "conv7_2_mbox_loc_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv7_2_mbox_conf" + type: "Convolution" + bottom: "conv7_2_h" + top: "conv7_2_mbox_conf" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 12 # 126 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv7_2_mbox_conf_perm" + type: "Permute" + bottom: "conv7_2_mbox_conf" + top: "conv7_2_mbox_conf_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv7_2_mbox_conf_flat" + type: "Flatten" + bottom: "conv7_2_mbox_conf_perm" + top: "conv7_2_mbox_conf_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv7_2_mbox_priorbox" + type: "PriorBox" + bottom: "conv7_2_h" + bottom: "data" + top: "conv7_2_mbox_priorbox" + prior_box_param { + min_size: 162.0 + max_size: 213.0 + aspect_ratio: 2 + aspect_ratio: 3 + flip: true + clip: false + variance: 0.1 + variance: 0.1 + variance: 0.2 + variance: 0.2 + step: 64 + offset: 0.5 + } +} +layer { + name: "conv8_2_mbox_loc" + type: "Convolution" + bottom: "conv8_2_h" + top: "conv8_2_mbox_loc" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 16 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv8_2_mbox_loc_perm" + type: "Permute" + bottom: "conv8_2_mbox_loc" + top: "conv8_2_mbox_loc_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv8_2_mbox_loc_flat" + type: "Flatten" + bottom: "conv8_2_mbox_loc_perm" + top: "conv8_2_mbox_loc_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv8_2_mbox_conf" + type: "Convolution" + bottom: "conv8_2_h" + top: "conv8_2_mbox_conf" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 8 # 84 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv8_2_mbox_conf_perm" + type: "Permute" + bottom: "conv8_2_mbox_conf" + top: "conv8_2_mbox_conf_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv8_2_mbox_conf_flat" + type: "Flatten" + bottom: "conv8_2_mbox_conf_perm" + top: "conv8_2_mbox_conf_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv8_2_mbox_priorbox" + type: "PriorBox" + bottom: "conv8_2_h" + bottom: "data" + top: "conv8_2_mbox_priorbox" + prior_box_param { + min_size: 213.0 + max_size: 264.0 + aspect_ratio: 2 + flip: true + clip: false + variance: 0.1 + variance: 0.1 + variance: 0.2 + variance: 0.2 + step: 100 + offset: 0.5 + } +} +layer { + name: "conv9_2_mbox_loc" + type: "Convolution" + bottom: "conv9_2_h" + top: "conv9_2_mbox_loc" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 16 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv9_2_mbox_loc_perm" + type: "Permute" + bottom: "conv9_2_mbox_loc" + top: "conv9_2_mbox_loc_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv9_2_mbox_loc_flat" + type: "Flatten" + bottom: "conv9_2_mbox_loc_perm" + top: "conv9_2_mbox_loc_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv9_2_mbox_conf" + type: "Convolution" + bottom: "conv9_2_h" + top: "conv9_2_mbox_conf" + param { + lr_mult: 1 + decay_mult: 1 + } + param { + lr_mult: 2 + decay_mult: 0 + } + convolution_param { + num_output: 8 # 84 + pad: 1 + kernel_size: 3 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layer { + name: "conv9_2_mbox_conf_perm" + type: "Permute" + bottom: "conv9_2_mbox_conf" + top: "conv9_2_mbox_conf_perm" + permute_param { + order: 0 + order: 2 + order: 3 + order: 1 + } +} +layer { + name: "conv9_2_mbox_conf_flat" + type: "Flatten" + bottom: "conv9_2_mbox_conf_perm" + top: "conv9_2_mbox_conf_flat" + flatten_param { + axis: 1 + } +} +layer { + name: "conv9_2_mbox_priorbox" + type: "PriorBox" + bottom: "conv9_2_h" + bottom: "data" + top: "conv9_2_mbox_priorbox" + prior_box_param { + min_size: 264.0 + max_size: 315.0 + aspect_ratio: 2 + flip: true + clip: false + variance: 0.1 + variance: 0.1 + variance: 0.2 + variance: 0.2 + step: 300 + offset: 0.5 + } +} +layer { + name: "mbox_loc" + type: "Concat" + bottom: "conv4_3_norm_mbox_loc_flat" + bottom: "fc7_mbox_loc_flat" + bottom: "conv6_2_mbox_loc_flat" + bottom: "conv7_2_mbox_loc_flat" + bottom: "conv8_2_mbox_loc_flat" + bottom: "conv9_2_mbox_loc_flat" + top: "mbox_loc" + concat_param { + axis: 1 + } +} +layer { + name: "mbox_conf" + type: "Concat" + bottom: "conv4_3_norm_mbox_conf_flat" + bottom: "fc7_mbox_conf_flat" + bottom: "conv6_2_mbox_conf_flat" + bottom: "conv7_2_mbox_conf_flat" + bottom: "conv8_2_mbox_conf_flat" + bottom: "conv9_2_mbox_conf_flat" + top: "mbox_conf" + concat_param { + axis: 1 + } +} +layer { + name: "mbox_priorbox" + type: "Concat" + bottom: "conv4_3_norm_mbox_priorbox" + bottom: "fc7_mbox_priorbox" + bottom: "conv6_2_mbox_priorbox" + bottom: "conv7_2_mbox_priorbox" + bottom: "conv8_2_mbox_priorbox" + bottom: "conv9_2_mbox_priorbox" + top: "mbox_priorbox" + concat_param { + axis: 2 + } +} + +layer { + name: "mbox_conf_reshape" + type: "Reshape" + bottom: "mbox_conf" + top: "mbox_conf_reshape" + reshape_param { + shape { + dim: 0 + dim: -1 + dim: 2 + } + } +} +layer { + name: "mbox_conf_softmax" + type: "Softmax" + bottom: "mbox_conf_reshape" + top: "mbox_conf_softmax" + softmax_param { + axis: 2 + } +} +layer { + name: "mbox_conf_flatten" + type: "Flatten" + bottom: "mbox_conf_softmax" + top: "mbox_conf_flatten" + flatten_param { + axis: 1 + } +} + +layer { + name: "detection_out" + type: "DetectionOutput" + bottom: "mbox_loc" + bottom: "mbox_conf_flatten" + bottom: "mbox_priorbox" + top: "detection_out" + include { + phase: TEST + } + detection_output_param { + num_classes: 2 + share_location: true + background_label_id: 0 + nms_param { + nms_threshold: 0.45 + top_k: 400 + } + code_type: CENTER_SIZE + keep_top_k: 200 + confidence_threshold: 0.01 + } +} diff --git a/tests/extras/demoapp/dnn_files/res10_300x300_ssd_iter_140000.caffemodel b/tests/extras/demoapp/dnn_files/res10_300x300_ssd_iter_140000.caffemodel new file mode 100644 index 00000000..809dfd78 Binary files /dev/null and b/tests/extras/demoapp/dnn_files/res10_300x300_ssd_iter_140000.caffemodel differ diff --git a/tests/extras/demoapp/scripts/.common b/tests/extras/demoapp/scripts/.common new file mode 100755 index 00000000..0c431ef5 --- /dev/null +++ b/tests/extras/demoapp/scripts/.common @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.vars" + +call_api() { + http "$1" "$BASE_URL/$2/" Authorization:"Token $AUTH_TOKEN" "${@:3}" +} + +set_variable() { + sed -i "s/$1=.*$/$1=$2/" .vars +} + +show_required_args() { + for var in "$@" + do + echo "$var is required" + done +} diff --git a/tests/extras/demoapp/scripts/.vars b/tests/extras/demoapp/scripts/.vars new file mode 100644 index 00000000..58dda3e1 --- /dev/null +++ b/tests/extras/demoapp/scripts/.vars @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +BASE_URL= +AUTH_TOKEN= +DEDUPLICATION_SET_ID= diff --git a/tests/extras/demoapp/scripts/README.md b/tests/extras/demoapp/scripts/README.md new file mode 100644 index 00000000..fd4e439e --- /dev/null +++ b/tests/extras/demoapp/scripts/README.md @@ -0,0 +1,44 @@ +# Scripts for API interaction + +## Requirements + +These scripts use `httpie` and `jq`, so make sure they are installed. + +## Scripts + +### Configuration + +#### Internal + +These scripts hold configuration and common functionality + +| Name | Arguments | Description | +|-----------------------|-----------|-------------------------------------------------| +| .vars | - | Contains configuration variables | +| .common | - | Contains common functions used by other scripts | + +#### Public + +| Name | Arguments | Description | +|-----------------------|----------------------|---------------------------| +| use_base_url | base url | Sets base url | +| use_auth_token | auth token | Sets authentication token | +| use_deduplication_set | deduplication set id | Sets deduplication set id | + +### API interaction + +| Name | Arguments | Description | +|---------------------------|-----------------------------------------|---------------------------------------------| +| create_deduplication_set | reference_pk | Creates new deduplication set | +| create_image | filename | Creates image in deduplication set | +| ignore | first reference pk, second reference pk | Makes API ignore specific reference pk pair | +| process_deduplication_set | - | Starts deduplication process | +| show_deduplication_set | - | Shows deduplication set data | +| show_duplicates | - | Shows duplicates found in deduplication set | + +### Test cases + +| Name | Arguments | Description | +|------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------| +| base_case | reference pk | Creates deduplication set, adds images to it and runs deduplication process | +| all_ignored_case | reference pk | Creates deduplication set, adds images to it, adds all possible reference pk pairs to ignored pairs and shows duplicates found | diff --git a/tests/extras/demoapp/scripts/all_ignored_case b/tests/extras/demoapp/scripts/all_ignored_case new file mode 100755 index 00000000..28c90d06 --- /dev/null +++ b/tests/extras/demoapp/scripts/all_ignored_case @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +if [[ $# -ne 1 ]] ; then + show_required_args "deduplication set reference pk" + exit 1 +fi + +./create_deduplication_set "$1" | jq -r ".id" | xargs ./use_deduplication_set + +./create_image Aaron_Eckhart_0001.jpg +./create_image Aaron_Guiel_0001.jpg +./create_image Aaron_Peirsol_0001.jpg +./create_image Aaron_Peirsol_0002.jpg +./create_image Cathy_Freeman_0001.jpg +./create_image Cathy_Freeman_0002.jpg +./create_image without_face.jpg +./create_image Ziwang_Xu_0001.jpg +./create_image Zoe_Ball_0001.jpg + +./ignore Aaron_Eckhart_0001.jpg Aaron_Guiel_0001.jpg +./ignore Aaron_Eckhart_0001.jpg Aaron_Peirsol_0001.jpg +./ignore Aaron_Eckhart_0001.jpg Aaron_Peirsol_0002.jpg +./ignore Aaron_Eckhart_0001.jpg Cathy_Freeman_0001.jpg +./ignore Aaron_Eckhart_0001.jpg Cathy_Freeman_0002.jpg +./ignore Aaron_Eckhart_0001.jpg without_face.jpg +./ignore Aaron_Eckhart_0001.jpg Ziwang_Xu_0001.jpg +./ignore Aaron_Eckhart_0001.jpg Zoe_Ball_0001.jpg +./ignore Aaron_Guiel_0001.jpg Aaron_Peirsol_0001.jpg +./ignore Aaron_Guiel_0001.jpg Aaron_Peirsol_0002.jpg +./ignore Aaron_Guiel_0001.jpg Cathy_Freeman_0001.jpg +./ignore Aaron_Guiel_0001.jpg Cathy_Freeman_0002.jpg +./ignore Aaron_Guiel_0001.jpg without_face.jpg +./ignore Aaron_Guiel_0001.jpg Ziwang_Xu_0001.jpg +./ignore Aaron_Guiel_0001.jpg Zoe_Ball_0001.jpg +./ignore Aaron_Peirsol_0001.jpg Aaron_Peirsol_0002.jpg +./ignore Aaron_Peirsol_0001.jpg Cathy_Freeman_0001.jpg +./ignore Aaron_Peirsol_0001.jpg Cathy_Freeman_0002.jpg +./ignore Aaron_Peirsol_0001.jpg without_face.jpg +./ignore Aaron_Peirsol_0001.jpg Ziwang_Xu_0001.jpg +./ignore Aaron_Peirsol_0001.jpg Zoe_Ball_0001.jpg +./ignore Aaron_Peirsol_0002.jpg Cathy_Freeman_0001.jpg +./ignore Aaron_Peirsol_0002.jpg Cathy_Freeman_0002.jpg +./ignore Aaron_Peirsol_0002.jpg without_face.jpg +./ignore Aaron_Peirsol_0002.jpg Ziwang_Xu_0001.jpg +./ignore Aaron_Peirsol_0002.jpg Zoe_Ball_0001.jpg +./ignore Cathy_Freeman_0001.jpg Cathy_Freeman_0002.jpg +./ignore Cathy_Freeman_0001.jpg without_face.jpg +./ignore Cathy_Freeman_0001.jpg Ziwang_Xu_0001.jpg +./ignore Cathy_Freeman_0001.jpg Zoe_Ball_0001.jpg +./ignore Cathy_Freeman_0002.jpg without_face.jpg +./ignore Cathy_Freeman_0002.jpg Ziwang_Xu_0001.jpg +./ignore Cathy_Freeman_0002.jpg Zoe_Ball_0001.jpg +./ignore without_face.jpg Ziwang_Xu_0001.jpg +./ignore without_face.jpg Zoe_Ball_0001.jpg +./ignore Ziwang_Xu_0001.jpg Zoe_Ball_0001.jpg + +./process_deduplication_set + +./show_duplicates diff --git a/tests/extras/demoapp/scripts/base_case b/tests/extras/demoapp/scripts/base_case new file mode 100755 index 00000000..bcce8600 --- /dev/null +++ b/tests/extras/demoapp/scripts/base_case @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +if [[ $# -ne 1 ]] ; then + show_required_args "deduplication set reference pk" + exit 1 +fi + +./create_deduplication_set "$1" | jq -r ".id" | xargs ./use_deduplication_set + +./create_image Aaron_Eckhart_0001.jpg +./create_image Aaron_Guiel_0001.jpg +./create_image Aaron_Peirsol_0001.jpg +./create_image Aaron_Peirsol_0002.jpg +./create_image Cathy_Freeman_0001.jpg +./create_image Cathy_Freeman_0002.jpg +./create_image without_face.jpg +./create_image Ziwang_Xu_0001.jpg +./create_image Zoe_Ball_0001.jpg + +./process_deduplication_set + +./show_duplicates diff --git a/tests/extras/demoapp/scripts/create_deduplication_set b/tests/extras/demoapp/scripts/create_deduplication_set new file mode 100755 index 00000000..4e9b6bcc --- /dev/null +++ b/tests/extras/demoapp/scripts/create_deduplication_set @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Create deduplication set >&2 + +if [[ $# -ne 1 ]] ; then + show_required_args "reference pk" + exit 1 +fi + +call_api POST deduplication_sets reference_pk="$1" diff --git a/tests/extras/demoapp/scripts/create_image b/tests/extras/demoapp/scripts/create_image new file mode 100755 index 00000000..eaf79827 --- /dev/null +++ b/tests/extras/demoapp/scripts/create_image @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Create image >&2 + +# We use image name provided as both reference pk and filename + +if [[ $# -ne 1 ]] ; then + show_required_args filename + exit 1 +fi + +call_api POST "deduplication_sets/$DEDUPLICATION_SET_ID/images" reference_pk="$1" filename="$1" diff --git a/tests/extras/demoapp/scripts/ignore b/tests/extras/demoapp/scripts/ignore new file mode 100755 index 00000000..4d880bb5 --- /dev/null +++ b/tests/extras/demoapp/scripts/ignore @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Ignore key pair >&2 + +if [[ $# -ne 2 ]] ; then + show_required_args "first reference pk" "second reference pk" + exit 1 +fi + +call_api POST "deduplication_sets/$DEDUPLICATION_SET_ID/ignored_keys" first_reference_pk="$1" second_reference_pk="$2" diff --git a/tests/extras/demoapp/scripts/process_deduplication_set b/tests/extras/demoapp/scripts/process_deduplication_set new file mode 100755 index 00000000..a7f66ef9 --- /dev/null +++ b/tests/extras/demoapp/scripts/process_deduplication_set @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Process deduplication set >&2 + +call_api POST "deduplication_sets/$DEDUPLICATION_SET_ID/process" + +./show_deduplication_set + +until [ "$(./show_deduplication_set | jq -r ".state")" != "Processing" ] +do + sleep 0.5 +done + +./show_deduplication_set diff --git a/tests/extras/demoapp/scripts/show_deduplication_set b/tests/extras/demoapp/scripts/show_deduplication_set new file mode 100755 index 00000000..324d0d0c --- /dev/null +++ b/tests/extras/demoapp/scripts/show_deduplication_set @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Show deduplication set >&2 + +call_api GET "deduplication_sets/$DEDUPLICATION_SET_ID" diff --git a/tests/extras/demoapp/scripts/show_duplicates b/tests/extras/demoapp/scripts/show_duplicates new file mode 100755 index 00000000..04918ae2 --- /dev/null +++ b/tests/extras/demoapp/scripts/show_duplicates @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Show dupicates >&2 + +call_api GET "deduplication_sets/$DEDUPLICATION_SET_ID/duplicates" diff --git a/tests/extras/demoapp/scripts/use_auth_token b/tests/extras/demoapp/scripts/use_auth_token new file mode 100755 index 00000000..208e5cf9 --- /dev/null +++ b/tests/extras/demoapp/scripts/use_auth_token @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Set auth token >&2 + +if [[ $# -ne 1 ]] ; then + show_required_args "auth token" + exit 1 +fi + +set_variable AUTH_TOKEN "$1" diff --git a/tests/extras/demoapp/scripts/use_base_url b/tests/extras/demoapp/scripts/use_base_url new file mode 100755 index 00000000..66500470 --- /dev/null +++ b/tests/extras/demoapp/scripts/use_base_url @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Set base URL >&2 + +if [[ $# -ne 1 ]] ; then + show_required_args "base url" + exit 1 +fi + +set_variable BASE_URL "$1" diff --git a/tests/extras/demoapp/scripts/use_deduplication_set b/tests/extras/demoapp/scripts/use_deduplication_set new file mode 100755 index 00000000..40d03d93 --- /dev/null +++ b/tests/extras/demoapp/scripts/use_deduplication_set @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "0")/.common" + +echo Set deduplication set id >&2 + +if [[ $# -ne 1 ]] ; then + show_required_args "deduplication set id" + exit 1 +fi + +set_variable DEDUPLICATION_SET_ID "$1" diff --git a/tests/extras/testutils/decorators.py b/tests/extras/testutils/decorators.py index a9d6216c..2755f1b9 100644 --- a/tests/extras/testutils/decorators.py +++ b/tests/extras/testutils/decorators.py @@ -10,4 +10,6 @@ def requires_env(*envs): if os.environ.get(env, None) is None: missing.append(env) - return pytest.mark.skipif(len(missing) > 0, reason=f"Not suitable environment {missing} for current test") + return pytest.mark.skipif( + len(missing) > 0, reason=f"Not suitable environment {missing} for current test" + ) diff --git a/tests/extras/testutils/duplicate_finders.py b/tests/extras/testutils/duplicate_finders.py new file mode 100644 index 00000000..605ab7aa --- /dev/null +++ b/tests/extras/testutils/duplicate_finders.py @@ -0,0 +1,35 @@ +from collections.abc import Generator +from itertools import combinations + +from hope_dedup_engine.apps.api.deduplication.registry import DuplicateKeyPair +from hope_dedup_engine.apps.api.models import DeduplicationSet + + +class AllDuplicateFinder: + weight = 1 + + def __init__(self, deduplication_set: DeduplicationSet) -> None: + self.deduplication_set = deduplication_set + + def run(self) -> Generator[DuplicateKeyPair, None, None]: + reference_pks = self.deduplication_set.image_set.values_list( + "reference_pk", flat=True + ).order_by("reference_pk") + for first, second in combinations(reference_pks, 2): + yield first, second, 1.0 + + +class NoDuplicateFinder: + weight = 1 + + def run(self) -> Generator[DuplicateKeyPair, None, None]: + # empty generator + return + yield + + +class FailingDuplicateFinder: + weight = 1 + + def run(self) -> Generator[DuplicateKeyPair, None, None]: + raise Exception diff --git a/tests/extras/testutils/factories/__init__.py b/tests/extras/testutils/factories/__init__.py index 2a687069..93130e16 100644 --- a/tests/extras/testutils/factories/__init__.py +++ b/tests/extras/testutils/factories/__init__.py @@ -5,20 +5,34 @@ from factory.django import DjangoModelFactory from pytest_factoryboy import register -from .base import AutoRegisterModelFactory, TAutoRegisterModelFactory, factories_registry +from .base import ( + AutoRegisterModelFactory, + TAutoRegisterModelFactory, + factories_registry, +) from .django_celery_beat import PeriodicTaskFactory # noqa from .social import SocialAuthUserFactory # noqa -from .user import ExternalSystemFactory, GroupFactory, SuperUserFactory, User, UserFactory # noqa +from .user import ( # noqa + ExternalSystemFactory, + GroupFactory, + SuperUserFactory, + User, + UserFactory, +) from .userrole import UserRole, UserRoleFactory # noqa for _, name, _ in pkgutil.iter_modules([str(Path(__file__).parent)]): importlib.import_module(f".{name}", __package__) -django_model_factories = {factory._meta.model: factory for factory in DjangoModelFactory.__subclasses__()} +django_model_factories = { + factory._meta.model: factory for factory in DjangoModelFactory.__subclasses__() +} -def get_factory_for_model(_model) -> type[TAutoRegisterModelFactory] | type[DjangoModelFactory]: +def get_factory_for_model( + _model, +) -> type[TAutoRegisterModelFactory] | type[DjangoModelFactory]: class Meta: model = _model @@ -29,4 +43,6 @@ class Meta: if _model in django_model_factories: return django_model_factories[_model] - return register(type(f"{_model._meta.model_name}AutoCreatedFactory", bases, {"Meta": Meta})) # noqa + return register( + type(f"{_model._meta.model_name}AutoCreatedFactory", bases, {"Meta": Meta}) + ) # noqa diff --git a/tests/extras/testutils/factories/api.py b/tests/extras/testutils/factories/api.py index 851e9c41..a30ef1e0 100644 --- a/tests/extras/testutils/factories/api.py +++ b/tests/extras/testutils/factories/api.py @@ -3,7 +3,13 @@ from testutils.factories import ExternalSystemFactory, UserFactory from hope_dedup_engine.apps.api.models import DeduplicationSet, HDEToken -from hope_dedup_engine.apps.api.models.deduplication import Duplicate, IgnoredKeyPair, Image +from hope_dedup_engine.apps.api.models.deduplication import ( + Config, + Duplicate, + IgnoredFilenamePair, + IgnoredReferencePkPair, + Image, +) class TokenFactory(DjangoModelFactory): @@ -13,12 +19,19 @@ class Meta: model = HDEToken +class ConfigFactory(DjangoModelFactory): + face_distance_threshold = fuzzy.FuzzyFloat(low=0.1, high=1.0) + + class Meta: + model = Config + + class DeduplicationSetFactory(DjangoModelFactory): - name = fuzzy.FuzzyText() reference_pk = fuzzy.FuzzyText() external_system = SubFactory(ExternalSystemFactory) state = DeduplicationSet.State.CLEAN - notification_url = fuzzy.FuzzyText() + notification_url = fuzzy.FuzzyText(prefix="https://") + config = SubFactory(ConfigFactory) class Meta: model = DeduplicationSet @@ -35,9 +48,7 @@ class Meta: class DuplicateFactory(DjangoModelFactory): deduplication_set = SubFactory(DeduplicationSetFactory) - first_filename = fuzzy.FuzzyText() first_reference_pk = fuzzy.FuzzyText() - second_filename = fuzzy.FuzzyText() second_reference_pk = fuzzy.FuzzyText() score = fuzzy.FuzzyFloat(low=0, high=1) @@ -45,10 +56,19 @@ class Meta: model = Duplicate -class IgnoredKeyPairFactory(DjangoModelFactory): +class IgnoredFilenamePairFactory(DjangoModelFactory): deduplication_set = SubFactory(DeduplicationSetFactory) - first_reference_pk = fuzzy.FuzzyText() - second_reference_pk = fuzzy.FuzzyText() + first = fuzzy.FuzzyText() + second = fuzzy.FuzzyText() + + class Meta: + model = IgnoredFilenamePair + + +class IgnoredReferencePkPairFactory(DjangoModelFactory): + deduplication_set = SubFactory(DeduplicationSetFactory) + first = fuzzy.FuzzyText() + second = fuzzy.FuzzyText() class Meta: - model = IgnoredKeyPair + model = IgnoredReferencePkPair diff --git a/tests/extras/testutils/factories/base.py b/tests/extras/testutils/factories/base.py index 31b5bb13..601ca5dd 100644 --- a/tests/extras/testutils/factories/base.py +++ b/tests/extras/testutils/factories/base.py @@ -3,7 +3,9 @@ import factory from factory.base import FactoryMetaClass -TAutoRegisterModelFactory = typing.TypeVar("TAutoRegisterModelFactory", bound="AutoRegisterModelFactory") +TAutoRegisterModelFactory = typing.TypeVar( + "TAutoRegisterModelFactory", bound="AutoRegisterModelFactory" +) factories_registry: dict[str, TAutoRegisterModelFactory] = {} @@ -15,5 +17,7 @@ def __new__(mcs, class_name, bases, attrs): return new_class -class AutoRegisterModelFactory(factory.django.DjangoModelFactory, metaclass=AutoRegisterFactoryMetaClass): +class AutoRegisterModelFactory( + factory.django.DjangoModelFactory, metaclass=AutoRegisterFactoryMetaClass +): pass diff --git a/tests/extras/testutils/factories/django_celery_beat.py b/tests/extras/testutils/factories/django_celery_beat.py index 30630356..c691ad40 100644 --- a/tests/extras/testutils/factories/django_celery_beat.py +++ b/tests/extras/testutils/factories/django_celery_beat.py @@ -1,7 +1,13 @@ from django.utils import timezone import factory -from django_celery_beat.models import SOLAR_SCHEDULES, ClockedSchedule, IntervalSchedule, PeriodicTask, SolarSchedule +from django_celery_beat.models import ( + SOLAR_SCHEDULES, + ClockedSchedule, + IntervalSchedule, + PeriodicTask, + SolarSchedule, +) from factory.fuzzy import FuzzyChoice from .base import AutoRegisterModelFactory diff --git a/tests/extras/testutils/factories/user.py b/tests/extras/testutils/factories/user.py index b2af0c3a..abc3f7b2 100644 --- a/tests/extras/testutils/factories/user.py +++ b/tests/extras/testutils/factories/user.py @@ -2,7 +2,7 @@ import factory.fuzzy -from hope_dedup_engine.apps.security.models import User, ExternalSystem +from hope_dedup_engine.apps.security.models import ExternalSystem, User from .base import AutoRegisterModelFactory diff --git a/tests/extras/testutils/perms.py b/tests/extras/testutils/perms.py index 49398c61..e7d64d5d 100644 --- a/tests/extras/testutils/perms.py +++ b/tests/extras/testutils/perms.py @@ -46,9 +46,13 @@ def get_group(name=None, permissions=None): except ValueError: raise ValueError(f"Invalid permission name {permission_name}") try: - permission = Permission.objects.get(content_type__app_label=app_label, codename=codename) + permission = Permission.objects.get( + content_type__app_label=app_label, codename=codename + ) except Permission.DoesNotExist: - raise Permission.DoesNotExist("Permission `{0}` does not exists", permission_name) + raise Permission.DoesNotExist( + "Permission `{0}` does not exists", permission_name + ) group.permissions.add(permission) return group diff --git a/tests/faces/conftest.py b/tests/faces/conftest.py index dba1f743..a2bb53e2 100644 --- a/tests/faces/conftest.py +++ b/tests/faces/conftest.py @@ -1,6 +1,10 @@ from io import BytesIO from unittest.mock import MagicMock, mock_open, patch +from django.contrib.auth import get_user_model +from django.core.files.storage import FileSystemStorage +from django.test import Client + import cv2 import numpy as np import pytest @@ -8,7 +12,9 @@ BLOB_SHAPE, DEPLOY_PROTO_CONTENT, DEPLOY_PROTO_SHAPE, + DNN_FILE, FACE_DETECTIONS, + FACE_DISTANCE_THRESHOLD, FACE_REGIONS_VALID, FILENAMES, IGNORE_PAIRS, @@ -18,31 +24,75 @@ from freezegun import freeze_time from PIL import Image from pytest_mock import MockerFixture +from storages.backends.azure_storage import AzureStorage from docker import from_env -from hope_dedup_engine.apps.core.storage import CV2DNNStorage, HDEAzureStorage, HOPEAzureStorage -from hope_dedup_engine.apps.faces.managers.net import DNNInferenceManager -from hope_dedup_engine.apps.faces.managers.storage import StorageManager -from hope_dedup_engine.apps.faces.services.duplication_detector import DuplicationDetector -from hope_dedup_engine.apps.faces.services.image_processor import BlobFromImageConfig, ImageProcessor +from hope_dedup_engine.apps.faces.managers import DNNInferenceManager, StorageManager +from hope_dedup_engine.apps.faces.managers.file_sync import ( + AzureFileDownloader, + GithubFileDownloader, +) +from hope_dedup_engine.apps.faces.services.duplication_detector import ( + DuplicationDetector, +) +from hope_dedup_engine.apps.faces.services.image_processor import ( + BlobFromImageConfig, + ImageProcessor, +) @pytest.fixture def mock_storage_manager(mocker: MockerFixture) -> StorageManager: - mocker.patch.object(CV2DNNStorage, "exists", return_value=True) - mocker.patch.object(HDEAzureStorage, "exists", return_value=True) - mocker.patch.object(HOPEAzureStorage, "exists", return_value=True) + mocker.patch.object(FileSystemStorage, "exists", return_value=True) + mocker.patch.object(AzureStorage, "exists", return_value=True) yield StorageManager() @pytest.fixture -def mock_hde_azure_storage(): - return MagicMock(spec=HDEAzureStorage) +def mock_encoded_azure_storage(mocker: MockerFixture): + return MagicMock(spec=AzureStorage) @pytest.fixture -def mock_hope_azure_storage(): - return MagicMock(spec=HOPEAzureStorage) +def mock_hope_azure_storage(mocker: MockerFixture): + return MagicMock(spec=AzureStorage) + + +@pytest.fixture +def mock_dnn_azure_storage(mocker: MockerFixture): + return MagicMock(spec=AzureStorage) + + +@pytest.fixture +def github_dnn_file_downloader(): + return GithubFileDownloader() + + +@pytest.fixture +def mock_requests_get(): + with patch("requests.get") as mock_get: + mock_response = mock_get.return_value.__enter__.return_value + mock_response.iter_content.return_value = DNN_FILE.get( + "content" + ) * DNN_FILE.get("chunks") + mock_response.raise_for_status = lambda: None + yield mock_get + + +@pytest.fixture +def azure_dnn_file_downloader(mocker): + downloader = AzureFileDownloader() + mocker.patch.object(downloader.remote_storage, "exists", return_value=True) + mock_remote_file = MagicMock() + mocker.patch.object( + downloader.remote_storage, "open", return_value=mock_remote_file + ) + return downloader + + +@pytest.fixture +def local_path(tmp_path): + return tmp_path / DNN_FILE.get("name") @pytest.fixture @@ -59,11 +109,20 @@ def mock_net_manager(mocker: MockerFixture) -> DNNInferenceManager: @pytest.fixture def mock_image_processor( - mocker: MockerFixture, mock_storage_manager, mock_net_manager, mock_open_context_manager + mocker: MockerFixture, + mock_storage_manager, + mock_net_manager, + mock_open_context_manager, ) -> ImageProcessor: - mocker.patch.object(BlobFromImageConfig, "_get_shape", return_value=DEPLOY_PROTO_SHAPE) - mock_processor = ImageProcessor() - mocker.patch.object(mock_processor.storages.get_storage("images"), "open", return_value=mock_open_context_manager) + mocker.patch.object( + BlobFromImageConfig, "_get_shape", return_value=DEPLOY_PROTO_SHAPE + ) + mock_processor = ImageProcessor(FACE_DISTANCE_THRESHOLD) + mocker.patch.object( + mock_processor.storages.get_storage("images"), + "open", + return_value=mock_open_context_manager, + ) yield mock_processor @@ -87,9 +146,13 @@ def mock_open_context_manager(image_bytes_io): @pytest.fixture def mock_net(): mock_net = MagicMock(spec=cv2.dnn_Net) # Mocking the neural network object - mock_detections = np.array([[FACE_DETECTIONS]], dtype=np.float32) # Mocking the detections array + mock_detections = np.array( + [[FACE_DETECTIONS]], dtype=np.float32 + ) # Mocking the detections array mock_expected_regions = FACE_REGIONS_VALID - mock_net.forward.return_value = mock_detections # Setting up the forward method of the mock network + mock_net.forward.return_value = ( + mock_detections # Setting up the forward method of the mock network + ) mock_imdecode = MagicMock(return_value=np.ones(IMAGE_SIZE, dtype=np.uint8)) mock_resize = MagicMock(return_value=np.ones(RESIZED_IMAGE_SIZE, dtype=np.uint8)) mock_blob = np.zeros(BLOB_SHAPE) @@ -98,7 +161,7 @@ def mock_net(): @pytest.fixture def mock_dd(mock_image_processor, mock_net_manager, mock_storage_manager): - detector = DuplicationDetector(FILENAMES, IGNORE_PAIRS) + detector = DuplicationDetector(FILENAMES, FACE_DISTANCE_THRESHOLD, IGNORE_PAIRS) yield detector @@ -111,7 +174,10 @@ def docker_client(): @pytest.fixture def mock_redis_client(): - with patch("redis.Redis.set") as mock_set, patch("redis.Redis.delete") as mock_delete: + with ( + patch("redis.Redis.set") as mock_set, + patch("redis.Redis.delete") as mock_delete, + ): yield mock_set, mock_delete @@ -120,7 +186,9 @@ def mock_dd_find(): with patch( "hope_dedup_engine.apps.faces.services.duplication_detector.DuplicationDetector.find_duplicates" ) as mock_find: - mock_find.return_value = (FILENAMES[:2],) # Assuming the first two are duplicates based on mock data + mock_find.return_value = [ + FILENAMES[:2], + ] # Assuming the first two are duplicates based on mock data yield mock_find @@ -128,3 +196,29 @@ def mock_dd_find(): def time_control(): with freeze_time("2024-01-01") as frozen_time: yield frozen_time + + +@pytest.fixture +def mock_file_sync_manager(): + with patch( + "hope_dedup_engine.apps.faces.celery_tasks.FileSyncManager" + ) as MockFileSyncManager: + mock_manager_instance = MockFileSyncManager.return_value + mock_downloader = MagicMock() + mock_manager_instance.downloader = mock_downloader + yield mock_manager_instance + + +@pytest.fixture +def admin_user(db): + User = get_user_model() + return User.objects.create_superuser( + username="admin", password="admin", email="admin@example.com" + ) + + +@pytest.fixture +def client(admin_user): + client = Client() + client.force_login(admin_user) + return client diff --git a/tests/faces/faces_const.py b/tests/faces/faces_const.py index 975b56c2..89ac631e 100644 --- a/tests/faces/faces_const.py +++ b/tests/faces/faces_const.py @@ -8,6 +8,24 @@ ["ignore_file.jpg", "ignore_file2.jpg"], ["ignore_file4.jpg", "ignore_file3.jpg"], ] +FACE_DISTANCE_THRESHOLD = 0.4 + +DNN_FILE = { + "name": FILENAME, + "content": [b"Hello, world!"], + "timeout": 3 * 60, + "chunks": 3, + "url": "https://raw.githubusercontent.com/sr6033/face-detection-with-OpenCV-and-DNN/master/deploy.prototxt.txt", +} + +DNN_FILES_TIMEOUT: Final[int] = (3 * 60,) +DNN_FILES_CHUNK_SIZE: Final[int] = (128 * 1024,) +DNN_FILES_BINARY_ITERABLE_FILE: Final[list[bytes]] = [b"Hello, world!"] * 3 +DNN_GITHUB_URL: Final[str] = ( + "https://raw.githubusercontent.com/sr6033/face-detection-with-OpenCV-and-DNN/master/deploy.prototxt.txt" +) +DNN_FILENAME: Final[str] = "deploy.prototxt" + CELERY_TASK_NAME: Final[str] = "Deduplicate" CELERY_TASK_TTL: Final[int] = 1 * 60 * 60 diff --git a/tests/faces/test_celery_tasks.py b/tests/faces/test_celery_tasks.py index e75fdb5b..5c926eee 100644 --- a/tests/faces/test_celery_tasks.py +++ b/tests/faces/test_celery_tasks.py @@ -1,26 +1,40 @@ from datetime import timedelta -from unittest.mock import patch +from unittest.mock import ANY, patch import pytest from celery import states from celery.exceptions import SoftTimeLimitExceeded, TimeLimitExceeded -from faces_const import CELERY_TASK_DELAYS, CELERY_TASK_NAME, CELERY_TASK_TTL, FILENAMES, IGNORE_PAIRS +from constance import test +from faces_const import ( + CELERY_TASK_DELAYS, + CELERY_TASK_NAME, + CELERY_TASK_TTL, + FILENAMES, + IGNORE_PAIRS, +) -from hope_dedup_engine.apps.faces.celery_tasks import deduplicate +from hope_dedup_engine.apps.faces.celery_tasks import deduplicate, sync_dnn_files from hope_dedup_engine.apps.faces.utils.celery_utils import _get_hash @pytest.mark.parametrize("lock_is_acquired", [True, False]) -def test_deduplicate_task_locking(mock_redis_client, mock_dd_find, mock_dd, lock_is_acquired): +def test_deduplicate_task_locking( + mock_redis_client, mock_dd_find, mock_dd, lock_is_acquired +): mock_set, mock_delete = mock_redis_client mock_set.return_value = lock_is_acquired mock_find = mock_dd_find - with patch("hope_dedup_engine.apps.faces.celery_tasks.DuplicationDetector", return_value=mock_dd): + with patch( + "hope_dedup_engine.apps.faces.celery_tasks.DuplicationDetector", + return_value=mock_dd, + ): task_result = deduplicate.apply(args=(FILENAMES, IGNORE_PAIRS)).get() hash_value = _get_hash(FILENAMES, IGNORE_PAIRS) - mock_set.assert_called_once_with(f"{CELERY_TASK_NAME}_{hash_value}", "true", nx=True, ex=CELERY_TASK_TTL) + mock_set.assert_called_once_with( + f"{CELERY_TASK_NAME}_{hash_value}", "true", nx=True, ex=CELERY_TASK_TTL + ) if lock_is_acquired: assert task_result == mock_find.return_value mock_find.assert_called_once() @@ -36,10 +50,15 @@ def test_deduplicate_task_locking(mock_redis_client, mock_dd_find, mock_dd, lock [ (CELERY_TASK_DELAYS["SoftTimeLimitExceeded"], SoftTimeLimitExceeded()), (CELERY_TASK_DELAYS["TimeLimitExceeded"], TimeLimitExceeded()), - (CELERY_TASK_DELAYS["CustomException"], Exception("Simulated custom task failure")), + ( + CELERY_TASK_DELAYS["CustomException"], + Exception("Simulated custom task failure"), + ), ], ) -def test_deduplicate_task_exception_handling(mock_redis_client, mock_dd_find, time_control, mock_dd, delay, exception): +def test_deduplicate_task_exception_handling( + mock_redis_client, mock_dd_find, time_control, mock_dd, delay, exception +): mock_set, mock_delete = mock_redis_client mock_find = mock_dd_find mock_find.side_effect = exception @@ -48,7 +67,10 @@ def test_deduplicate_task_exception_handling(mock_redis_client, mock_dd_find, ti with ( pytest.raises(type(exception)) as exc_info, - patch("hope_dedup_engine.apps.faces.celery_tasks.DuplicationDetector", return_value=mock_dd), + patch( + "hope_dedup_engine.apps.faces.celery_tasks.DuplicationDetector", + return_value=mock_dd, + ), ): task = deduplicate.apply(args=(FILENAMES, IGNORE_PAIRS)) assert exc_info.value == exception @@ -58,6 +80,42 @@ def test_deduplicate_task_exception_handling(mock_redis_client, mock_dd_find, ti assert task.traceback is not None hash_value = _get_hash(FILENAMES, IGNORE_PAIRS) - mock_set.assert_called_once_with(f"{CELERY_TASK_NAME}_{hash_value}", "true", nx=True, ex=3600) - mock_delete.assert_called_once_with(f"{CELERY_TASK_NAME}_{hash_value}") # Lock is released + mock_set.assert_called_once_with( + f"{CELERY_TASK_NAME}_{hash_value}", "true", nx=True, ex=3600 + ) + mock_delete.assert_called_once_with( + f"{CELERY_TASK_NAME}_{hash_value}" + ) # Lock is released mock_find.assert_called_once() + + +@pytest.mark.parametrize( + "force, source", + [ + (False, "github"), + (True, "github"), + (False, "azure"), + (True, "azure"), + ], +) +def test_sync_dnn_files_success(mock_file_sync_manager, force, source): + mock_file_sync_manager.downloader.sync.return_value = True + with test.pytest.override_config(DNN_FILES_SOURCE=source): + is_downloaded = sync_dnn_files(force=force) + assert is_downloaded is True + assert mock_file_sync_manager.downloader.sync.call_count == 2 + + +def test_sync_dnn_files_exception_handling(mock_file_sync_manager): + mock_file_sync_manager.downloader.sync.side_effect = Exception("Download error") + with ( + patch( + "hope_dedup_engine.apps.faces.celery_tasks.sync_dnn_files.update_state" + ) as mock_update_state, + pytest.raises(Exception), + ): + sync_dnn_files() + mock_update_state.assert_called_once_with( + state=states.FAILURE, + meta={"exc_message": "Download error", "traceback": ANY}, + ) diff --git a/tests/faces/test_downloader.py b/tests/faces/test_downloader.py new file mode 100644 index 00000000..aa135896 --- /dev/null +++ b/tests/faces/test_downloader.py @@ -0,0 +1,123 @@ +from unittest.mock import mock_open, patch + +import pytest +from faces_const import DNN_FILE +from requests.exceptions import RequestException + +from hope_dedup_engine.apps.faces.managers import FileSyncManager +from hope_dedup_engine.apps.faces.managers.file_sync import ( + AzureFileDownloader, + FileDownloader, + GithubFileDownloader, +) + + +def test_github_sync_success(github_dnn_file_downloader, mock_requests_get): + url = DNN_FILE["url"] + with patch("pathlib.Path.open", mock_open()) as mocked_file: + result = github_dnn_file_downloader.sync(DNN_FILE["name"], url) + + assert result is True + mock_requests_get.assert_called_once_with( + url, stream=True, timeout=DNN_FILE["timeout"] + ) + mocked_file.assert_called_once_with("wb") + assert mocked_file().write.call_count == DNN_FILE["chunks"] + + +def test_github_sync_raises_exception(github_dnn_file_downloader, mock_requests_get): + mock_requests_get.side_effect = RequestException("Failed to connect") + + with pytest.raises(RequestException): + github_dnn_file_downloader.sync(DNN_FILE["name"], DNN_FILE["url"]) + mock_requests_get.assert_called_once_with( + DNN_FILE["url"], stream=True, timeout=DNN_FILE["timeout"] + ) + + +def test_azure_sync_success(azure_dnn_file_downloader): + with ( + patch.object( + azure_dnn_file_downloader.remote_storage, + "listdir", + return_value=([], [DNN_FILE["name"]]), + ), + patch.object( + azure_dnn_file_downloader.remote_storage, "size", return_value=1024 + ), + patch("pathlib.Path.open", mock_open()) as mocked_file, + ): + result = azure_dnn_file_downloader.sync(DNN_FILE["name"], DNN_FILE["name"]) + + assert result is True + azure_dnn_file_downloader.remote_storage.listdir.assert_called_once_with("") + azure_dnn_file_downloader.remote_storage.open.assert_called_once_with( + DNN_FILE["name"], "rb" + ) + mocked_file.assert_called_once_with("wb") + + +def test_azure_sync_raises_filenotfounderror(azure_dnn_file_downloader): + with patch.object( + azure_dnn_file_downloader.remote_storage, "listdir", return_value=([], []) + ) as mock_listdir: + with pytest.raises(FileNotFoundError): + azure_dnn_file_downloader.sync(DNN_FILE["name"], DNN_FILE["name"]) + + mock_listdir.assert_called_once_with("") + azure_dnn_file_downloader.remote_storage.open.assert_not_called() + + +def test_filesyncmanager_creates_correct_downloader(): + github_manager = FileSyncManager("github") + assert isinstance(github_manager.downloader, GithubFileDownloader) + + azure_manager = FileSyncManager("azure") + assert isinstance(azure_manager.downloader, AzureFileDownloader) + + +def test_file_downloader_prepare_local_filepath_exists( + github_dnn_file_downloader, mocker +): + mock_path = mocker.patch("pathlib.Path.exists", return_value=True) + + result = github_dnn_file_downloader._prepare_local_filepath( + DNN_FILE["name"], force=False + ) + assert result is None + mock_path.assert_called_once() + + +def test_file_downloader_prepare_local_filepath_force( + github_dnn_file_downloader, mocker +): + mocker.patch("pathlib.Path.exists", return_value=True) + mock_path_mkdir = mocker.patch("pathlib.Path.mkdir") + + result = github_dnn_file_downloader._prepare_local_filepath( + DNN_FILE["name"], force=True + ) + assert result is not None + mock_path_mkdir.assert_called_once_with(parents=True, exist_ok=True) + + +@pytest.mark.parametrize( + "downloaded, total, expect_call, expected_percent", + [ + (50, 100, True, 50), + (50, 0, False, None), + (0, 100, True, 0), + (100, 100, True, 100), + ], +) +def test_report_progress(downloaded, total, expect_call, expected_percent, mocker): + mock_on_progress = mocker.Mock() + + downloader = FileDownloader() + + downloader._report_progress("test_file.txt", downloaded, total, mock_on_progress) + + if expect_call: + mock_on_progress.assert_called_once_with("test_file.txt", expected_percent) + else: + mock_on_progress.assert_not_called() diff --git a/tests/faces/test_duplicate_groups_builder.py b/tests/faces/test_duplicate_groups_builder.py deleted file mode 100644 index a5aca5ec..00000000 --- a/tests/faces/test_duplicate_groups_builder.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest.mock import MagicMock, patch - -import pytest - -from hope_dedup_engine.apps.faces.utils.duplicate_groups_builder import DuplicateGroupsBuilder - - -@pytest.mark.parametrize( - "checked, threshold, expected_groups", - [ - ({("path1", "path2", 0.2), ("path2", "path3", 0.1)}, 0.3, (("path1", "path2"), ("path3", "path2"))), - ({("path1", "path2", 0.2), ("path2", "path3", 0.4)}, 0.3, (("path1", "path2"),)), - ({("path1", "path2", 0.4), ("path2", "path3", 0.4)}, 0.3, ()), - ( - {("path1", "path2", 0.2), ("path2", "path3", 0.2), ("path3", "path4", 0.2)}, - 0.3, - (("path4", "path3"), ("path1", "path2")), - ), - ], -) -def test_duplicate_groups_builder(checked, threshold, expected_groups): - def sort_nested_tuples(nested_tuples: tuple[tuple[str]]) -> tuple[tuple[str]]: - sorted_inner = tuple(tuple(sorted(inner_tuple)) for inner_tuple in nested_tuples) - sorted_outer = tuple(sorted(sorted_inner)) - return sorted_outer - - mock_config = MagicMock() - mock_config.FACE_DISTANCE_THRESHOLD = threshold - with patch("hope_dedup_engine.apps.faces.utils.duplicate_groups_builder.config", mock_config): - DuplicateGroupsBuilder.build(checked) diff --git a/tests/faces/test_duplication_detector.py b/tests/faces/test_duplication_detector.py index 1bcb61bb..c55c1016 100644 --- a/tests/faces/test_duplication_detector.py +++ b/tests/faces/test_duplication_detector.py @@ -7,12 +7,15 @@ import numpy as np import pytest from constance import config -from faces_const import FILENAME, FILENAME_ENCODED_FORMAT, FILENAMES - -from hope_dedup_engine.apps.faces.managers.storage import StorageManager -from hope_dedup_engine.apps.faces.services.duplication_detector import ( - DuplicationDetector, +from faces_const import ( + FACE_DISTANCE_THRESHOLD, + FILENAME, + FILENAME_ENCODED_FORMAT, + FILENAMES, ) + +from hope_dedup_engine.apps.faces.managers import StorageManager +from hope_dedup_engine.apps.faces.services import DuplicationDetector from hope_dedup_engine.apps.faces.services.image_processor import ImageProcessor @@ -67,7 +70,7 @@ def test_init_successful(mock_dd): def test_get_pairs_to_ignore_success( mock_storage_manager, mock_image_processor, ignore_input, expected_output ): - dd = DuplicationDetector(FILENAMES, ignore_input) + dd = DuplicationDetector(FILENAMES, FACE_DISTANCE_THRESHOLD, ignore_input) assert dd.ignore_set == expected_output @@ -88,7 +91,7 @@ def test_get_pairs_to_ignore_exception_handling( mock_storage_manager, mock_image_processor, ignore_input ): with pytest.raises(ValidationError): - DuplicationDetector(filenames=FILENAMES, ignore_pairs=ignore_input) + DuplicationDetector(FILENAMES, 0.2, ignore_pairs=ignore_input) def test_encodings_filename(mock_dd): @@ -194,10 +197,24 @@ def open_mock(filename, mode="rb"): ( True, { - filename: [np.array([0.1, 0.2, 0.3 + i * 0.001])] - for i, filename in enumerate(FILENAMES) + "test_file.jpg": [np.array([0.1, 0.2, 0.3])], + "test_file2.jpg": [np.array([0.1, 0.25, 0.35])], + "test_file3.jpg": [np.array([0.4, 0.5, 0.6])], }, - (tuple(FILENAMES),), + [ + ( + "test_file.jpg", + "test_file2.jpg", + 0.36, + ), # config.FACE_DISTANCE_THRESHOLD + 0.04 + ( + "test_file.jpg", + "test_file3.jpg", + 0.2, + ), # config.FACE_DISTANCE_THRESHOLD - 0.2 + # last pair will not be included in the result because the distance is greater than the threshold + # ("test_file2.jpg", "test_file3.jpg", 0.44), # config.FACE_DISTANCE_THRESHOLD + 0.04 + ], ), ( False, @@ -208,7 +225,7 @@ def open_mock(filename, mode="rb"): ) def test_find_duplicates_successful( mock_dd, - mock_hde_azure_storage, + mock_encoded_azure_storage, mock_hope_azure_storage, image_bytes_io, has_encodings, @@ -216,52 +233,81 @@ def test_find_duplicates_successful( expected_duplicates, ): with ( + patch.object( + mock_dd.storages, + "get_storage", + side_effect=lambda key: { + "encoded": mock_encoded_azure_storage, + "images": mock_hope_azure_storage, + }[key], + ), patch.object( mock_dd.storages.get_storage("images"), "open", side_effect=image_bytes_io.fake_open, ), + patch.object( + mock_dd.storages.get_storage("images"), + "listdir", + return_value=([], FILENAMES), + ), patch.object( mock_dd.storages.get_storage("encoded"), "open", side_effect=image_bytes_io.fake_open, ), - patch.object( - mock_dd.storages, - "get_storage", - side_effect=lambda key: { - "encoded": mock_hde_azure_storage, - "images": mock_hope_azure_storage, - }[key], - ), patch.object(mock_dd, "_has_encodings", return_value=has_encodings), patch.object( mock_dd, "_load_encodings_all", return_value=mock_encodings ) as mock_load_encodings, patch.object(mock_dd.image_processor, "encode_face"), - patch("face_recognition.face_distance", return_value=np.array([0.05])), + patch( + "face_recognition.face_distance", + side_effect=[ + np.array([config.FACE_DISTANCE_THRESHOLD - 0.04]), + np.array([config.FACE_DISTANCE_THRESHOLD - 0.2]), + np.array([config.FACE_DISTANCE_THRESHOLD + 0.04]), + ], + ), ): - duplicates = mock_dd.find_duplicates() + duplicates = list(mock_dd.find_duplicates()) if has_encodings: - assert {frozenset(t) for t in duplicates} == { - frozenset(t) for t in expected_duplicates - } + assert duplicates == expected_duplicates mock_dd.image_processor.encode_face.assert_not_called() mock_dd._load_encodings_all.assert_called_once() - # mock_hde_azure_storage.exists.assert_called_with(FILENAME_ENCODED_FORMAT.format(FILENAMES[-1])) else: mock_load_encodings.assert_called_once() mock_dd.image_processor.encode_face.assert_called() -def test_find_duplicates_exception_handling(mock_dd): +def test_find_duplicates_exception_handling( + mock_dd, mock_hope_azure_storage, mock_encoded_azure_storage, image_bytes_io +): with ( pytest.raises(Exception, match="Test exception"), + patch.object( + mock_dd.storages, + "get_storage", + side_effect=lambda key: { + "encoded": mock_encoded_azure_storage, + "images": mock_hope_azure_storage, + }[key], + ), + patch.object( + mock_dd.storages.get_storage("images"), + "listdir", + return_value=([], FILENAMES), + ), + patch.object( + mock_dd.storages.get_storage("images"), + "open", + side_effect=image_bytes_io.fake_open, + ), patch.object( mock_dd, "_load_encodings_all", side_effect=Exception("Test exception") ), patch.object(mock_dd.logger, "exception") as mock_logger_exception, ): - mock_dd.find_duplicates() + list(mock_dd.find_duplicates()) mock_logger_exception.assert_called_once() diff --git a/tests/faces/test_file_sync.py b/tests/faces/test_file_sync.py new file mode 100644 index 00000000..2832b5df --- /dev/null +++ b/tests/faces/test_file_sync.py @@ -0,0 +1,72 @@ +from unittest.mock import patch + +from django.urls import reverse + +import pytest + +from hope_dedup_engine.apps.core.exceptions import DownloaderKeyError +from hope_dedup_engine.apps.faces.managers import FileSyncManager +from hope_dedup_engine.apps.faces.managers.file_sync import ( + AzureFileDownloader, + GithubFileDownloader, +) + + +@pytest.mark.parametrize( + "downloader_key, expected_downloader", + [ + ("github", GithubFileDownloader), + ("azure", AzureFileDownloader), + ], +) +def test_create_downloader(downloader_key, expected_downloader): + file_sync_manager = FileSyncManager(downloader_key) + assert isinstance(file_sync_manager.downloader, expected_downloader) + + +def test_create_downloader_failure(): + with pytest.raises(DownloaderKeyError): + FileSyncManager("unknown") + + +@pytest.mark.parametrize( + "active_queues, expected_call_count, multiple_workers, delay_called", + [ + (None, 0, False, False), + ({"worker1": "queue"}, 1, False, True), + ({"worker1": "queue", "worker2": "queue"}, 2, True, False), + ], +) +def test_sync_dnn_files( + client, active_queues, expected_call_count, multiple_workers, delay_called +): + with ( + patch( + "hope_dedup_engine.apps.faces.admin.celery_app.control.inspect" + ) as mock_inspect, + patch( + "hope_dedup_engine.apps.faces.admin.DummyModelAdmin.message_user" + ) as mock_message_user, + patch("hope_dedup_engine.apps.faces.admin.sync_dnn_files.s") as mock_s, + patch("hope_dedup_engine.apps.faces.admin.group") as mock_group, + patch("hope_dedup_engine.apps.faces.admin.sync_dnn_files.delay") as mock_delay, + ): + + mock_inspect.return_value.active_queues.return_value = active_queues + + url = reverse("admin:faces_dummymodel_sync_dnn_files") + response = client.get(url, follow=True) + + assert response.status_code == 200 + + if multiple_workers: + list(mock_group.call_args[0][0]) + mock_group.assert_called_once() + assert mock_s.call_count == expected_call_count + mock_group.return_value.apply_async.assert_called_once() + + if delay_called: + mock_delay.assert_called_once_with(force=True) + + mock_inspect.return_value.active_queues.assert_called_once() + mock_message_user.assert_called_once() diff --git a/tests/faces/test_forms.py b/tests/faces/test_forms.py index 4fcc3bb5..da337007 100644 --- a/tests/faces/test_forms.py +++ b/tests/faces/test_forms.py @@ -13,9 +13,15 @@ def test_to_python_valid_case(): @pytest.mark.parametrize( "input_value, expected_error_message", [ - ("104.0, 177.0", "Enter a valid tuple of three float values separated by commas and spaces"), + ( + "104.0, 177.0", + "Enter a valid tuple of three float values separated by commas and spaces", + ), ("104.0, 177.0, 256.0", "Each value must be between -255 and 255."), - ("104.0, abc, 123.0", "Enter a valid tuple of three float values separated by commas and spaces"), + ( + "104.0, abc, 123.0", + "Enter a valid tuple of three float values separated by commas and spaces", + ), ], ) def test_to_python_invalid_cases(input_value, expected_error_message): @@ -27,7 +33,10 @@ def test_to_python_invalid_cases(input_value, expected_error_message): @pytest.mark.parametrize( "input_value, expected_output", - [((104.0, 177.0, 123.0), "104.0, 177.0, 123.0"), ("104.0, 177.0, 123.0", "104.0, 177.0, 123.0")], + [ + ((104.0, 177.0, 123.0), "104.0, 177.0, 123.0"), + ("104.0, 177.0, 123.0", "104.0, 177.0, 123.0"), + ], ) def test_prepare_value(input_value, expected_output): field = MeanValuesTupleField() diff --git a/tests/faces/test_image_processor.py b/tests/faces/test_image_processor.py index 747b253f..1347eb75 100644 --- a/tests/faces/test_image_processor.py +++ b/tests/faces/test_image_processor.py @@ -10,33 +10,51 @@ BLOB_FROM_IMAGE_MEAN_VALUES, BLOB_FROM_IMAGE_SCALE_FACTOR, DEPLOY_PROTO_SHAPE, + FACE_DISTANCE_THRESHOLD, FACE_REGIONS_INVALID, FACE_REGIONS_VALID, FILENAME, FILENAME_ENCODED, ) -from hope_dedup_engine.apps.faces.managers.net import DNNInferenceManager -from hope_dedup_engine.apps.faces.managers.storage import StorageManager -from hope_dedup_engine.apps.faces.services.image_processor import BlobFromImageConfig, FaceEncodingsConfig +from hope_dedup_engine.apps.faces.managers import DNNInferenceManager, StorageManager +from hope_dedup_engine.apps.faces.services.image_processor import ( + BlobFromImageConfig, + FaceEncodingsConfig, +) -def test_init_creates_expected_attributes(mock_net_manager: DNNInferenceManager, mock_image_processor): +def test_init_creates_expected_attributes( + mock_net_manager: DNNInferenceManager, mock_image_processor +): assert isinstance(mock_image_processor.storages, StorageManager) assert mock_image_processor.net is mock_net_manager assert isinstance(mock_image_processor.blob_from_image_cfg, BlobFromImageConfig) - assert mock_image_processor.blob_from_image_cfg.scale_factor == config.BLOB_FROM_IMAGE_SCALE_FACTOR + assert ( + mock_image_processor.blob_from_image_cfg.scale_factor + == config.BLOB_FROM_IMAGE_SCALE_FACTOR + ) assert isinstance(mock_image_processor.face_encodings_cfg, FaceEncodingsConfig) - assert mock_image_processor.face_encodings_cfg.num_jitters == config.FACE_ENCODINGS_NUM_JITTERS + assert ( + mock_image_processor.face_encodings_cfg.num_jitters + == config.FACE_ENCODINGS_NUM_JITTERS + ) assert mock_image_processor.face_encodings_cfg.model == config.FACE_ENCODINGS_MODEL - assert mock_image_processor.face_detection_confidence == config.FACE_DETECTION_CONFIDENCE - assert mock_image_processor.distance_threshold == config.FACE_DISTANCE_THRESHOLD + assert ( + mock_image_processor.face_detection_confidence + == config.FACE_DETECTION_CONFIDENCE + ) + assert mock_image_processor.distance_threshold == FACE_DISTANCE_THRESHOLD assert mock_image_processor.nms_threshold == config.NMS_THRESHOLD def test_get_shape_valid(mock_prototxt_file): with patch("builtins.open", mock_prototxt_file): - config = BlobFromImageConfig(scale_factor=BLOB_FROM_IMAGE_SCALE_FACTOR, mean_values=BLOB_FROM_IMAGE_MEAN_VALUES) + config = BlobFromImageConfig( + scale_factor=BLOB_FROM_IMAGE_SCALE_FACTOR, + mean_values=BLOB_FROM_IMAGE_MEAN_VALUES, + prototxt_path="test.prototxt", + ) shape = config._get_shape() assert shape == DEPLOY_PROTO_SHAPE @@ -44,16 +62,24 @@ def test_get_shape_valid(mock_prototxt_file): def test_get_shape_invalid(): with patch("builtins.open", mock_open(read_data="invalid_prototxt_content")): with pytest.raises(ValidationError): - BlobFromImageConfig(scale_factor=BLOB_FROM_IMAGE_SCALE_FACTOR, mean_values=BLOB_FROM_IMAGE_MEAN_VALUES) + BlobFromImageConfig( + scale_factor=BLOB_FROM_IMAGE_SCALE_FACTOR, + mean_values=BLOB_FROM_IMAGE_MEAN_VALUES, + prototxt_path="test.prototxt", + ) -def test_get_face_detections_dnn_with_detections(mock_image_processor, mock_net, mock_open_context_manager): +def test_get_face_detections_dnn_with_detections( + mock_image_processor, mock_net, mock_open_context_manager +): dnn, imdecode, resize, _, expected_regions = mock_net with ( patch("cv2.imdecode", imdecode), patch("cv2.resize", resize), patch.object( - mock_image_processor.storages.get_storage("images"), "open", return_value=mock_open_context_manager + mock_image_processor.storages.get_storage("images"), + "open", + return_value=mock_open_context_manager, ), patch.object(mock_image_processor, "net", dnn), ): @@ -66,19 +92,41 @@ def test_get_face_detections_dnn_with_detections(mock_image_processor, mock_net, def test_get_face_detections_dnn_no_detections(mock_image_processor): - with (patch.object(mock_image_processor, "_get_face_detections_dnn", return_value=[]),): + with ( + patch.object(mock_image_processor, "_get_face_detections_dnn", return_value=[]), + ): face_regions = mock_image_processor._get_face_detections_dnn() assert len(face_regions) == 0 +def test_get_face_detections_dnn_exception( + mock_image_processor, mock_open_context_manager +): + with ( + patch.object( + mock_image_processor.storages.get_storage("images"), + "open", + return_value=mock_open_context_manager, + ), + patch.object(mock_open_context_manager, "read", return_value=b"fake_data"), + patch("cv2.imdecode", side_effect=TypeError("Test exception")), + ): + with pytest.raises(TypeError, match="Test exception"): + mock_image_processor._get_face_detections_dnn(FILENAME) + + @pytest.mark.parametrize("face_regions", (FACE_REGIONS_VALID, FACE_REGIONS_INVALID)) def test_encode_face(mock_image_processor, image_bytes_io, face_regions): with ( patch.object( - mock_image_processor.storages.get_storage("images"), "open", side_effect=image_bytes_io.fake_open + mock_image_processor.storages.get_storage("images"), + "open", + side_effect=image_bytes_io.fake_open, ) as mocked_image_open, patch.object( - mock_image_processor.storages.get_storage("encoded"), "open", side_effect=image_bytes_io.fake_open + mock_image_processor.storages.get_storage("encoded"), + "open", + side_effect=image_bytes_io.fake_open, ) as mocked_encoded_open, patch.object( mock_image_processor, "_get_face_detections_dnn", return_value=face_regions @@ -109,11 +157,15 @@ def test_encode_face(mock_image_processor, image_bytes_io, face_regions): (str("face_encodings"), "Test face_encodings exception"), ), ) -def test_encode_face_exception_handling(mock_image_processor, mock_net, method: str, exception_str): +def test_encode_face_exception_handling( + mock_image_processor, mock_net, method: str, exception_str +): dnn, imdecode, *_ = mock_net with ( pytest.raises(Exception, match=exception_str), - patch.object(face_recognition, method, side_effect=Exception(exception_str)) as mock_exception, + patch.object( + face_recognition, method, side_effect=Exception(exception_str) + ) as mock_exception, patch.object(mock_image_processor, "net", dnn), patch("cv2.imdecode", imdecode), patch.object(mock_image_processor.logger, "exception") as mock_logger_exception, diff --git a/tests/faces/test_net_manager.py b/tests/faces/test_net_manager.py index 3a080bd8..0ca06995 100644 --- a/tests/faces/test_net_manager.py +++ b/tests/faces/test_net_manager.py @@ -1,11 +1,13 @@ from constance import config -from hope_dedup_engine.apps.faces.managers.net import DNNInferenceManager +from hope_dedup_engine.apps.faces.managers import DNNInferenceManager def test_successful(mock_storage_manager, mock_net_manager): - dnn_manager = DNNInferenceManager(mock_storage_manager.storages["cv2dnn"]) - mock_net_manager.setPreferableBackend.assert_called_once_with(int(config.DNN_BACKEND)) + dnn_manager = DNNInferenceManager(mock_storage_manager.storages["cv2"]) + mock_net_manager.setPreferableBackend.assert_called_once_with( + int(config.DNN_BACKEND) + ) mock_net_manager.setPreferableTarget.assert_called_once_with(int(config.DNN_TARGET)) assert isinstance(dnn_manager, DNNInferenceManager) diff --git a/tests/faces/test_storage_manager.py b/tests/faces/test_storage_manager.py index b211de8a..f7d2209f 100644 --- a/tests/faces/test_storage_manager.py +++ b/tests/faces/test_storage_manager.py @@ -1,14 +1,25 @@ +from django.core.files.storage import FileSystemStorage + import pytest +from storages.backends.azure_storage import AzureStorage -from hope_dedup_engine.apps.core.storage import CV2DNNStorage, HDEAzureStorage, HOPEAzureStorage -from hope_dedup_engine.apps.faces.exceptions import StorageKeyError -from hope_dedup_engine.apps.faces.managers.storage import StorageManager +from hope_dedup_engine.apps.core.exceptions import StorageKeyError +from hope_dedup_engine.apps.faces.managers import StorageManager def test_initialization(mock_storage_manager): - assert isinstance(mock_storage_manager.storages["images"], HOPEAzureStorage) - assert isinstance(mock_storage_manager.storages["cv2dnn"], CV2DNNStorage) - assert isinstance(mock_storage_manager.storages["encoded"], HDEAzureStorage) + assert isinstance( + mock_storage_manager.storages["cv2"], + FileSystemStorage, + ) + assert isinstance( + mock_storage_manager.storages["images"], + AzureStorage, + ) + assert isinstance( + mock_storage_manager.storages["encoded"], + FileSystemStorage, + ) def test_missing_file(): @@ -22,13 +33,13 @@ def test_invalid_key(mock_storage_manager): @pytest.mark.parametrize( - "test_input, expected_output", + "test_input, expected_method", [ - ("images", HOPEAzureStorage), - ("cv2dnn", CV2DNNStorage), - ("encoded", HDEAzureStorage), + ("images", "exists"), + ("cv2", "exists"), + ("encoded", "exists"), ], ) -def test_valid_key(mock_storage_manager, test_input, expected_output): +def test_valid_key(mock_storage_manager, test_input, expected_method): storage_object = mock_storage_manager.get_storage(test_input) - assert isinstance(storage_object, expected_output) + assert hasattr(storage_object, expected_method) diff --git a/tests/test_command_demo.py b/tests/test_command_demo.py new file mode 100644 index 00000000..68d0bf97 --- /dev/null +++ b/tests/test_command_demo.py @@ -0,0 +1,48 @@ +import os +from io import StringIO +from unittest import mock + +from django.core.management import call_command + +import pytest +from pytest_mock import MockerFixture + + +@pytest.fixture() +def environment(): + return { + "DEMO_IMAGES_PATH": "demo_images", + "DNN_FILES_PATH": "dnn_files", + } + + +@pytest.fixture +def mock_azurite_manager(mocker: MockerFixture): + with mock.patch( + "hope_dedup_engine.apps.core.management.commands.utils.azurite_manager.AzuriteManager" + ) as MockAzuriteManager: + yield MockAzuriteManager + + +def test_demo_handle_success(environment, mock_azurite_manager): + out = StringIO() + with ( + mock.patch.dict("os.environ", environment, clear=True), + mock.patch("pathlib.Path.exists", return_value=True), + ): + call_command( + "demo", + demo_images="/path/to/demo/images", + dnn_files="/path/to/dnn/files", + stdout=out, + ) + assert "error" not in str(out.getvalue()) + assert mock_azurite_manager.call_count == 4 + assert mock_azurite_manager.return_value.upload_files.call_count == 2 + + +def test_demo_handle_exception(environment, mock_azurite_manager): + mock_azurite_manager.side_effect = Exception() + with mock.patch.dict(os.environ, environment, clear=True): + with pytest.raises(Exception): + call_command("demo", ignore_errors=False) diff --git a/tests/test_command_dnnsetup.py b/tests/test_command_dnnsetup.py new file mode 100644 index 00000000..a55baeae --- /dev/null +++ b/tests/test_command_dnnsetup.py @@ -0,0 +1,90 @@ +from io import StringIO +from typing import Final +from unittest import mock + +from django.core.exceptions import ValidationError +from django.core.management import call_command +from django.core.management.base import CommandError, SystemCheckError + +import pytest +from pytest_mock import MockerFixture + +DNN_FILES: Final[tuple[dict[str, str]]] = ( + {"url": "http://example.com/file1", "filename": "file1"}, + {"url": "http://example.com/file2", "filename": "file2"}, +) + + +@pytest.fixture +def mock_requests_get(): + with mock.patch("requests.get") as mock_get: + mock_response = mock_get.return_value.__enter__.return_value + mock_response.iter_content.return_value = [b"Hello, world!"] * 3 + mock_response.raise_for_status = lambda: None + yield mock_get + + +@pytest.fixture +def mock_azurite_manager(mocker: MockerFixture): + yield mocker.patch( + "hope_dedup_engine.apps.core.management.commands.dnnsetup.AzureStorage", + ) + + +@pytest.fixture +def mock_dnn_files(mocker: MockerFixture): + yield mocker.patch( + "hope_dedup_engine.apps.core.management.commands.dnnsetup.Command.dnn_files", + new_callable=mocker.PropertyMock, + ) + + +@pytest.mark.parametrize( + "force, expected_count, existing_files", + [ + (False, 2, []), + (False, 1, [DNN_FILES[0]["filename"]]), + (False, 0, [f["filename"] for f in DNN_FILES][:2]), + (True, 2, []), + (True, 2, [DNN_FILES[0]["filename"]]), + (True, 2, [f["filename"] for f in DNN_FILES][:2]), + ], +) +def test_dnnsetup_handle_success( + mock_requests_get, + mock_azurite_manager, + mock_dnn_files, + force, + expected_count, + existing_files, +): + mock_dnn_files.return_value = DNN_FILES + mock_azurite_manager().listdir.return_value = ([], existing_files) + out = StringIO() + + call_command("dnnsetup", stdout=out, force=force) + + assert "SYSTEM HALTED" not in out.getvalue() + assert mock_requests_get.call_count == expected_count + assert mock_azurite_manager().open.call_count == expected_count + + +@pytest.mark.parametrize( + "side_effect, expected_exception", + [ + (FileNotFoundError("File not found"), SystemExit), + (ValidationError("Invalid argument"), SystemExit), + (CommandError("Command execution failed"), SystemExit), + (SystemCheckError("System check failed"), SystemExit), + (Exception("Unknown error"), SystemExit), + ], +) +def test_dnnsetup_handle_exception( + mock_requests_get, mock_azurite_manager, side_effect, expected_exception +): + mock_azurite_manager.side_effect = side_effect + out = StringIO() + with pytest.raises(expected_exception): + call_command("dnnsetup", stdout=out) + + assert "SYSTEM HALTED" in out.getvalue() diff --git a/tests/test_commands.py b/tests/test_commands.py index 488e3fc5..bd625ce1 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,7 +2,7 @@ from io import StringIO from unittest import mock -from django.core.management import CommandError, call_command +from django.core.management import call_command import pytest from testutils.factories import SuperUserFactory @@ -18,21 +18,40 @@ def environment(): "CELERY_BROKER_URL": "", "DATABASE_URL": "", "SECRET_KEY": "", + "DEFAULT_ROOT": "/tmp/default", "MEDIA_ROOT": "/tmp/media", "STATIC_ROOT": "/tmp/static", "SECURE_SSL_REDIRECT": "1", "SESSION_COOKIE_SECURE": "1", + "DJANGO_SETTINGS_MODULE": "hope_dedup_engine.config.settings", } -@pytest.mark.parametrize("static_root", ["static", ""], ids=["static_missing", "static_existing"]) +@pytest.fixture +def mock_settings(): + with mock.patch("django.conf.settings") as mock_settings: + mock_settings.AZURE_CONTAINER_HOPE = "hope-container" + mock_settings.AZURE_CONTAINER_DNN = "dnn-container" + mock_settings.AZURE_CONTAINER_HDE = "hde-container" + yield mock_settings + + +@pytest.mark.parametrize( + "static_root", ["static", ""], ids=["static_missing", "static_existing"] +) @pytest.mark.parametrize("static", [True, False], ids=["static", "no-static"]) @pytest.mark.parametrize("verbosity", [1, 0], ids=["verbose", ""]) @pytest.mark.parametrize("migrate", [True, False], ids=["migrate", ""]) -def test_upgrade_init(verbosity, migrate, monkeypatch, environment, static, static_root, tmp_path): +def test_upgrade_init( + verbosity, migrate, monkeypatch, environment, static, static_root, tmp_path +): static_root_path = tmp_path / static_root out = StringIO() - with mock.patch.dict(os.environ, {**environment, "STATIC_ROOT": str(static_root_path.absolute())}, clear=True): + with mock.patch.dict( + os.environ, + {**environment, "STATIC_ROOT": str(static_root_path.absolute())}, + clear=True, + ): call_command( "upgrade", static=static, @@ -41,6 +60,7 @@ def test_upgrade_init(verbosity, migrate, monkeypatch, environment, static, stat migrate=migrate, stdout=out, check=False, + dnn_setup=False, verbosity=verbosity, ) assert "error" not in str(out.getvalue()) @@ -54,14 +74,20 @@ def test_upgrade(verbosity, migrate, monkeypatch, environment): out = StringIO() SuperUserFactory() with mock.patch.dict(os.environ, environment, clear=True): - call_command("upgrade", stdout=out, check=False, verbosity=verbosity) + call_command( + "upgrade", + stdout=out, + check=False, + dnn_setup=False, + verbosity=verbosity, + ) assert "error" not in str(out.getvalue()) -def test_upgrade_check(mocked_responses, admin_user, environment): - out = StringIO() - with mock.patch.dict(os.environ, environment, clear=True): - call_command("upgrade", stdout=out, check=True) +# def test_upgrade_check(mocked_responses, admin_user, environment): +# out = StringIO() +# with mock.patch.dict(os.environ, environment, clear=True): +# call_command("upgrade", stdout=out, check=True) def test_upgrade_noadmin(db, mocked_responses, environment): @@ -80,50 +106,31 @@ def test_upgrade_admin(db, mocked_responses, environment, admin): out = StringIO() with mock.patch.dict(os.environ, environment, clear=True): - call_command("upgrade", stdout=out, check=True, admin_email=email) - - -@pytest.mark.parametrize("verbosity", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("develop", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("diff", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("config", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("check", [0, 1], ids=["0", "1"]) -def test_env(mocked_responses, verbosity, develop, diff, config, check): - out = StringIO() - environ = { - "ADMIN_URL_PREFIX": "test", - "SECURE_SSL_REDIRECT": "1", - "SECRET_KEY": "a" * 120, - "SESSION_COOKIE_SECURE": "1", - } - with mock.patch.dict(os.environ, environ, clear=True): call_command( - "env", - ignore_errors=True if check == 1 else False, + "upgrade", stdout=out, - verbosity=verbosity, - develop=develop, - diff=diff, - config=config, - check=check, + check=False, + dnn_setup=False, + static=False, + admin_email=email, ) - assert "error" not in str(out.getvalue()) - - -def test_env_raise(mocked_responses): - environ = {"ADMIN_URL_PREFIX": "test"} - with mock.patch.dict(os.environ, environ, clear=True): - with pytest.raises(CommandError): - call_command("env", ignore_errors=False, check=True) def test_upgrade_exception(mocked_responses, environment): - with mock.patch("hope_dedup_engine.apps.core.management.commands.upgrade.call_command") as m: + with ( + mock.patch.dict( + os.environ, + {"ADMIN_EMAIL": "2222", "ADMIN_USER": "admin", **environment}, + clear=True, + ), + mock.patch( + "hope_dedup_engine.apps.core.management.commands.upgrade.call_command" + ) as m, + ): m.side_effect = Exception with pytest.raises(SystemExit): call_command("upgrade") - out = StringIO() - with mock.patch.dict(os.environ, {"ADMIN_EMAIL": "2222", "ADMIN_USER": "admin", **environment}, clear=True): + out = StringIO() with pytest.raises(SystemExit): call_command("upgrade", stdout=out, check=True, admin_email="") diff --git a/tests/test_smartenv.py b/tests/test_smartenv.py deleted file mode 100644 index 637e2473..00000000 --- a/tests/test_smartenv.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -from unittest import mock - -import pytest - -from hope_dedup_engine.config import SmartEnv - - -@pytest.fixture() -def env(): - return SmartEnv(STORAGE_DEFAULT=(str, "")) - - -@pytest.mark.parametrize( - "storage", - [ - "storage.SampleStorage?bucket=container&option=value&connection_string=Defaul", - "storage.SampleStorage?bucket=container&option=value&connection_string=DefaultEndpointsProtocol=http;Account" - "Name=devstoreaccount1;AccountKey=ke1==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;", - ], -) -def test_storage_options(storage, env): - with mock.patch.dict(os.environ, {"STORAGE_DEFAULT": storage}, clear=True): - ret = env.storage("STORAGE_DEFAULT") - - assert ret["BACKEND"] == "storage.SampleStorage" - assert sorted(ret["OPTIONS"].keys()) == ["bucket", "connection_string", "option"] - - -@pytest.mark.parametrize("storage", ["storage.SampleStorage"]) -def test_storage(storage, env): - with mock.patch.dict(os.environ, {"STORAGE_DEFAULT": storage}, clear=True): - ret = env.storage("STORAGE_DEFAULT") - - assert ret["BACKEND"] == "storage.SampleStorage" - - -def test_storage_empty(env): - with mock.patch.dict(os.environ, {}, clear=True): - assert not env.storage("STORAGE_DEFAULT") - - -def test_env(): - e = SmartEnv( - **{ - "T1": (str, "a@b.com"), - "T2": (str, "a@b.com", "help"), - "T3": (str, "a@b.com", "help", "dev@b.com"), - "T4": (int, None), - } - ) - - assert e("T1") == "a@b.com" - assert e.get_help("T1") == "" - assert e.for_develop("T1") == "a@b.com" - assert e.get_default("T1") == "a@b.com" - - assert e("T2") == "a@b.com" - assert e.get_help("T2") == "help" - assert e.for_develop("T2") == "a@b.com" - assert e.get_default("T2") == "a@b.com" - - assert e("T3") == "a@b.com" - assert e.get_help("T3") == "help" - assert e.for_develop("T3") == "dev@b.com" - assert e.get_default("T3") == "a@b.com" - - assert e.get_default("cc") == "" - - with pytest.raises(TypeError): - assert e.get_default("T4") diff --git a/tests/test_state.py b/tests/test_state.py index 90502366..c3cc79c6 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -42,7 +42,9 @@ def test_configure(state): @freeze_time("2000-01-01T00:00:00Z") def test_add_cookies(state): - state.add_cookie("test", 22, 3600, None, "/path/", "domain.example.com", True, True, "lax") + state.add_cookie( + "test", 22, 3600, None, "/path/", "domain.example.com", True, True, "lax" + ) r: HttpResponse = HttpResponse() state.set_cookies(r) diff --git a/tests/test_storage.py b/tests/test_storage.py deleted file mode 100644 index 23b4b198..00000000 --- a/tests/test_storage.py +++ /dev/null @@ -1,28 +0,0 @@ -from django.core.files.base import ContentFile - -import pytest - -from hope_dedup_engine.apps.core.storage import CV2DNNStorage, HOPEAzureStorage - - -def test_fs(tmp_path): - s = CV2DNNStorage(tmp_path) - s.save("test", ContentFile("aa", "test.txt")) - s.save("test", ContentFile("bb", "test.txt")) - assert s.listdir(".") == ([], ["test"]) - with s.open("test") as fd: - assert fd.read() == b"bb" - s.delete("test") - - -def test_azure(tmp_path): - s = HOPEAzureStorage() - with pytest.raises(RuntimeError): - s.open("test", "rw") - with pytest.raises(RuntimeError): - s.save("test", ContentFile("aa", "test.txt")) - with pytest.raises(RuntimeError): - s.delete("test") - - assert s.listdir(".") == ([], []) - assert s.open("test", "r") diff --git a/tests/utils/test_utils_http.py b/tests/utils/test_utils_http.py index 2a696f2c..51ab2897 100644 --- a/tests/utils/test_utils_http.py +++ b/tests/utils/test_utils_http.py @@ -4,7 +4,12 @@ import pytest from hope_dedup_engine.state import state -from hope_dedup_engine.utils.http import absolute_reverse, absolute_uri, get_server_host, get_server_url +from hope_dedup_engine.utils.http import ( + absolute_reverse, + absolute_uri, + get_server_host, + get_server_url, +) if TYPE_CHECKING: from django.http import HttpRequest