Merge branch 'staging' into master-to-stg

.github/helpers/.env-selenium

@@ -1,15 +1,11 @@
-REDIS_INSTANCE=redis:6379
 PYTHONUNBUFFERED=1
 SECRET_KEY=secretkey
 ENV=dev
 DEBUG=true
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CACHE_LOCATION=redis://redis:6379/1
+CONSTANCE_REDIS_CONNECTION=redis://redis:6379/0
 DATABASE_URL=postgis://postgres:postgres@db:5432/postgres
-DATABASE_URL_HUB_MIS=postgis://postgres:postgres@db:5432/mis_datahub
-DATABASE_URL_HUB_CA=postgis://postgres:postgres@db:5432/ca_datahub
-DATABASE_URL_HUB_ERP=postgis://postgres:postgres@db:5432/erp_datahub
-DATABASE_URL_HUB_REGISTRATION=postgis://postgres:postgres@db:5432/rdi_datahub
 USE_DUMMY_EXCHANGE_RATES=yes
 CELERY_TASK_ALWAYS_EAGER=true

.github/helpers/.env-unit

@@ -4,10 +4,6 @@ POSTGRES_DB=postgres
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 DATABASE_URL=postgis://postgres:postgres@db:5432/postgres
-DATABASE_URL_HUB_MIS=postgis://postgres:postgres@db:5432/mis_datahub
-DATABASE_URL_HUB_CA=postgis://postgres:postgres@db:5432/ca_datahub
-DATABASE_URL_HUB_ERP=postgis://postgres:postgres@db:5432/erp_datahub
-DATABASE_URL_HUB_REGISTRATION=postgis://postgres:postgres@db:5432/rdi_datahub
 POSTGRES_SSL_MODE=off
 EMAIL_HOST=TBD
 EMAIL_HOST_USER=TBD
@@ -16,5 +12,6 @@ KOBO_KF_URL=https://kobo.humanitarianresponse.info
 KOBO_KC_URL=https://kobo.humanitarianresponse.info
 KOBO_MASTER_API_TOKEN=token
 EXCHANGE_RATES_API_KEY=token
-REDIS_INSTANCE=redis:6379
+CACHE_LOCATION=redis://redis:6379/1
+CONSTANCE_REDIS_CONNECTION=redis://redis:6379/0
 PYTHONUNBUFFERED=1

.github/helpers/docker-compose.selenium.yml

@@ -31,7 +31,7 @@ services:
       - backend-web-app:/tmp/
     command: |
       sh -c "
-        cp -r /packages/__pypackages__/3.11/lib/hct_mis_api/apps/web/* /tmp/
+        cp -r /packages/__pypackages__/3.12/lib/hct_mis_api/apps/web/* /tmp/
       "
     restart: "no"
 

.github/helpers/docker-compose.tst.yml

@@ -17,7 +17,6 @@ services:
     volumes:
       - ../../development_tools/postgres/init:/docker-entrypoint-initdb.d
     environment:
-      - POSTGRES_MULTIPLE_DATABASES=unicef_hct_mis_cashassist,rdi_datahub,mis_datahub,erp_datahub,ca_datahub
       - POSTGRES_DB=postgres
       - POSTGRES_USER=postgres
       - POSTGRES_PASS=postgres

.github/workflows/ci.yml

@@ -264,15 +264,15 @@ jobs:
             -f ./.github/helpers/docker-compose.selenium.yml \
             run backend bash -c "
               waitforit -host=db -port=5432 -timeout=30
-              pytest -svvv $extra_options ./tests/selenium --cov-report xml:test-coverage/coverage.xml --html-report=./tests/selenium/output_data/report/report.html --randomly-seed=42
+              pytest -x -svvv $extra_options ./tests/selenium --cov-report xml:test-coverage/coverage.xml --html-report=./tests/selenium/output_data/report/report.html --randomly-seed=42
             "
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         if: always()
         continue-on-error: true
         with:
           name: report
-          path: ./tests/report/
+          path: ./tests/selenium/output_data/report/
           retention-days: 5
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

README.md

@@ -39,15 +39,18 @@ Following the Principles for Digital Development, the “reuse and improve” an
 
 ### Why
 
-* Put cash recipients at the center with secure personal data processing
-* Unified consolidated reporting for beneficiary data
-* Increase accountability beneficiaries and donors
-* Financial Inclusion of Beneficiaries
-* To Enhance Traceability
-* Simplify current process in EMOPS
-* Ensuring due diligence in-cash transfer
-* Grow the use of effective cash program for children in a risk informed manner
-* Eliminate redundancies and dupes in how we do things
+|#| HOPE Functions  | Added Value |		
+|---|--------------------------|----------------------------------------------|
+|1|Registration of payees data	| Quality Assurance for data collection, verification of eligibility |
+|2|Deduplication of personal records|Mitigate the risk of duplicate records|
+|3|Target payments|Ensure inclusion of relevant payees in payment plan|
+|4|Entitlment Calculations|Ensure cash benefit entitlement rules are uphold|
+|5|Payment Management|Approval, Authorization and Financial Release tracking|
+|6|Reconciliation of Individual payment|Support liquidation or advance or reimbursement to Financial Service Provider|
+|7|Payment Verification|Mitigate the risk of inaccurate FSP reports and fraud|
+|8|Grievances Redressals and Payees Communications|Ensure payment quality and direct communication with payees|
+|9|Roles base access|Uphold segragation of duties and data protection principles|
+|10|Reporting|Access process and output indicators|
 
 ## Legal
 Humanitarian cash Operations and Programme Ecosystem

codecov.yml

@@ -25,3 +25,13 @@ coverage:
     - "hct_mis_api/conftest.py"
     - "hct_mis_api/config/settings.py"
     - "hct_mis_api/apps/core/management/commands/*"
+    - "pragma: no cover"
+    - "pragma: no-cover"
+    - "def __repr__"
+    - "raise AssertionError"
+    - "raise NotImplementedError"
+    - "except ImportError"
+    - "if __name__ == .__main__."
+    - "if TYPE_CHECKING"
+    - "^\\s*(import\\s.+|from\\s+.+import\\s+.+)"
+    - "logger.exception(e)"

development_tools/.env.example

@@ -10,10 +10,6 @@ POSTGRES_PASS=postgres
 PGUSER=postgres
 POSTGRES_HOST_AUTH_METHOD=trust
 DATABASE_URL=postgis://postgres:postgres@db:5432/postgres
-DATABASE_URL_HUB_MIS=postgis://postgres:postgres@db:5432/mis_datahub
-DATABASE_URL_HUB_CA=postgis://postgres:postgres@db:5432/ca_datahub
-DATABASE_URL_HUB_ERP=postgis://postgres:postgres@db:5432/erp_datahub
-DATABASE_URL_HUB_REGISTRATION=postgis://postgres:postgres@db:5432/rdi_datahub
 POSTGRES_SSL_MODE=off
 EMAIL_HOST=TBD
 EMAIL_HOST_USER=TBD
@@ -33,9 +29,14 @@ DATAMART_URL=https://datamart.unicef.io
 EXCHANGE_RATES_API_KEY=
 ADMIN_PANEL_URL=unicorn
 ROOT_ACCESS_TOKEN=test
-CELERY_BROKER_URL="redis://redis:6379/0"
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CONSTANCE_REDIS_CONNECTION=redis://redis:6379/0
+CACHE_LOCATION=redis://redis:6379/1
+
 PROFILING=off
 USE_DUMMY_EXCHANGE_RATES=no
+OPENAPI_URL=127.0.0.1:8080/api/rest/
 
 KOBO_API_URL=https://kobo.humanitarianresponse.info
 KOBO_KF_URL=https://kobo.humanitarianresponse.info

development_tools/compose.yml

@@ -16,7 +16,6 @@ services:
     env_file:
       - .env
     environment:
-      - REDIS_INSTANCE=redis:6379
       - PYTHONUNBUFFERED=1
     build:
       context: ..
@@ -53,16 +52,14 @@ services:
       timeout: 10s
       retries: 10
 
-  celery_worker:
+  celery-worker:
     profiles:
       - default
     stdin_open: true
     tty: true
     image: unicef/hct-mis-backend
     env_file:
       - .env
-    environment:
-      - REDIS_INSTANCE=redis:6379
     volumes:
       - ../src:/code/
       - backend-data:/data
@@ -90,8 +87,6 @@ services:
     image: unicef/hct-mis-backend
     env_file:
       - .env
-    environment:
-      - REDIS_INSTANCE=redis:6379
     volumes:
       - ../src:/code/
       - backend-data:/data
@@ -119,8 +114,6 @@ services:
     volumes:
       - db:/var/lib/postgresql/data
       - ./postgres/init:/docker-entrypoint-initdb.d
-    environment:
-      POSTGRES_MULTIPLE_DATABASES: unicef_hct_mis_cashassist,rdi_datahub,mis_datahub,erp_datahub,ca_datahub
     env_file:
       - .env
     ports:
@@ -131,7 +124,6 @@ services:
       timeout: 10s
       retries: 5
 
-
   redis:
     image: redis:4.0.11-alpine3.8
     profiles:
@@ -168,23 +160,28 @@ services:
     volumes:
       - data_es:/usr/share/elasticsearch/data
     ports:
-      - 9200:9200
+      - "9200:9200"
     healthcheck:
       test: [ "CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1" ]
       interval: 10s
       timeout: 10s
       retries: 3
 
   celery-flower:
-    image: johniak/flower:1.6
     profiles:
       - default
-    command: [ "flower", "--broker=redis://redis:6379/0", "--port=5555" ]
-    environment:
-      - FLOWER_AUTH_PROVIDER=""
-      - FLOWER_DEBUG="1"
-      - FLOWER_ADDRESS=0.0.0.0
+    stdin_open: true
+    image: unicef/hct-mis-backend
+    env_file:
+      - .env
+    command: "celery-flower"
+    volumes:
+      - ../src:/code/
+      - backend-data:/data
     ports:
       - "5555:5555"
+    environment:
+      - PYTHONUNBUFFERED=1
     depends_on:
+      - celery-worker
       - redis

development_tools/local_selenium_init.sh

@@ -11,10 +11,6 @@ export POSTGRES_PASS=postgres
 export PGUSER=postgres
 export POSTGRES_HOST_AUTH_METHOD=trust
 export DATABASE_URL=postgis://postgres:postgres@localhost:5432/postgres
-export DATABASE_URL_HUB_MIS=postgis://postgres:postgres@localhost:5432/mis_datahub
-export DATABASE_URL_HUB_CA=postgis://postgres:postgres@localhost:5432/ca_datahub
-export DATABASE_URL_HUB_ERP=postgis://postgres:postgres@localhost:5432/erp_datahub
-export DATABASE_URL_HUB_REGISTRATION=postgis://postgres:postgres@localhost:5432/rdi_datahub
 export POSTGRES_SSL_MODE=off
 export EMAIL_HOST=TBD
 export EMAIL_HOST_USER=TBD
@@ -31,13 +27,12 @@ export KOBO_KF_URL=https://kobo.humanitarianresponse.info
 export KOBO_KC_URL=https://kobo.humanitarianresponse.info
 export ADMIN_PANEL_URL=unicorn
 export PROFILING=off
-export CYPRESS_TESTING=yes
 export DJANGO_ALLOWED_HOST=localhost
 export HCT_MIS_FRONTEND_HOST=localhost:8080
-export REDIS_INSTANCE=localhost:6379
 export PYTHONUNBUFFERED=1
 export CELERY_BROKER_URL=redis://localhost:6379/0
 export CELERY_RESULT_BACKEND=redis://localhost:6379/0
+export CONSTANCE_REDIS_CONNECTION=redis://localhost:6379/0
 export CACHE_LOCATION=redis://localhost:6379/1
 export USE_DUMMY_EXCHANGE_RATES=yes
 export ELASTICSEARCH_HOST=http://localhost:9200

development_tools/postgres/init/init-multiple-db.sh

@@ -16,11 +16,4 @@ EOSQL
 		CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;
 EOSQL
 
-}
-if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
-	echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES"
-	for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do
-		create_user_and_database $db
-	done
-	echo "Multiple databases created"
-fi
+}

docker/Dockerfile

@@ -9,7 +9,7 @@ FROM curl as certs
 RUN curl -o /data/psql-cert.crt -L https://dl.cacerts.digicert.com/DigiCertGlobalRootCA.crt.pem
 
 # Base image
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.12.7-slim-bookworm as base
 
 ARG UID=82
 
@@ -42,7 +42,7 @@ RUN apt-get update \
 ENV PDM_PACKAGES=/packages
 ENV CODE=/code
 ENV PDM_PROJECT=$PDM_PACKAGES
-ENV PYPACKAGES=$PDM_PACKAGES/__pypackages__/3.11
+ENV PYPACKAGES=$PDM_PACKAGES/__pypackages__/3.12
 ENV PYTHONPYCACHEPREFIX=/tmp/pycache \
     PYTHONPATH=$PYPACKAGES/lib:$PYTHONPATH \
     PATH=$PYPACKAGES/bin:$PATH \
@@ -55,7 +55,8 @@ COPY --from=waitforit /data/waitforit /usr/local/bin/waitforit
 # Dist builder image
 FROM base as pdm
 RUN pip install --upgrade pip &&\
-    pip install pdm==2.15.2 &&\
+    pip install pdm==2.19.2 &&\
+    pip install pdm-backend &&\
     pip install setuptools==71.1.0 &&\
     pdm config cache_dir /var/cache/pdm &&\
     pdm config python.use_venv false &&\

docker/entrypoint.sh

@@ -30,6 +30,10 @@ else
     "celery-worker")
       watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A hct_mis_api.apps.core.celery worker -E -l info -Q default,priority
       ;;
+    "celery-flower")
+      waitforit -host=backend -port=8000 --timeout 300 && \
+      celery -A hct_mis_api.apps.core.celery flower --port=5555
+      ;;
     *)
       exec "$@"
       ;;