From c6b1e910442e92521c343d273828977d777ed60a Mon Sep 17 00:00:00 2001 From: marekbiczysko Date: Mon, 4 Nov 2024 14:22:09 +0100 Subject: [PATCH 01/12] hotfix_disable_biometrics_schema_for_all_programs --- src/hct_mis_api/apps/program/schema.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hct_mis_api/apps/program/schema.py b/src/hct_mis_api/apps/program/schema.py index 2127c6070d..94199ea827 100644 --- a/src/hct_mis_api/apps/program/schema.py +++ b/src/hct_mis_api/apps/program/schema.py @@ -262,11 +262,17 @@ class Query(graphene.ObjectType): def resolve_can_run_deduplication(self, info: Any, **kwargs: Any) -> bool: encoded_program_id = info.context.headers.get("Program") + if encoded_program_id == "all": + return False + program = Program.objects.only("biometric_deduplication_enabled").get(id=decode_id_string(encoded_program_id)) return program.biometric_deduplication_enabled def resolve_is_deduplication_disabled(self, info: Any, **kwargs: Any) -> bool: encoded_program_id = info.context.headers.get("Program") + if encoded_program_id == "all": + return False + program = Program.objects.only("id").get(id=decode_id_string(encoded_program_id)) # deduplication engine in progress is_still_processing = RegistrationDataImport.objects.filter( From eda7d482d6fc5afad397f895302432840b18e18a Mon Sep 17 00:00:00 2001 From: marekbiczysko Date: Mon, 4 Nov 2024 14:23:34 +0100 Subject: [PATCH 02/12] hotfix_disable_biometrics_schema_for_all_programs --- src/hct_mis_api/apps/program/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hct_mis_api/apps/program/schema.py b/src/hct_mis_api/apps/program/schema.py index 94199ea827..65fae7da75 100644 --- a/src/hct_mis_api/apps/program/schema.py +++ b/src/hct_mis_api/apps/program/schema.py @@ -271,7 +271,7 @@ def resolve_can_run_deduplication(self, info: Any, **kwargs: Any) -> bool: def resolve_is_deduplication_disabled(self, info: Any, **kwargs: Any) -> bool: encoded_program_id = info.context.headers.get("Program") if encoded_program_id == "all": - return False + return True program = Program.objects.only("id").get(id=decode_id_string(encoded_program_id)) # deduplication engine in progress From b309a161b29d847a2c9cc336f5d8cce2249550d2 Mon Sep 17 00:00:00 2001 From: marekbiczysko Date: Mon, 4 Nov 2024 14:39:21 +0100 Subject: [PATCH 03/12] hotfix_disable_biometrics_schema_for_all_programs --- src/hct_mis_api/apps/program/schema.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hct_mis_api/apps/program/schema.py b/src/hct_mis_api/apps/program/schema.py index 65fae7da75..3d9eb08394 100644 --- a/src/hct_mis_api/apps/program/schema.py +++ b/src/hct_mis_api/apps/program/schema.py @@ -43,8 +43,8 @@ from hct_mis_api.apps.core.utils import ( chart_filters_decoder, chart_permission_decorator, - decode_id_string, to_choice_object, + get_program_id_from_headers, ) from hct_mis_api.apps.payment.filters import ( CashPlanFilter, @@ -261,19 +261,19 @@ class Query(graphene.ObjectType): is_deduplication_disabled = graphene.Boolean() def resolve_can_run_deduplication(self, info: Any, **kwargs: Any) -> bool: - encoded_program_id = info.context.headers.get("Program") - if encoded_program_id == "all": + program_id = get_program_id_from_headers(info.context.headers) + if not program_id: return False - program = Program.objects.only("biometric_deduplication_enabled").get(id=decode_id_string(encoded_program_id)) + program = Program.objects.only("biometric_deduplication_enabled").get(id=program_id) return program.biometric_deduplication_enabled def resolve_is_deduplication_disabled(self, info: Any, **kwargs: Any) -> bool: - encoded_program_id = info.context.headers.get("Program") - if encoded_program_id == "all": + program_id = get_program_id_from_headers(info.context.headers) + if not program_id: return True - program = Program.objects.only("id").get(id=decode_id_string(encoded_program_id)) + program = Program.objects.only("id").get(id=program_id) # deduplication engine in progress is_still_processing = RegistrationDataImport.objects.filter( program=program, From 3836afb91d246945d1f0ea9d4ebfd36a6e506a5c Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 5 Nov 2024 15:36:53 +0100 Subject: [PATCH 04/12] AB#220846 Access denied and gray page on access household through rdi --- .../rdi/details/households/RegistrationDetails.tsx | 2 +- .../apps/registration_datahub/tasks/rdi_kobo_create.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx b/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx index c8b8dcc6d1..67af86fa9c 100644 --- a/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx +++ b/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx @@ -34,7 +34,7 @@ export function RegistrationDetails({ const { t } = useTranslation(); const { data } = useRegistrationDataImportQuery({ variables: { - id: btoa(`RegistrationDataImportNode:${hctId}`), + id: hctId, }, }); if (!data) { diff --git a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py index 0190011404..1c83501b95 100644 --- a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py +++ b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py @@ -87,6 +87,7 @@ def __init__( super().__init__() def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union[str, File]]: + logger.info(f"Processing image field: {value}") if not self.registration_data_import.pull_pictures: return None if self.attachments is None: @@ -101,6 +102,7 @@ def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union file = File(image_bytes, name=value) if is_flex_field: return default_storage.save(value, file) + logger.info(f"Image field processed: {value}") return file def _handle_geopoint_field(self, value: Any, is_flex_field: bool) -> Point: @@ -220,7 +222,11 @@ def execute( collectors_to_create = defaultdict(list) household_hash_list = [] household_batch_size = 50 + logger.info(f"Processing {len(self.reduced_submissions)} households") + chunk_index = 0 for reduced_submission_chunk in chunks(self.reduced_submissions, household_batch_size): + chunk_index += 1 + logger.info(f"Processing chunk {chunk_index}/{len(self.reduced_submissions) // household_batch_size}") for household in reduced_submission_chunk: # AB#199540 household_hash = calculate_hash_for_kobo_submission(household) From 01a317a4cbf45264e69f3466cc4997aa425939b7 Mon Sep 17 00:00:00 2001 From: Domenico DiNicola Date: Tue, 5 Nov 2024 15:37:11 +0100 Subject: [PATCH 05/12] generic import: size --- .../contrib/aurora/services/generic_registration_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py b/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py index d414cda68f..4e65fa6217 100644 --- a/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py +++ b/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py @@ -215,9 +215,10 @@ def create_household_data( "country": str(Country.objects.get(iso_code2=mapping["defaults"][COUNTRY]).pk), "consent": True, "collect_individual_data": YES, - "size": len(record_data_dict[individuals_key]), "flex_fields": flex_fields, } + if "size" not in household_data: + household_data["size"] = len(record_data_dict[individuals_key]) return self._create_object_and_validate(household_data, PendingHousehold) def create_individuals( From 71c232bbab969213b85b1a3aa0200ea4c2d1a764 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 5 Nov 2024 15:36:53 +0100 Subject: [PATCH 06/12] AB#220846 Access denied and gray page on access household through rdi --- .../components/rdi/details/households/RegistrationDetails.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx b/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx index c8b8dcc6d1..67af86fa9c 100644 --- a/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx +++ b/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx @@ -34,7 +34,7 @@ export function RegistrationDetails({ const { t } = useTranslation(); const { data } = useRegistrationDataImportQuery({ variables: { - id: btoa(`RegistrationDataImportNode:${hctId}`), + id: hctId, }, }); if (!data) { From 7ba1880b87a800248301a2ef6a96a7b84fd1aee3 Mon Sep 17 00:00:00 2001 From: Domenico DiNicola Date: Tue, 5 Nov 2024 15:45:42 +0100 Subject: [PATCH 07/12] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d84337f4c2..f91edeb1b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ distribution = true [project] name = "hope" -version = "2.12.2" +version = "2.12.3" description = "HCT MIS is UNICEF's humanitarian cash transfer platform." authors = [ { name = "Tivix" }, From c225085337355d411704c10016c4ce30760ec7fd Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 5 Nov 2024 15:45:57 +0100 Subject: [PATCH 08/12] remove it --- .../apps/registration_datahub/tasks/rdi_kobo_create.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py index 1c83501b95..b6a3943d31 100644 --- a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py +++ b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py @@ -222,11 +222,7 @@ def execute( collectors_to_create = defaultdict(list) household_hash_list = [] household_batch_size = 50 - logger.info(f"Processing {len(self.reduced_submissions)} households") - chunk_index = 0 for reduced_submission_chunk in chunks(self.reduced_submissions, household_batch_size): - chunk_index += 1 - logger.info(f"Processing chunk {chunk_index}/{len(self.reduced_submissions) // household_batch_size}") for household in reduced_submission_chunk: # AB#199540 household_hash = calculate_hash_for_kobo_submission(household) From f8b95716ff7d82baee3902208d085ed5a9914e49 Mon Sep 17 00:00:00 2001 From: marekbiczysko Date: Tue, 5 Nov 2024 15:47:08 +0100 Subject: [PATCH 09/12] lint --- src/hct_mis_api/apps/program/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hct_mis_api/apps/program/schema.py b/src/hct_mis_api/apps/program/schema.py index 3d9eb08394..16884aa4bb 100644 --- a/src/hct_mis_api/apps/program/schema.py +++ b/src/hct_mis_api/apps/program/schema.py @@ -43,8 +43,8 @@ from hct_mis_api.apps.core.utils import ( chart_filters_decoder, chart_permission_decorator, - to_choice_object, get_program_id_from_headers, + to_choice_object, ) from hct_mis_api.apps.payment.filters import ( CashPlanFilter, From 5aa40d306cbb096670ed9ab1d2797a118f348979 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 5 Nov 2024 15:47:20 +0100 Subject: [PATCH 10/12] fix --- .../apps/registration_datahub/tasks/rdi_kobo_create.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py index b6a3943d31..0190011404 100644 --- a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py +++ b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py @@ -87,7 +87,6 @@ def __init__( super().__init__() def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union[str, File]]: - logger.info(f"Processing image field: {value}") if not self.registration_data_import.pull_pictures: return None if self.attachments is None: @@ -102,7 +101,6 @@ def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union file = File(image_bytes, name=value) if is_flex_field: return default_storage.save(value, file) - logger.info(f"Image field processed: {value}") return file def _handle_geopoint_field(self, value: Any, is_flex_field: bool) -> Point: From e0d2ff675a9e44f7352fa872366215c689685ed3 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 5 Nov 2024 15:36:53 +0100 Subject: [PATCH 11/12] AB#220846 Access denied and gray page on access household through rdi --- .../apps/registration_datahub/tasks/rdi_kobo_create.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py index 0190011404..1c83501b95 100644 --- a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py +++ b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py @@ -87,6 +87,7 @@ def __init__( super().__init__() def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union[str, File]]: + logger.info(f"Processing image field: {value}") if not self.registration_data_import.pull_pictures: return None if self.attachments is None: @@ -101,6 +102,7 @@ def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union file = File(image_bytes, name=value) if is_flex_field: return default_storage.save(value, file) + logger.info(f"Image field processed: {value}") return file def _handle_geopoint_field(self, value: Any, is_flex_field: bool) -> Point: @@ -220,7 +222,11 @@ def execute( collectors_to_create = defaultdict(list) household_hash_list = [] household_batch_size = 50 + logger.info(f"Processing {len(self.reduced_submissions)} households") + chunk_index = 0 for reduced_submission_chunk in chunks(self.reduced_submissions, household_batch_size): + chunk_index += 1 + logger.info(f"Processing chunk {chunk_index}/{len(self.reduced_submissions) // household_batch_size}") for household in reduced_submission_chunk: # AB#199540 household_hash = calculate_hash_for_kobo_submission(household) From 7871c919838231e56ed3e4604fb233e8d91af57c Mon Sep 17 00:00:00 2001 From: Domenico DiNicola Date: Tue, 5 Nov 2024 16:42:47 +0100 Subject: [PATCH 12/12] fix size --- .../aurora/services/generic_registration_service.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py b/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py index 4e65fa6217..e32cb53369 100644 --- a/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py +++ b/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py @@ -202,7 +202,6 @@ def create_household_data( flex_fields = household_payload.pop("flex_fields", dict()) flex_fields.update(**self.get_extra_ff(mapping.get("flex_fields", list()), record_data_dict)) - individuals_key = mapping["defaults"].get("individuals_key", "individuals") household_data = { **household_payload, # "flex_registrations_record": record, @@ -217,8 +216,6 @@ def create_household_data( "collect_individual_data": YES, "flex_fields": flex_fields, } - if "size" not in household_data: - household_data["size"] = len(record_data_dict[individuals_key]) return self._create_object_and_validate(household_data, PendingHousehold) def create_individuals( @@ -354,7 +351,6 @@ def create_household_for_rdi_household(self, record: Any, registration_data_impo mapping, ) - household.size = len(individuals) if head: household.head_of_household = head @@ -367,6 +363,7 @@ def create_household_for_rdi_household(self, record: Any, registration_data_impo individual=sec_collector, household=household, role=ROLE_ALTERNATE ) - household.registration_id = record.source_id + household.registration_id = record.source_id # TODO to be removed + household.detail_id = record.source_id household.save() record.mark_as_imported()