diff --git a/src/frontend/package.json b/src/frontend/package.json index 8d10905135..23e1b4b559 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "2.12.1", + "version": "2.13.0", "private": true, "type": "module", "scripts": { diff --git a/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx b/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx index 5f83319824..82c53ad26f 100644 --- a/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx +++ b/src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx @@ -34,7 +34,7 @@ export function RegistrationDetails({ const { t } = useTranslation(); const { data } = useRegistrationDataImportQuery({ variables: { - id: btoa(`RegistrationDataImportNode:${hctId}`), + id: hctId, }, }); if (!data) { diff --git a/src/hct_mis_api/apps/household/models.py b/src/hct_mis_api/apps/household/models.py index 0e1f0feee7..fc95d80aca 100644 --- a/src/hct_mis_api/apps/household/models.py +++ b/src/hct_mis_api/apps/household/models.py @@ -504,7 +504,9 @@ class CollectType(models.TextChoices): org_name_enumerator = models.CharField(max_length=250, blank=True, default=BLANK) village = models.CharField(max_length=250, blank=True, default=BLANK) registration_method = models.CharField(max_length=250, choices=REGISTRATION_METHOD_CHOICES, default=BLANK) - collect_individual_data = models.CharField(max_length=250, choices=COLLECT_TYPES, default=COLLECT_TYPE_UNKNOWN) + collect_individual_data = models.CharField( + max_length=250, choices=COLLECT_TYPES, default=COLLECT_TYPE_UNKNOWN + ) # TODO remove currency = models.CharField(max_length=250, choices=CURRENCY_CHOICES, default=BLANK) unhcr_id = models.CharField(max_length=250, blank=True, default=BLANK, db_index=True) user_fields = JSONField(default=dict, blank=True) diff --git a/src/hct_mis_api/apps/household/services/household_recalculate_data.py b/src/hct_mis_api/apps/household/services/household_recalculate_data.py index f55ef7d4ef..120b4ba968 100644 --- a/src/hct_mis_api/apps/household/services/household_recalculate_data.py +++ b/src/hct_mis_api/apps/household/services/household_recalculate_data.py @@ -6,7 +6,6 @@ from dateutil.relativedelta import relativedelta from hct_mis_api.apps.household.models import ( - COLLECT_TYPE_FULL, COLLECT_TYPE_PARTIAL, DISABLED, FEMALE, @@ -40,10 +39,7 @@ def recalculate_data( ) -> Tuple[Household, List[str]]: household = Household.objects.select_for_update().get(id=household.id) - if ( - household.collect_individual_data not in (COLLECT_TYPE_FULL, COLLECT_TYPE_PARTIAL) - and not household.program.data_collecting_type.recalculate_composition - ): + if not household.program.data_collecting_type.recalculate_composition: return household, [] individuals_to_update = [] diff --git a/src/hct_mis_api/apps/payment/services/payment_gateway.py b/src/hct_mis_api/apps/payment/services/payment_gateway.py index 05c8402441..5096f053b3 100644 --- a/src/hct_mis_api/apps/payment/services/payment_gateway.py +++ b/src/hct_mis_api/apps/payment/services/payment_gateway.py @@ -54,7 +54,7 @@ class PaymentInstructionStatus(Enum): class PaymentInstructionFromDeliveryMechanismPerPaymentPlanSerializer(ReadOnlyModelSerializer): remote_id = serializers.CharField(source="id") - unicef_id = serializers.CharField(source="payment_plan.unicef_id") + external_code = serializers.CharField(source="payment_plan.unicef_id") fsp = serializers.SerializerMethodField() payload = serializers.SerializerMethodField() extra = serializers.SerializerMethodField() @@ -78,7 +78,7 @@ class Meta: model = DeliveryMechanismPerPaymentPlan fields = [ "remote_id", - "unicef_id", + "external_code", "fsp", "payload", "extra", @@ -86,16 +86,16 @@ class Meta: class PaymentInstructionFromSplitSerializer(PaymentInstructionFromDeliveryMechanismPerPaymentPlanSerializer): - unicef_id = serializers.SerializerMethodField() # type: ignore + external_code = serializers.SerializerMethodField() # type: ignore - def get_unicef_id(self, obj: Any) -> str: + def get_external_code(self, obj: Any) -> str: return f"{obj.payment_plan.unicef_id}-{obj.order}" class Meta: model = PaymentPlanSplit fields = [ "remote_id", - "unicef_id", + "external_code", "fsp", "payload", "extra", @@ -210,7 +210,7 @@ def get_transferred_status_based_on_delivery_amount() -> str: @dataclasses.dataclass() class PaymentInstructionData(FlexibleArgumentsDataclassMixin): remote_id: str - unicef_id: str + external_code: str status: str # "DRAFT" fsp: str system: int diff --git a/src/hct_mis_api/apps/program/schema.py b/src/hct_mis_api/apps/program/schema.py index 2127c6070d..16884aa4bb 100644 --- a/src/hct_mis_api/apps/program/schema.py +++ b/src/hct_mis_api/apps/program/schema.py @@ -43,7 +43,7 @@ from hct_mis_api.apps.core.utils import ( chart_filters_decoder, chart_permission_decorator, - decode_id_string, + get_program_id_from_headers, to_choice_object, ) from hct_mis_api.apps.payment.filters import ( @@ -261,13 +261,19 @@ class Query(graphene.ObjectType): is_deduplication_disabled = graphene.Boolean() def resolve_can_run_deduplication(self, info: Any, **kwargs: Any) -> bool: - encoded_program_id = info.context.headers.get("Program") - program = Program.objects.only("biometric_deduplication_enabled").get(id=decode_id_string(encoded_program_id)) + program_id = get_program_id_from_headers(info.context.headers) + if not program_id: + return False + + program = Program.objects.only("biometric_deduplication_enabled").get(id=program_id) return program.biometric_deduplication_enabled def resolve_is_deduplication_disabled(self, info: Any, **kwargs: Any) -> bool: - encoded_program_id = info.context.headers.get("Program") - program = Program.objects.only("id").get(id=decode_id_string(encoded_program_id)) + program_id = get_program_id_from_headers(info.context.headers) + if not program_id: + return True + + program = Program.objects.only("id").get(id=program_id) # deduplication engine in progress is_still_processing = RegistrationDataImport.objects.filter( program=program, diff --git a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py index e4a978724c..ed886b071b 100644 --- a/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py +++ b/src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py @@ -88,6 +88,7 @@ def __init__( super().__init__() def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union[str, File]]: + logger.info(f"Processing image field: {value}") if not self.registration_data_import.pull_pictures: return None if self.attachments is None: @@ -102,6 +103,7 @@ def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union file = File(image_bytes, name=value) if is_flex_field: return default_storage.save(value, file) + logger.info(f"Image field processed: {value}") return file def _handle_geopoint_field(self, value: Any, is_flex_field: bool) -> Point: @@ -221,10 +223,14 @@ def execute( collectors_to_create = defaultdict(list) household_hash_list = [] household_batch_size = 50 + logger.info(f"Processing {len(self.reduced_submissions)} households") + chunk_index = 0 delivery_mechanism_xlsx_fields = PendingDeliveryMechanismData.get_scope_delivery_mechanisms_fields( by="xlsx_field" ) for reduced_submission_chunk in chunks(self.reduced_submissions, household_batch_size): + chunk_index += 1 + logger.info(f"Processing chunk {chunk_index}/{len(self.reduced_submissions) // household_batch_size}") for household in reduced_submission_chunk: # AB#199540 household_hash = calculate_hash_for_kobo_submission(household) diff --git a/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py b/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py index ce6953cb2e..1e2ef642ce 100644 --- a/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py +++ b/src/hct_mis_api/contrib/aurora/services/generic_registration_service.py @@ -115,24 +115,9 @@ def _create_household_dict(cls, data_dict: Dict, mapping_dict: Dict) -> Dict: my_dict.update(cls._create_household_dict(data_dict[key], mapping_dict[key])) # update admin areas values - admin2 = cls.get(data_dict, "admin2_h_c") - admin3 = cls.get(data_dict, "admin3_h_c") - admin4 = cls.get(data_dict, "admin4_h_c") - - my_dict["admin2"] = str(Area.objects.get(p_code=admin2).id) if admin2 else None - my_dict["admin3"] = str(Area.objects.get(p_code=admin3).id) if admin3 else None - my_dict["admin4"] = str(Area.objects.get(p_code=admin4).id) if admin4 else None - - if admin2 and Area.objects.filter(p_code=admin2).exists(): - my_dict["admin1"] = str(Area.objects.get(p_code=admin2).parent.id) - - if admin4 and Area.objects.filter(p_code=admin4).exists(): - my_dict["admin_area"] = str(Area.objects.get(p_code=admin4).id) - elif admin3 and Area.objects.filter(p_code=admin3).exists(): - my_dict["admin_area"] = str(Area.objects.get(p_code=admin3).id) - elif admin2 and Area.objects.filter(p_code=admin2).exists(): - my_dict["admin_area"] = str(Area.objects.get(p_code=admin2).id) - + for key in ["admin1", "admin2", "admin3", "admin4"]: + if key in my_dict and Area.objects.filter(p_code=my_dict[key]).exists(): + my_dict[key] = str(Area.objects.get(p_code=my_dict[key]).id) return my_dict @staticmethod @@ -202,7 +187,6 @@ def create_household_data( flex_fields = household_payload.pop("flex_fields", dict()) flex_fields.update(**self.get_extra_ff(mapping.get("flex_fields", list()), record_data_dict)) - individuals_key = mapping["defaults"].get("individuals_key", "individuals") household_data = { **household_payload, # "flex_registrations_record": record, @@ -215,7 +199,6 @@ def create_household_data( "country": str(Country.objects.get(iso_code2=mapping["defaults"][COUNTRY]).pk), "consent": True, "collect_individual_data": YES, - "size": len(record_data_dict[individuals_key]), "flex_fields": flex_fields, } return self._create_object_and_validate(household_data, PendingHousehold) @@ -233,6 +216,7 @@ def create_individuals( program=household.program, first_registration_date=record.timestamp, last_registration_date=record.timestamp, + detail_id=record.source_id, ) record_data_dict = record.get_data() @@ -352,7 +336,6 @@ def create_household_for_rdi_household(self, record: Any, registration_data_impo mapping, ) - household.size = len(individuals) if head: household.head_of_household = head @@ -365,6 +348,7 @@ def create_household_for_rdi_household(self, record: Any, registration_data_impo individual=sec_collector, household=household, role=ROLE_ALTERNATE ) - household.registration_id = record.source_id + household.registration_id = record.source_id # TODO to be removed + household.detail_id = record.source_id household.save() record.mark_as_imported() diff --git a/tests/unit/apps/household/test_collecting_individual_data.py b/tests/unit/apps/household/test_collecting_individual_data.py deleted file mode 100644 index 54a2daec97..0000000000 --- a/tests/unit/apps/household/test_collecting_individual_data.py +++ /dev/null @@ -1,85 +0,0 @@ -from datetime import date - -from hct_mis_api.apps.account.fixtures import BusinessAreaFactory, UserFactory -from hct_mis_api.apps.core.base_test_case import APITestCase -from hct_mis_api.apps.household.fixtures import create_household_and_individuals -from hct_mis_api.apps.household.models import ( - COLLECT_TYPE_FULL, - COLLECT_TYPE_NONE, - COLLECT_TYPE_PARTIAL, - HEAD, - MALE, - Household, -) -from hct_mis_api.apps.household.services.household_recalculate_data import ( - recalculate_data, -) -from hct_mis_api.apps.registration_data.fixtures import RegistrationDataImportFactory - - -class TestOptionalRecalculationOfIndividuals(APITestCase): - @classmethod - def setUpTestData(cls) -> None: - super().setUpTestData() - cls.user = UserFactory.create() - cls.business_area = BusinessAreaFactory( - code="0060", - name="Afghanistan", - long_name="THE ISLAMIC REPUBLIC OF AFGHANISTAN", - region_code="64", - region_name="SAR", - slug="afghanistan", - has_data_sharing_agreement=True, - ) - cls.registration_data_import = RegistrationDataImportFactory(business_area=cls.business_area) - - def create_hh(self, collect_individual_data: str) -> Household: - household, _ = create_household_and_individuals( - household_data={ - "registration_data_import": self.registration_data_import, - "business_area": self.business_area, - "collect_individual_data": collect_individual_data, - }, - individuals_data=[ - { - "registration_data_import": self.registration_data_import, - "given_name": "Test", - "full_name": "Test Testowski", - "middle_name": "", - "family_name": "Testowski", - "phone_no": "123-123-123", - "phone_no_alternative": "", - "relationship": HEAD, - "sex": MALE, - "birth_date": date.today(), - }, - ], - ) - return household - - def test_recalculating_data_when_flag_is_full(self) -> None: - household = self.create_hh(COLLECT_TYPE_FULL) - self.assertEqual(household.collect_individual_data, COLLECT_TYPE_FULL) - household.female_age_group_0_5_count = 123 - household.save() - recalculate_data(household) - household.refresh_from_db() - self.assertEqual(household.female_age_group_0_5_count, 0) - - def test_recalculating_data_when_flag_is_partial(self) -> None: - household = self.create_hh(COLLECT_TYPE_PARTIAL) - self.assertEqual(household.collect_individual_data, COLLECT_TYPE_PARTIAL) - household.female_age_group_0_5_count = 123 - household.save() - recalculate_data(household) - household.refresh_from_db() - self.assertEqual(household.female_age_group_0_5_count, None) - - def test_recalculating_data_when_flag_is_none(self) -> None: - household = self.create_hh(COLLECT_TYPE_NONE) - self.assertEqual(household.collect_individual_data, COLLECT_TYPE_NONE) - household.female_age_group_0_5_count = 123 - household.save() - recalculate_data(household) - household.refresh_from_db() - self.assertEqual(household.female_age_group_0_5_count, 123) diff --git a/tests/unit/apps/household/test_recalculate_data.py b/tests/unit/apps/household/test_recalculate_data.py index a7c1bba91c..e8c2f502d9 100644 --- a/tests/unit/apps/household/test_recalculate_data.py +++ b/tests/unit/apps/household/test_recalculate_data.py @@ -24,6 +24,7 @@ from hct_mis_api.apps.household.services.household_recalculate_data import ( recalculate_data, ) +from hct_mis_api.apps.program.fixtures import ProgramFactory from hct_mis_api.apps.registration_data.fixtures import RegistrationDataImportFactory @@ -32,12 +33,16 @@ class TestRecalculateData(TestCase): def setUpTestData(cls) -> None: super().setUpTestData() create_afghanistan() - + cls.program = ProgramFactory() + data_collecting_type = cls.program.data_collecting_type + data_collecting_type.recalculate_composition = True + data_collecting_type.save() business_area = BusinessArea.objects.first() registration_data_import = RegistrationDataImportFactory(business_area=business_area) cls.household_data = { "business_area": business_area, + "program": cls.program, "registration_data_import": registration_data_import, "female_age_group_0_5_count": 2, "female_age_group_6_11_count": 1, @@ -76,6 +81,7 @@ def setUpTestData(cls) -> None: "pregnant": True, "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2020-10-29", "%Y-%m-%d")), "physical_disability": "LOT_DIFFICULTY", + "program": cls.program, }, { "registration_data_import": registration_data_import, @@ -86,6 +92,7 @@ def setUpTestData(cls) -> None: "pregnant": True, "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-07-03", "%Y-%m-%d")), "selfcare_disability": "CANNOT_DO", + "program": cls.program, }, { "registration_data_import": registration_data_import, @@ -96,6 +103,7 @@ def setUpTestData(cls) -> None: "pregnant": False, "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-01-11", "%Y-%m-%d")), "memory_disability": "LOT_DIFFICULTY", + "program": cls.program, }, { "registration_data_import": registration_data_import, @@ -111,6 +119,7 @@ def setUpTestData(cls) -> None: "memory_disability": "LOT_DIFFICULTY", "selfcare_disability": "LOT_DIFFICULTY", "comms_disability": "LOT_DIFFICULTY", + "program": cls.program, }, { "registration_data_import": registration_data_import, @@ -121,6 +130,7 @@ def setUpTestData(cls) -> None: "pregnant": False, "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-01-11", "%Y-%m-%d")), "hearing_disability": "CANNOT_DO", + "program": cls.program, }, { "registration_data_import": registration_data_import, @@ -131,6 +141,7 @@ def setUpTestData(cls) -> None: "pregnant": False, "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-01-11", "%Y-%m-%d")), "hearing_disability": "CANNOT_DO", + "program": cls.program, }, { "registration_data_import": registration_data_import, @@ -142,6 +153,7 @@ def setUpTestData(cls) -> None: "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2020-10-29", "%Y-%m-%d")), "memory_disability": "LOT_DIFFICULTY", "comms_disability": "LOT_DIFFICULTY", + "program": cls.program, }, ] diff --git a/tests/unit/apps/registration_datahub/test_generic_registration_service.py b/tests/unit/apps/registration_datahub/test_generic_registration_service.py index cd9bd3127d..25eb77a20f 100644 --- a/tests/unit/apps/registration_datahub/test_generic_registration_service.py +++ b/tests/unit/apps/registration_datahub/test_generic_registration_service.py @@ -257,8 +257,6 @@ def test_import_data_to_datahub( self.assertEqual(PendingIndividualRoleInHousehold.objects.filter(role=ROLE_PRIMARY).count(), 1) self.assertEqual(PendingIndividualRoleInHousehold.objects.filter(role=ROLE_ALTERNATE).count(), 1) - self.assertEqual(pending_household.admin_area.p_code, self.household[0][admin_area_field]) - def test_import_data_to_datahub_household_individual(self) -> None: records = [ Record( diff --git a/tests/unit/apps/registration_datahub/test_rdi_merge.py b/tests/unit/apps/registration_datahub/test_rdi_merge.py index 9e343de178..1f28d65b69 100644 --- a/tests/unit/apps/registration_datahub/test_rdi_merge.py +++ b/tests/unit/apps/registration_datahub/test_rdi_merge.py @@ -29,7 +29,6 @@ from hct_mis_api.apps.household.models import ( BROTHER_SISTER, COLLECT_TYPE_FULL, - COLLECT_TYPE_PARTIAL, COUSIN, HEAD, NON_BENEFICIARY, @@ -240,6 +239,7 @@ def test_merge_rdi_and_recalculation(self) -> None: kobo_submission_uuid="c09130af-6c9c-4dba-8c7f-1b2ff1970d19", kobo_submission_time="2022-02-22T12:22:22", flex_fields={"enumerator_id": 1234567890}, + program=self.rdi.program, ) dct = self.rdi.program.data_collecting_type dct.recalculate_composition = True @@ -508,65 +508,6 @@ def test_merge_rdi_create_collections(self, household_representation_exists: boo self.assertIsNotNone(household.household_collection) self.assertEqual(household.household_collection.households.count(), 2) - @freeze_time("2022-01-01") - def test_merge_rdi_and_recalculation_for_collect_data_partial(self) -> None: - household = PendingHouseholdFactory( - collect_individual_data=COLLECT_TYPE_PARTIAL, - registration_data_import=self.rdi, - ) - dct = self.rdi.program.data_collecting_type - dct.recalculate_composition = True - dct.save() - - self.set_imported_individuals(household) - - household.head_of_household = PendingIndividual.objects.first() - household.save() - - with capture_on_commit_callbacks(execute=True): - RdiMergeTask().execute(self.rdi.pk) - - households = Household.objects.all() - individuals = Individual.objects.all() - - self.assertEqual(1, households.count()) - self.assertEqual(households[0].collect_individual_data, COLLECT_TYPE_PARTIAL) - self.assertEqual(8, individuals.count()) - - household_data = model_to_dict( - households[0], - ( - "female_age_group_0_5_count", - "female_age_group_6_11_count", - "female_age_group_12_17_count", - "female_age_group_18_59_count", - "female_age_group_60_count", - "male_age_group_0_5_count", - "male_age_group_6_11_count", - "male_age_group_12_17_count", - "male_age_group_18_59_count", - "male_age_group_60_count", - "children_count", - "size", - ), - ) - - expected = { - "female_age_group_0_5_count": None, - "female_age_group_6_11_count": None, - "female_age_group_12_17_count": None, - "female_age_group_18_59_count": None, - "female_age_group_60_count": None, - "male_age_group_0_5_count": None, - "male_age_group_6_11_count": None, - "male_age_group_12_17_count": None, - "male_age_group_18_59_count": None, - "male_age_group_60_count": None, - "children_count": None, - "size": None, - } - self.assertEqual(household_data, expected) - def test_merging_external_collector(self) -> None: household = PendingHouseholdFactory( collect_individual_data=COLLECT_TYPE_FULL,