Staging back

src/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "frontend",
-  "version": "2.12.1",
+  "version": "2.13.0",
   "private": true,
   "type": "module",
   "scripts": {

src/frontend/src/components/rdi/details/households/RegistrationDetails.tsx

@@ -34,7 +34,7 @@ export function RegistrationDetails({
   const { t } = useTranslation();
   const { data } = useRegistrationDataImportQuery({
     variables: {
-      id: btoa(`RegistrationDataImportNode:${hctId}`),
+      id: hctId,
     },
   });
   if (!data) {

src/hct_mis_api/apps/household/models.py

@@ -504,7 +504,9 @@ class CollectType(models.TextChoices):
     org_name_enumerator = models.CharField(max_length=250, blank=True, default=BLANK)
     village = models.CharField(max_length=250, blank=True, default=BLANK)
     registration_method = models.CharField(max_length=250, choices=REGISTRATION_METHOD_CHOICES, default=BLANK)
-    collect_individual_data = models.CharField(max_length=250, choices=COLLECT_TYPES, default=COLLECT_TYPE_UNKNOWN)
+    collect_individual_data = models.CharField(
+        max_length=250, choices=COLLECT_TYPES, default=COLLECT_TYPE_UNKNOWN
+    )  # TODO remove
     currency = models.CharField(max_length=250, choices=CURRENCY_CHOICES, default=BLANK)
     unhcr_id = models.CharField(max_length=250, blank=True, default=BLANK, db_index=True)
     user_fields = JSONField(default=dict, blank=True)

src/hct_mis_api/apps/household/services/household_recalculate_data.py

@@ -6,7 +6,6 @@
 from dateutil.relativedelta import relativedelta
 
 from hct_mis_api.apps.household.models import (
-    COLLECT_TYPE_FULL,
     COLLECT_TYPE_PARTIAL,
     DISABLED,
     FEMALE,
@@ -40,10 +39,7 @@ def recalculate_data(
 ) -> Tuple[Household, List[str]]:
     household = Household.objects.select_for_update().get(id=household.id)
 
-    if (
-        household.collect_individual_data not in (COLLECT_TYPE_FULL, COLLECT_TYPE_PARTIAL)
-        and not household.program.data_collecting_type.recalculate_composition
-    ):
+    if not household.program.data_collecting_type.recalculate_composition:
         return household, []
 
     individuals_to_update = []

src/hct_mis_api/apps/payment/services/payment_gateway.py

@@ -54,7 +54,7 @@ class PaymentInstructionStatus(Enum):
 
 class PaymentInstructionFromDeliveryMechanismPerPaymentPlanSerializer(ReadOnlyModelSerializer):
     remote_id = serializers.CharField(source="id")
-    unicef_id = serializers.CharField(source="payment_plan.unicef_id")
+    external_code = serializers.CharField(source="payment_plan.unicef_id")
     fsp = serializers.SerializerMethodField()
     payload = serializers.SerializerMethodField()
     extra = serializers.SerializerMethodField()
@@ -78,24 +78,24 @@ class Meta:
         model = DeliveryMechanismPerPaymentPlan
         fields = [
             "remote_id",
-            "unicef_id",
+            "external_code",
             "fsp",
             "payload",
             "extra",
         ]
 
 
 class PaymentInstructionFromSplitSerializer(PaymentInstructionFromDeliveryMechanismPerPaymentPlanSerializer):
-    unicef_id = serializers.SerializerMethodField()  # type: ignore
+    external_code = serializers.SerializerMethodField()  # type: ignore
 
-    def get_unicef_id(self, obj: Any) -> str:
+    def get_external_code(self, obj: Any) -> str:
         return f"{obj.payment_plan.unicef_id}-{obj.order}"
 
     class Meta:
         model = PaymentPlanSplit
         fields = [
             "remote_id",
-            "unicef_id",
+            "external_code",
             "fsp",
             "payload",
             "extra",
@@ -210,7 +210,7 @@ def get_transferred_status_based_on_delivery_amount() -> str:
 @dataclasses.dataclass()
 class PaymentInstructionData(FlexibleArgumentsDataclassMixin):
     remote_id: str
-    unicef_id: str
+    external_code: str
     status: str  # "DRAFT"
     fsp: str
     system: int

src/hct_mis_api/apps/program/schema.py

@@ -43,7 +43,7 @@
 from hct_mis_api.apps.core.utils import (
     chart_filters_decoder,
     chart_permission_decorator,
-    decode_id_string,
+    get_program_id_from_headers,
     to_choice_object,
 )
 from hct_mis_api.apps.payment.filters import (
@@ -261,13 +261,19 @@ class Query(graphene.ObjectType):
     is_deduplication_disabled = graphene.Boolean()
 
     def resolve_can_run_deduplication(self, info: Any, **kwargs: Any) -> bool:
-        encoded_program_id = info.context.headers.get("Program")
-        program = Program.objects.only("biometric_deduplication_enabled").get(id=decode_id_string(encoded_program_id))
+        program_id = get_program_id_from_headers(info.context.headers)
+        if not program_id:
+            return False
+
+        program = Program.objects.only("biometric_deduplication_enabled").get(id=program_id)
         return program.biometric_deduplication_enabled
 
     def resolve_is_deduplication_disabled(self, info: Any, **kwargs: Any) -> bool:
-        encoded_program_id = info.context.headers.get("Program")
-        program = Program.objects.only("id").get(id=decode_id_string(encoded_program_id))
+        program_id = get_program_id_from_headers(info.context.headers)
+        if not program_id:
+            return True
+
+        program = Program.objects.only("id").get(id=program_id)
         # deduplication engine in progress
         is_still_processing = RegistrationDataImport.objects.filter(
             program=program,

src/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py

@@ -88,6 +88,7 @@ def __init__(
         super().__init__()
 
     def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union[str, File]]:
+        logger.info(f"Processing image field: {value}")
         if not self.registration_data_import.pull_pictures:
             return None
         if self.attachments is None:
@@ -102,6 +103,7 @@ def _handle_image_field(self, value: Any, is_flex_field: bool) -> Optional[Union
         file = File(image_bytes, name=value)
         if is_flex_field:
             return default_storage.save(value, file)
+        logger.info(f"Image field processed: {value}")
         return file
 
     def _handle_geopoint_field(self, value: Any, is_flex_field: bool) -> Point:
@@ -221,10 +223,14 @@ def execute(
         collectors_to_create = defaultdict(list)
         household_hash_list = []
         household_batch_size = 50
+        logger.info(f"Processing {len(self.reduced_submissions)} households")
+        chunk_index = 0
         delivery_mechanism_xlsx_fields = PendingDeliveryMechanismData.get_scope_delivery_mechanisms_fields(
             by="xlsx_field"
         )
         for reduced_submission_chunk in chunks(self.reduced_submissions, household_batch_size):
+            chunk_index += 1
+            logger.info(f"Processing chunk {chunk_index}/{len(self.reduced_submissions) // household_batch_size}")
             for household in reduced_submission_chunk:
                 # AB#199540
                 household_hash = calculate_hash_for_kobo_submission(household)

src/hct_mis_api/contrib/aurora/services/generic_registration_service.py

@@ -115,24 +115,9 @@ def _create_household_dict(cls, data_dict: Dict, mapping_dict: Dict) -> Dict:
                     my_dict.update(cls._create_household_dict(data_dict[key], mapping_dict[key]))
 
         # update admin areas values
-        admin2 = cls.get(data_dict, "admin2_h_c")
-        admin3 = cls.get(data_dict, "admin3_h_c")
-        admin4 = cls.get(data_dict, "admin4_h_c")
-
-        my_dict["admin2"] = str(Area.objects.get(p_code=admin2).id) if admin2 else None
-        my_dict["admin3"] = str(Area.objects.get(p_code=admin3).id) if admin3 else None
-        my_dict["admin4"] = str(Area.objects.get(p_code=admin4).id) if admin4 else None
-
-        if admin2 and Area.objects.filter(p_code=admin2).exists():
-            my_dict["admin1"] = str(Area.objects.get(p_code=admin2).parent.id)
-
-        if admin4 and Area.objects.filter(p_code=admin4).exists():
-            my_dict["admin_area"] = str(Area.objects.get(p_code=admin4).id)
-        elif admin3 and Area.objects.filter(p_code=admin3).exists():
-            my_dict["admin_area"] = str(Area.objects.get(p_code=admin3).id)
-        elif admin2 and Area.objects.filter(p_code=admin2).exists():
-            my_dict["admin_area"] = str(Area.objects.get(p_code=admin2).id)
-
+        for key in ["admin1", "admin2", "admin3", "admin4"]:
+            if key in my_dict and Area.objects.filter(p_code=my_dict[key]).exists():
+                my_dict[key] = str(Area.objects.get(p_code=my_dict[key]).id)
         return my_dict
 
     @staticmethod
@@ -202,7 +187,6 @@ def create_household_data(
         flex_fields = household_payload.pop("flex_fields", dict())
 
         flex_fields.update(**self.get_extra_ff(mapping.get("flex_fields", list()), record_data_dict))
-        individuals_key = mapping["defaults"].get("individuals_key", "individuals")
         household_data = {
             **household_payload,
             # "flex_registrations_record": record,
@@ -215,7 +199,6 @@ def create_household_data(
             "country": str(Country.objects.get(iso_code2=mapping["defaults"][COUNTRY]).pk),
             "consent": True,
             "collect_individual_data": YES,
-            "size": len(record_data_dict[individuals_key]),
             "flex_fields": flex_fields,
         }
         return self._create_object_and_validate(household_data, PendingHousehold)
@@ -233,6 +216,7 @@ def create_individuals(
             program=household.program,
             first_registration_date=record.timestamp,
             last_registration_date=record.timestamp,
+            detail_id=record.source_id,
         )
 
         record_data_dict = record.get_data()
@@ -352,7 +336,6 @@ def create_household_for_rdi_household(self, record: Any, registration_data_impo
             mapping,
         )
 
-        household.size = len(individuals)
         if head:
             household.head_of_household = head
 
@@ -365,6 +348,7 @@ def create_household_for_rdi_household(self, record: Any, registration_data_impo
                 individual=sec_collector, household=household, role=ROLE_ALTERNATE
             )
 
-        household.registration_id = record.source_id
+        household.registration_id = record.source_id  # TODO to be removed
+        household.detail_id = record.source_id
         household.save()
         record.mark_as_imported()

tests/unit/apps/household/test_recalculate_data.py

@@ -24,6 +24,7 @@
 from hct_mis_api.apps.household.services.household_recalculate_data import (
     recalculate_data,
 )
+from hct_mis_api.apps.program.fixtures import ProgramFactory
 from hct_mis_api.apps.registration_data.fixtures import RegistrationDataImportFactory
 
 
@@ -32,12 +33,16 @@ class TestRecalculateData(TestCase):
     def setUpTestData(cls) -> None:
         super().setUpTestData()
         create_afghanistan()
-
+        cls.program = ProgramFactory()
+        data_collecting_type = cls.program.data_collecting_type
+        data_collecting_type.recalculate_composition = True
+        data_collecting_type.save()
         business_area = BusinessArea.objects.first()
         registration_data_import = RegistrationDataImportFactory(business_area=business_area)
 
         cls.household_data = {
             "business_area": business_area,
+            "program": cls.program,
             "registration_data_import": registration_data_import,
             "female_age_group_0_5_count": 2,
             "female_age_group_6_11_count": 1,
@@ -76,6 +81,7 @@ def setUpTestData(cls) -> None:
                 "pregnant": True,
                 "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2020-10-29", "%Y-%m-%d")),
                 "physical_disability": "LOT_DIFFICULTY",
+                "program": cls.program,
             },
             {
                 "registration_data_import": registration_data_import,
@@ -86,6 +92,7 @@ def setUpTestData(cls) -> None:
                 "pregnant": True,
                 "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-07-03", "%Y-%m-%d")),
                 "selfcare_disability": "CANNOT_DO",
+                "program": cls.program,
             },
             {
                 "registration_data_import": registration_data_import,
@@ -96,6 +103,7 @@ def setUpTestData(cls) -> None:
                 "pregnant": False,
                 "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-01-11", "%Y-%m-%d")),
                 "memory_disability": "LOT_DIFFICULTY",
+                "program": cls.program,
             },
             {
                 "registration_data_import": registration_data_import,
@@ -111,6 +119,7 @@ def setUpTestData(cls) -> None:
                 "memory_disability": "LOT_DIFFICULTY",
                 "selfcare_disability": "LOT_DIFFICULTY",
                 "comms_disability": "LOT_DIFFICULTY",
+                "program": cls.program,
             },
             {
                 "registration_data_import": registration_data_import,
@@ -121,6 +130,7 @@ def setUpTestData(cls) -> None:
                 "pregnant": False,
                 "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-01-11", "%Y-%m-%d")),
                 "hearing_disability": "CANNOT_DO",
+                "program": cls.program,
             },
             {
                 "registration_data_import": registration_data_import,
@@ -131,6 +141,7 @@ def setUpTestData(cls) -> None:
                 "pregnant": False,
                 "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2021-01-11", "%Y-%m-%d")),
                 "hearing_disability": "CANNOT_DO",
+                "program": cls.program,
             },
             {
                 "registration_data_import": registration_data_import,
@@ -142,6 +153,7 @@ def setUpTestData(cls) -> None:
                 "first_registration_date": timezone.make_aware(datetime.datetime.strptime("2020-10-29", "%Y-%m-%d")),
                 "memory_disability": "LOT_DIFFICULTY",
                 "comms_disability": "LOT_DIFFICULTY",
+                "program": cls.program,
             },
         ]