Merge pull request #63 from unicef/smartenvupgrade

smartenv storages
unicef · Oct 30, 2024 · 8a48bb4 · 8a48bb4
2 parents 95dd8aa + e776911
commit 8a48bb4
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 115 deletions.
diff --git a/compose.yml b/compose.yml
@@ -28,7 +28,7 @@ x-django-env: &django-env
   - AZURE_TENANT_ID=
   - AZURE_CLIENT_KEY=
   - FLOWER_URL=http://flower:5555
-  - STATIC_FILE_STORAGE=django.core.files.storage.FileSystemStorage
+  - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage
 
 services:
   backend:

diff --git a/src/hope_country_report/apps/core/management/commands/inspect_hope.py b/src/hope_country_report/apps/core/management/commands/inspect_hope.py
@@ -199,7 +199,7 @@ def strip_prefix(s):
             yield "from django.contrib.gis.db import models"
 
             yield "from hope_country_report.apps.hope.models._base import HopeModel"
-            yield "from hope_country_report.apps.power_query.storage import HopeStorage"
+            yield "from hope_country_report.apps.core.storage import get_hope_storage"
             basemodel = "HopeModel"
 
             known_models = []
@@ -326,7 +326,7 @@ def strip_prefix(s):
                         field_desc += f", related_name='{_related_name}'"
 
                     if field_type.startswith("ImageField("):
-                        field_desc += "storage=HopeStorage()"
+                        field_desc += "storage=get_hope_storage()"
 
                     if extra_params:
                         if not field_desc.endswith("("):

diff --git a/src/hope_country_report/apps/core/storage.py b/src/hope_country_report/apps/core/storage.py
@@ -0,0 +1,10 @@
+import importlib
+
+from django.conf import settings
+
+
+def get_hope_storage():
+    options = settings.STORAGES["hope"]["OPTIONS"]
+    package_name, klass_name = settings.STORAGES["hope"]["BACKEND"].rsplit(".", 1)
+    module = importlib.import_module(package_name)
+    return getattr(module, klass_name)(**options)
diff --git a/src/hope_country_report/apps/hope/models/_inspect.py b/src/hope_country_report/apps/hope/models/_inspect.py
@@ -8,8 +8,8 @@
 import django.contrib.postgres.fields
 from django.contrib.gis.db import models
 
+from hope_country_report.apps.core.storage import get_hope_storage
 from hope_country_report.apps.hope.models._base import HopeModel
-from hope_country_report.apps.power_query.storage import HopeStorage
 
 
 class BusinessArea(HopeModel):
@@ -23,9 +23,9 @@ class BusinessArea(HopeModel):
     region_name = models.CharField(max_length=8, null=True)
     kobo_username = models.CharField(max_length=255, blank=True, null=True)
     slug = models.CharField(unique=True, max_length=250, null=True)
+    rapid_pro_payment_verification_token = models.CharField(max_length=40, blank=True, null=True)
     rapid_pro_host = models.CharField(max_length=200, blank=True, null=True)
     has_data_sharing_agreement = models.BooleanField(null=True)
-    rapid_pro_payment_verification_token = models.CharField(max_length=40, blank=True, null=True)
     is_split = models.BooleanField(null=True)
     parent = models.ForeignKey(
         "self", on_delete=models.DO_NOTHING, related_name="businessarea_parent", blank=True, null=True
@@ -62,7 +62,6 @@ def __str__(self) -> str:
 
 
 class BusinessareaCountries(HopeModel):
-    id = models.BigAutoField(primary_key=True)
     businessarea = models.ForeignKey(
         BusinessArea, on_delete=models.DO_NOTHING, related_name="businessareacountries_businessarea", null=True
     )
@@ -745,14 +744,14 @@ class Ticketpaymentverificationdetails(HopeModel):
     )
     approve_status = models.BooleanField(null=True)
     new_received_amount = models.DecimalField(max_digits=12, decimal_places=2, blank=True, null=True)
+    new_status = models.CharField(max_length=50, blank=True, null=True)
     payment_verification = models.ForeignKey(
         "Paymentverification",
         on_delete=models.DO_NOTHING,
         related_name="ticketpaymentverificationdetails_payment_verification",
         blank=True,
         null=True,
     )
-    new_status = models.CharField(max_length=50, blank=True, null=True)
     old_received_amount = models.DecimalField(max_digits=12, decimal_places=2, blank=True, null=True)
 
     class Meta:
@@ -764,7 +763,6 @@ class Tenant:
 
 
 class TicketpaymentverificationdetailsPaymentVerificaf7C9(HopeModel):
-    id = models.BigAutoField(primary_key=True)
     ticketpaymentverificationdetails = models.ForeignKey(
         Ticketpaymentverificationdetails,
         on_delete=models.DO_NOTHING,
@@ -928,7 +926,7 @@ class Document(HopeModel):
     created_at = models.DateTimeField(null=True)
     updated_at = models.DateTimeField(null=True)
     document_number = models.CharField(max_length=255, null=True)
-    photo = models.ImageField(storage=HopeStorage(), null=True)
+    photo = models.ImageField(storage=get_hope_storage(), null=True)
     individual = models.ForeignKey(
         "Individual", on_delete=models.DO_NOTHING, related_name="document_individual", null=True
     )
@@ -1061,7 +1059,7 @@ class Household(HopeModel):
     )
     child_hoh = models.BooleanField(blank=True, null=True)
     consent_sharing = models.CharField(max_length=63, null=True)
-    consent_sign = models.ImageField(storage=HopeStorage(), null=True)
+    consent_sign = models.ImageField(storage=get_hope_storage(), null=True)
     deviceid = models.CharField(max_length=250, null=True)
     fchild_hoh = models.BooleanField(blank=True, null=True)
     name_enumerator = models.CharField(max_length=250, null=True)
@@ -1071,6 +1069,7 @@ class Household(HopeModel):
     village = models.CharField(max_length=250, null=True)
     consent = models.BooleanField(blank=True, null=True)
     is_removed = models.BooleanField(null=True)
+    collect_individual_data = models.CharField(max_length=250, null=True)
     currency = models.CharField(max_length=250, null=True)
     female_age_group_18_59_count = models.IntegerField(blank=True, null=True)
     female_age_group_18_59_disabled_count = models.IntegerField(blank=True, null=True)
@@ -1081,12 +1080,11 @@ class Household(HopeModel):
     male_age_group_60_count = models.IntegerField(blank=True, null=True)
     male_age_group_60_disabled_count = models.IntegerField(blank=True, null=True)
     registration_method = models.CharField(max_length=250, null=True)
-    collect_individual_data = models.CharField(max_length=250, null=True)
     unhcr_id = models.CharField(max_length=250, null=True)
     version = models.BigIntegerField(null=True)
-    removed_date = models.DateTimeField(blank=True, null=True)
     withdrawn = models.BooleanField(null=True)
     withdrawn_date = models.DateTimeField(blank=True, null=True)
+    removed_date = models.DateTimeField(blank=True, null=True)
     user_fields = models.JSONField(null=True)
     country = models.ForeignKey(
         Country, on_delete=models.DO_NOTHING, related_name="household_country", blank=True, null=True
@@ -1119,7 +1117,7 @@ class Household(HopeModel):
         Area, on_delete=models.DO_NOTHING, related_name="household_admin4", blank=True, null=True
     )
     zip_code = models.CharField(max_length=12, blank=True, null=True)
-    registration_id = models.TextField(blank=True, null=True)  # This field type is a guess.
+    registration_id = models.TextField(unique=True, blank=True, null=True)  # This field type is a guess.
     copied_from = models.ForeignKey(
         "self", on_delete=models.DO_NOTHING, related_name="household_copied_from", blank=True, null=True
     )
@@ -1157,7 +1155,6 @@ class Tenant:
 
 
 class HouseholdPrograms(HopeModel):
-    id = models.BigAutoField(primary_key=True)
     household = models.ForeignKey(
         Household, on_delete=models.DO_NOTHING, related_name="householdprograms_household", null=True
     )
@@ -1190,7 +1187,7 @@ class Individual(HopeModel):
     created_at = models.DateTimeField(null=True)
     updated_at = models.DateTimeField(null=True)
     individual_id = models.CharField(max_length=255, null=True)
-    photo = models.ImageField(storage=HopeStorage(), null=True)
+    photo = models.ImageField(storage=get_hope_storage(), null=True)
     full_name = models.TextField(null=True)  # This field type is a guess.
     given_name = models.TextField(null=True)  # This field type is a guess.
     middle_name = models.TextField(null=True)  # This field type is a guess.
@@ -1221,10 +1218,10 @@ class Individual(HopeModel):
     first_registration_date = models.DateField(null=True)
     last_registration_date = models.DateField(null=True)
     unicef_id = models.CharField(max_length=255, blank=True, null=True)
+    deduplication_golden_record_status = models.CharField(max_length=50, null=True)
+    deduplication_golden_record_results = models.JSONField(null=True)
     sanction_list_possible_match = models.BooleanField(null=True)
     pregnant = models.BooleanField(blank=True, null=True)
-    deduplication_golden_record_results = models.JSONField(null=True)
-    deduplication_golden_record_status = models.CharField(max_length=50, null=True)
     deduplication_batch_results = models.JSONField(null=True)
     deduplication_batch_status = models.CharField(max_length=50, null=True)
     imported_individual_id = models.UUIDField(blank=True, null=True)
@@ -1242,16 +1239,16 @@ class Individual(HopeModel):
     )
     is_removed = models.BooleanField(null=True)
     version = models.BigIntegerField(null=True)
-    duplicate_date = models.DateTimeField(blank=True, null=True)
-    removed_date = models.DateTimeField(blank=True, null=True)
-    withdrawn_date = models.DateTimeField(blank=True, null=True)
     duplicate = models.BooleanField(null=True)
+    duplicate_date = models.DateTimeField(blank=True, null=True)
     withdrawn = models.BooleanField(null=True)
+    withdrawn_date = models.DateTimeField(blank=True, null=True)
+    removed_date = models.DateTimeField(blank=True, null=True)
     sanction_list_confirmed_match = models.BooleanField(null=True)
     user_fields = models.JSONField(null=True)
     child_hoh = models.BooleanField(null=True)
     fchild_hoh = models.BooleanField(null=True)
-    disability_certificate_picture = models.ImageField(storage=HopeStorage(), blank=True, null=True)
+    disability_certificate_picture = models.ImageField(storage=get_hope_storage(), blank=True, null=True)
     vector_column = models.TextField(blank=True, null=True)  # This field type is a guess.
     phone_no_alternative_valid = models.BooleanField(blank=True, null=True)
     phone_no_valid = models.BooleanField(blank=True, null=True)
@@ -2065,7 +2062,7 @@ class Program(HopeModel):
     name = models.TextField(null=True)  # This field type is a guess.
     status = models.CharField(max_length=10, null=True)
     start_date = models.DateField(null=True)
-    end_date = models.DateField(blank=True, null=True)
+    end_date = models.DateField(null=True)
     description = models.CharField(max_length=255, null=True)
     budget = models.DecimalField(max_digits=11, decimal_places=2, null=True)
     frequency_of_payments = models.CharField(max_length=50, null=True)
@@ -2105,7 +2102,6 @@ def __str__(self) -> str:
 
 
 class ProgramAdminAreas(HopeModel):
-    id = models.BigAutoField(primary_key=True)
     program = models.ForeignKey(
         Program, on_delete=models.DO_NOTHING, related_name="programadminareas_program", null=True
     )
@@ -2120,6 +2116,7 @@ class Tenant:
 
 
 class ProgramCycle(HopeModel):
+    is_removed = models.BooleanField(null=True)
     id = models.UUIDField(primary_key=True)
     created_at = models.DateTimeField(null=True)
     updated_at = models.DateTimeField(null=True)
@@ -2500,9 +2497,7 @@ class TargetPoulation(HopeModel):
         blank=True,
         null=True,
     )
-    program = models.ForeignKey(
-        Program, on_delete=models.DO_NOTHING, related_name="targetpoulation_program", blank=True, null=True
-    )
+    program = models.ForeignKey(Program, on_delete=models.DO_NOTHING, related_name="targetpoulation_program", null=True)
     change_date = models.DateTimeField(blank=True, null=True)
     finalized_at = models.DateTimeField(blank=True, null=True)
     business_area = models.ForeignKey(

diff --git a/src/hope_country_report/apps/power_query/processors.py b/src/hope_country_report/apps/power_query/processors.py
@@ -21,7 +21,6 @@
 from strategy_field.registry import Registry
 from strategy_field.utils import fqn
 
-from hope_country_report.apps.power_query.storage import HopeStorage
 from hope_country_report.apps.power_query.utils import (
     convert_pdf_to_image_pdf,
     get_field_rect,
@@ -30,6 +29,8 @@
     to_dataset,
 )
 
+from ..core.storage import get_hope_storage
+
 logger = logging.getLogger(__name__)
 if TYPE_CHECKING:
     from .models import Dataset, Formatter, ReportTemplate
@@ -318,7 +319,7 @@ def is_special_language_field(self, field_name: str) -> Optional[str]:
         return None
 
     def load_image_from_blob_storage(self, image_path: str) -> BytesIO:
-        with HopeStorage().open(image_path, "rb") as img_file:
+        with get_hope_storage().open(image_path, "rb") as img_file:
             return BytesIO(img_file.read())
 
 

diff --git a/src/hope_country_report/apps/power_query/storage.py b/src/hope_country_report/apps/power_query/storage.py
diff --git a/src/hope_country_report/config/__init__.py b/src/hope_country_report/config/__init__.py
@@ -61,9 +61,24 @@ class Group(Enum):
         "https://django-environ.readthedocs.io/en/latest/types.html#environ-env-db-url",
     ),
     "DEBUG": (bool, False, setting("debug")),
-    "DEFAULT_FILE_STORAGE": (
+    "FILE_STORAGE_DEFAULT": (
         str,
-        "hope_country_report.apps.power_query.storage.MediaStorage",
+        "django.core.files.storage.FileSystemStorage",
+        setting("storages"),
+    ),
+    "FILE_STORAGE_MEDIA": (
+        str,
+        "django.core.files.storage.FileSystemStorage",
+        setting("storages"),
+    ),
+    "FILE_STORAGE_STATIC": (
+        str,
+        "django.contrib.staticfiles.storage.StaticFilesStorage",
+        setting("storages"),
+    ),
+    "FILE_STORAGE_HOPE": (
+        str,
+        "storages.backends.azure_storage.AzureStorage",
         setting("storages"),
     ),
     "EMAIL_BACKEND": (str, "anymail.backends.mailjet.EmailBackend", "Do not change in prod"),
@@ -77,11 +92,6 @@ class Group(Enum):
     "HOPE_AZURE_ACCOUNT_KEY": (str, ""),
     "HOPE_AZURE_AZURE_CONTAINER": (str, ""),
     "HOPE_AZURE_SAS_TOKEN": (str, ""),
-    "HOPE_FILE_STORAGE": (
-        str,
-        "hope_country_report.apps.power_query.storage.HopeStorage",
-        setting("storages"),
-    ),
     "HOST": (str, "http://localhost:8000"),
     "MAILJET_API_KEY": (str, NOT_SET),
     "MAILJET_SECRET_KEY": (str, NOT_SET),
@@ -91,11 +101,6 @@ class Group(Enum):
     "MEDIA_AZURE_ACCOUNT_KEY": (str, ""),
     "MEDIA_AZURE_AZURE_CONTAINER": (str, ""),
     "MEDIA_AZURE_SAS_TOKEN": (str, ""),
-    "MEDIA_FILE_STORAGE": (
-        str,
-        "hope_country_report.apps.power_query.storage.MediaStorage",
-        setting("storages"),
-    ),
     "MEDIA_ROOT": (str, "/tmp/media/", setting("media-root")),
     "MEDIA_URL": (str, "/media/", setting("media-url")),
     "POWER_QUERY_FLOWER_ADDRESS": (str, "http://localhost:5555", "Flower address"),
@@ -117,11 +122,6 @@ class Group(Enum):
         True,
         "https://python-social-auth.readthedocs.io/en/latest/configuration/settings.html",
     ),
-    "STATIC_FILE_STORAGE": (
-        str,
-        "django.core.files.storage.FileSystemStorage",
-        setting("storages"),
-    ),
     "STATIC_ROOT": (str, "/tmp/static/", setting("static-root")),
     "STATIC_URL": (str, "/static/", setting("static-url")),
     "TIME_ZONE": (str, "UTC", setting("std-setting-TIME_ZONE")),

diff --git a/src/hope_country_report/config/fragments/storage.py b/src/hope_country_report/config/fragments/storage.py