From d1f875d43e7a62ca2e2bf8bf22b3bf0cf5904e52 Mon Sep 17 00:00:00 2001 From: Allan Stockman Rugano Date: Wed, 13 Nov 2024 06:22:38 +0100 Subject: [PATCH 1/2] Updates for missing data --- src/hct_mis_api/apps/dashboard/services.py | 73 +++++++++++-------- .../templates/dashboard/dashboard.html | 20 ++--- 2 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/hct_mis_api/apps/dashboard/services.py b/src/hct_mis_api/apps/dashboard/services.py index 2c8ce5d88c..7acbeeb205 100644 --- a/src/hct_mis_api/apps/dashboard/services.py +++ b/src/hct_mis_api/apps/dashboard/services.py @@ -4,8 +4,8 @@ from typing import Any, Dict, Optional from django.core.cache import cache -from django.db.models import Count, F, Sum -from django.db.models.functions import ExtractMonth, ExtractYear +from django.db.models import Count, F, Sum, Value +from django.db.models.functions import Coalesce, ExtractMonth, ExtractYear from rest_framework.utils.serializer_helpers import ReturnDict @@ -16,16 +16,16 @@ CACHE_TIMEOUT = 60 * 60 * 24 # 24 hours pwdSum = Sum( - F("household__female_age_group_0_5_disabled_count") - + F("household__female_age_group_6_11_disabled_count") - + F("household__female_age_group_12_17_disabled_count") - + F("household__female_age_group_18_59_disabled_count") - + F("household__female_age_group_60_disabled_count") - + F("household__male_age_group_0_5_disabled_count") - + F("household__male_age_group_6_11_disabled_count") - + F("household__male_age_group_12_17_disabled_count") - + F("household__male_age_group_18_59_disabled_count") - + F("household__male_age_group_60_disabled_count"), + Coalesce(F("household__female_age_group_0_5_disabled_count"), 0) + + Coalesce(F("household__female_age_group_6_11_disabled_count"), 0) + + Coalesce(F("household__female_age_group_12_17_disabled_count"), 0) + + Coalesce(F("household__female_age_group_18_59_disabled_count"), 0) + + Coalesce(F("household__female_age_group_60_disabled_count"), 0) + + Coalesce(F("household__male_age_group_0_5_disabled_count"), 0) + + Coalesce(F("household__male_age_group_6_11_disabled_count"), 0) + + Coalesce(F("household__male_age_group_12_17_disabled_count"), 0) + + Coalesce(F("household__male_age_group_18_59_disabled_count"), 0) + + Coalesce(F("household__male_age_group_60_disabled_count"), 0), default=0, ) @@ -73,16 +73,23 @@ def refresh_data(cls, business_area_slug: str) -> ReturnDict: for area in list_country: payments_aggregated = ( Payment.objects.using("read_only") - .select_related("business_area", "household", "program") - .filter(business_area=area) + .select_related( + "business_area", + "household", + "program", + "household__admin1", + "financial_service_provider", + "delivery_type", + ) + .filter(business_area=area, household__is_removed=False) .annotate( - month=ExtractMonth("delivery_date"), - year=ExtractYear("delivery_date"), - programs=F("household__program__name"), + year=ExtractYear(Coalesce("delivery_date", "entitlement_date", "status_date")), + month=ExtractMonth(Coalesce("delivery_date", "entitlement_date", "status_date")), + programs=Coalesce(F("household__program__name"), Value("Unknown")), sectors=F("household__program__sector"), - admin1=F("household__admin1__name"), - fsp=F("financial_service_provider__name"), - delivery_types=F("delivery_type__name"), + admin1=Coalesce(F("household__admin1__name"), Value("Unknown")), + fsp=Coalesce(F("financial_service_provider__name"), Value("Unknown")), + delivery_types=Coalesce(F("delivery_type__name"), F("delivery_type_choice")), ) .distinct() .values( @@ -96,8 +103,8 @@ def refresh_data(cls, business_area_slug: str) -> ReturnDict: "delivery_types", ) .annotate( - total_usd=Sum("delivered_quantity_usd"), - total_quantity=Sum("delivered_quantity"), + total_usd=Sum("delivered_quantity_usd", default=0), + total_quantity=Sum("delivered_quantity", default=0), total_payments=Count("id", distinct=True), individuals=Sum("household__size"), households=Count("household", distinct=True), @@ -108,16 +115,18 @@ def refresh_data(cls, business_area_slug: str) -> ReturnDict: payment_records_aggregated = ( PaymentRecord.objects.using("read_only") - .select_related("business_area", "household", "program") - .filter(business_area=area) + .select_related( + "business_area", "household", "program", "household__admin1", "service_provider", "delivery_type" + ) + .filter(business_area=area, household__is_removed=False) .annotate( - month=ExtractMonth("delivery_date"), - year=ExtractYear("delivery_date"), - programs=F("household__program__name"), + year=ExtractYear(Coalesce("delivery_date", "status_date")), + month=ExtractMonth(Coalesce("delivery_date", "status_date")), + programs=Coalesce(F("household__program__name"), Value("Unknown")), sectors=F("household__program__sector"), - admin1=F("household__admin1__name"), - fsp=F("service_provider__short_name"), - delivery_types=F("delivery_type__name"), + admin1=Coalesce(F("household__admin1__name"), Value("Unknown")), + fsp=Coalesce(F("service_provider__short_name"), Value("Unknown")), + delivery_types=Coalesce(F("delivery_type__name"), F("delivery_type_choice")), ) .distinct() .values( @@ -131,8 +140,8 @@ def refresh_data(cls, business_area_slug: str) -> ReturnDict: "delivery_types", ) .annotate( - total_usd=Sum("delivered_quantity_usd"), - total_quantity=Sum("delivered_quantity"), + total_usd=Sum("delivered_quantity_usd", default=0), + total_quantity=Sum("delivered_quantity", default=0), total_payments=Count("id", distinct=True), individuals=Sum("household__size"), households=Count("household", distinct=True), diff --git a/src/hct_mis_api/apps/dashboard/templates/dashboard/dashboard.html b/src/hct_mis_api/apps/dashboard/templates/dashboard/dashboard.html index 4ace570aee..4ad547c620 100644 --- a/src/hct_mis_api/apps/dashboard/templates/dashboard/dashboard.html +++ b/src/hct_mis_api/apps/dashboard/templates/dashboard/dashboard.html @@ -227,7 +227,10 @@

const deliveryGroup = deliveryDim.group().reduceSum(d => d.delivered_quantity_usd); const sectorGroup = sectorDim.group().reduceSum(d => d.delivered_quantity_usd); const volumeGroup = volumeDim.group().reduceSum(d => d.delivered_quantity_usd); - const admin1Group = admin1Dim.group().reduceSum(d => d.delivered_quantity_usd); + const admin1Group = admin1Dim.group().reduceSum(d => d.delivered_quantity_usd).order((d) => -d); + const topAdmin1Group = { + all: () => admin1Group.all().slice(0, 20) // Top 20 admin1 + }; const monthGroup = monthDim.group().reduceSum(d => d.delivered_quantity_usd); // Define charts @@ -286,15 +289,14 @@

.title(d => `${d.key}: ${numberFormatter(d.value)} USD`) .ordering(d => monthOrder[d.key]); - const admin1Chart = dc.pieChart("#payments-by-admin1") + const admin1Chart = dc.rowChart("#payments-by-admin1") .dimension(admin1Dim) - .group(admin1Group) - .radius(280) - .innerRadius(40) - .renderLabel(true) - .useViewBoxResizing(true) - .label(d => `${d.key} ${(d.value / admin1Group.all().reduce((sum, g) => sum + g.value, 0) * 100).toFixed(0)}%`) - .title(d => `${d.key}: ${numberFormatter(d.value)} USD`); + .group(topAdmin1Group) + .elasticX(true) + .height(400) + .label(d => `${d.key}: ${numberFormatter(d.value)} USD (${(d.value / admin1Group.all().reduce((sum, g) => sum + g.value, 0) * 100).toFixed(0)}%)`) + .title(d => `${d.key}: ${numberFormatter(d.value)} USD`) + .xAxis().ticks(5); function updateTopMetrics() { From 0a21ca8c6876b2a0acedf8dae468091ada7e18eb Mon Sep 17 00:00:00 2001 From: Allan Stockman Rugano Date: Wed, 13 Nov 2024 06:29:28 +0100 Subject: [PATCH 2/2] More precise --- src/hct_mis_api/apps/dashboard/services.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/hct_mis_api/apps/dashboard/services.py b/src/hct_mis_api/apps/dashboard/services.py index 7acbeeb205..672307b6e7 100644 --- a/src/hct_mis_api/apps/dashboard/services.py +++ b/src/hct_mis_api/apps/dashboard/services.py @@ -85,10 +85,10 @@ def refresh_data(cls, business_area_slug: str) -> ReturnDict: .annotate( year=ExtractYear(Coalesce("delivery_date", "entitlement_date", "status_date")), month=ExtractMonth(Coalesce("delivery_date", "entitlement_date", "status_date")), - programs=Coalesce(F("household__program__name"), Value("Unknown")), - sectors=F("household__program__sector"), - admin1=Coalesce(F("household__admin1__name"), Value("Unknown")), - fsp=Coalesce(F("financial_service_provider__name"), Value("Unknown")), + programs=Coalesce(F("household__program__name"), Value("Unknown program")), + sectors=Coalesce(F("household__program__sector"), Value("Unknown sector")), + admin1=Coalesce(F("household__admin1__name"), Value("Unknown admin1")), + fsp=Coalesce(F("financial_service_provider__name"), Value("Unknown fsp")), delivery_types=Coalesce(F("delivery_type__name"), F("delivery_type_choice")), ) .distinct() @@ -122,10 +122,10 @@ def refresh_data(cls, business_area_slug: str) -> ReturnDict: .annotate( year=ExtractYear(Coalesce("delivery_date", "status_date")), month=ExtractMonth(Coalesce("delivery_date", "status_date")), - programs=Coalesce(F("household__program__name"), Value("Unknown")), - sectors=F("household__program__sector"), - admin1=Coalesce(F("household__admin1__name"), Value("Unknown")), - fsp=Coalesce(F("service_provider__short_name"), Value("Unknown")), + programs=Coalesce(F("household__program__name"), Value("Unknown program")), + sectors=Coalesce(F("household__program__sector"), Value("Unknown sector")), + admin1=Coalesce(F("household__admin1__name"), Value("Unknown admin1")), + fsp=Coalesce(F("service_provider__short_name"), Value("Unknown fsp")), delivery_types=Coalesce(F("delivery_type__name"), F("delivery_type_choice")), ) .distinct()