v4.5.1: 🖩 Fix % population vax (2nd dose) (#14)

* add dose types

* calculate % population vaccinated using the second dose

* improve on celery configuration to avoid warning

* use hash for record id (vax collections)

* fix update_national_collection response

* update translations

app/__init__.py

@@ -85,10 +85,7 @@ def create_app():
     set_robots_txt_rule(app)
     set_favicon_rule(app)
     celery.config_from_object(app.config)
-    celery.conf.update(
-        broker_url=app.config['BROKER_URL'],
-        result_backend=app.config['RESULT_BACKEND']
-    )
+    celery.conf.update(app.config.get("CELERY_CONFIG", {}))
 
     from .ui import pandemic, vaccines
     app.register_blueprint(pandemic)

app/data/__init__.py

@@ -23,8 +23,9 @@
     VAX_LATEST_UPDATE_KEY, VAX_DATE_FMT, VAX_UPDATE_FMT, VAX_AREA_KEY,
     VAX_AGE_KEY, HEALTHCARE_PERS_KEY, NONHEALTHCARE_PERS_KEY, HFE_GUESTS_KEY,
     OD_TO_PC_MAP, ITALY_POPULATION, URL_VAX_SUMMARY_DATA, VAX_ADMINS_PERC_KEY,
-    ADMINS_DOSES_KEY, DELIVERED_DOSES_KEY, VAX_DATE_KEY, VAX_DAILY_ADMINS_KEY,
-    CHART_DATE_FMT, OVER_80_KEY, POP_KEY
+    ADMINS_DOSES_KEY, DELIVERED_DOSES_KEY, VAX_DATE_KEY, VAX_TOT_ADMINS_KEY,
+    CHART_DATE_FMT, OVER_80_KEY, POP_KEY, VAX_FIRST_DOSE_KEY,
+    VAX_SECOND_DOSE_KEY
 )
 
 DATA_SERIES = [VARS[key]["title"] for key in VARS]
@@ -282,13 +283,14 @@ def enrich_frontend_data(area=None, **data):
 def get_tot_admins(dtype, area=None):
     """
     Return the total of one of the three main vaccine data types
-    allowed_types ('totale', 'prima_dose', 'seconda_dose')
+    allowed_types (VAX_TOT_ADMINS_KEY, VAX_FIRST_DOSE_KEY, VAX_SECOND_DOSE_KEY)
     :param dtype: str: must be in allowed_types
     :param area: str: region
     :return: int: the total administrations of a given data type
         if the data type is in allowed_types else 0
     """
-    allowed_types = ('totale', 'prima_dose', 'seconda_dose')
+    allowed_types = (
+        VAX_TOT_ADMINS_KEY, VAX_FIRST_DOSE_KEY, VAX_SECOND_DOSE_KEY)
     tot_adms = 0
     if dtype in allowed_types:
         if area:
@@ -317,7 +319,12 @@ def get_age_chart_data(area=None):
     """Return age series data"""
     chart_data = {}
     match = {'$match': {VAX_AREA_KEY: area}}
-    group = {'$group': {'_id': f'${VAX_AGE_KEY}', 'tot': {'$sum': '$totale'}}}
+    group = {
+        '$group': {
+            '_id': f'${VAX_AGE_KEY}',
+            'tot': {'$sum': f'${VAX_TOT_ADMINS_KEY}'}
+        }
+    }
     sort = {'$sort': {'_id': 1}}
     try:
         if area is not None:
@@ -399,7 +406,7 @@ def get_region_chart_data(tot_admins=1):
             {
                 '$group': {
                     '_id': f'${VAX_AREA_KEY}',
-                    'tot': {'$sum': '$totale'}
+                    'tot': {'$sum': f'${VAX_TOT_ADMINS_KEY}'}
                 }
             },
             {'$sort': {'tot': -1}}
@@ -476,7 +483,7 @@ def get_admins_timeseries_chart_data():
         data = [{
             'name': OD_TO_PC_MAP[r],
             'data': (
-                    df[df[VAX_AREA_KEY] == r][VAX_DAILY_ADMINS_KEY].cumsum() /
+                    df[df[VAX_AREA_KEY] == r][VAX_SECOND_DOSE_KEY].cumsum() /
                     df[df[VAX_AREA_KEY] == r][POP_KEY] * 100
             ).round(2).to_list()
         } for r in sorted(df[VAX_AREA_KEY].unique())]

app/data/etl.py

@@ -14,7 +14,7 @@
     REGIONS, PROVINCES, DATE_KEY, CHART_DATE_FMT, STATE_KEY,
     NEW_POSITIVE_MA_KEY, VAX_AGE_KEY, M_SEX_KEY, F_SEX_KEY, VAX_DATE_KEY,
     VAX_DATE_FMT, VAX_AREA_KEY, VAX_TYPE_KEY, ITALY_POPULATION, OD_TO_PC_MAP,
-    POP_KEY
+    POP_KEY, VAX_TOT_ADMINS_KEY
 )
 
 COLUMNS_TO_DROP = [STATE_KEY]
@@ -439,14 +439,14 @@ def build_provincial_series(df):
 def augment_vax_df(df):
     """Clean 'fascia_anagrafica key, add 'total' and 'id' to df"""
     df[VAX_AGE_KEY] = df[VAX_AGE_KEY].apply(lambda x: x.strip())
-    df['totale'] = df[M_SEX_KEY] + df[F_SEX_KEY]
+    df[VAX_TOT_ADMINS_KEY] = df[M_SEX_KEY] + df[F_SEX_KEY]
     df['_id'] = (
             df[VAX_DATE_KEY].apply(
                 lambda x: x.strftime(VAX_DATE_FMT)) +
             df[VAX_AREA_KEY] +
             df[VAX_AGE_KEY] +
             df[VAX_TYPE_KEY]
-    )
+    ).apply(lambda x: hash(x))
     return df
 
 
@@ -464,14 +464,15 @@ def augment_summary_vax_df(df):
         for col in reg_df:
             if isinstance(reg_df[col].values[-1], str):
                 reg_df[col].ffill(inplace=True)
-        else:
-            reg_df.fillna(0, inplace=True)
+            else:
+                reg_df.fillna(0, inplace=True)
         out_df = out_df.append(reg_df)
     out_df.reset_index(inplace=True)
     out_df['_id'] = (
             out_df[VAX_DATE_KEY].apply(
                 lambda x: x.strftime(VAX_DATE_FMT)) + out_df[VAX_AREA_KEY]
     )
+    out_df['_id'] = out_df['_id'].apply(lambda x: hash(x))
     out_df[POP_KEY] = out_df[VAX_AREA_KEY].apply(
         lambda x: ITALY_POPULATION[OD_TO_PC_MAP[x]])
     return out_df

app/db/tasks.py

@@ -41,17 +41,16 @@ def update_national_collection():
                 inserted_ids.extend(r.inserted_ids)
                 response["n_inserted_docs"] = len(inserted_ids)
                 response["inserted_ids"] = inserted_ids
-                response["status"] = "ok"
         else:
             msg = f"Filling empty {NAT_DATA_COLL.name}"
             app.logger.warning(msg)
             r = NAT_DATA_COLL.insert_many(df.to_dict(orient='records'))
             inserted_ids.extend(r.inserted_ids)
             response["n_inserted_docs"] = len(inserted_ids)
-            response["status"] = "ok"
             response["msg"] = msg
         msg = f"{len(inserted_ids)} docs updated in {NAT_DATA_COLL.name}"
         app.logger.warning(msg)
+        response["status"] = "ok"
     except Exception as e:
         response["errors"], response["msg"] = f"{e}", f"{e}"
     return response

app/templates/vaccines/charts.html

@@ -94,8 +94,8 @@
         adminsPerRegionCategories = {{ region_chart_data.categories|safe }};
         adminsTimeseriesDates = {{ admins_timeseries_data.dates|safe }};
         adminsTimeseriesData = {{ admins_timeseries_data.data|safe }};
-        adminsTimeseriesTitle = "{{ gettext('Time series of the percentage of people vaccinated') }}";
-        adminsTimeseriesyAxisTitle = "% {{ gettext('Vaccinated') }}";
+        adminsTimeseriesTitle = "{{ gettext('Percentage of people vaccinated') }}";
+        adminsTimeseriesyAxisTitle = "{{ gettext('Population vaccinated') }} [%]";
         adminsTimeseriesSubtitle = {
             text: "{{ gettext('<b>Select one or more regions</b><br>') }}" + subtitle.text,
             html: true