v4.4.0: 💼 Use celery for collection-update long background job (#11)

* Fix collections update request timeout: use Celery!

* Refactor: rename config.py to constants.py

* Update .env.example

* Update .dockerignore

* Update requirements.txt

.dockerignore

@@ -1,12 +1,31 @@
+# python compiled
+*.pyc
+__pycache__/
+
+# ide stuff
+.idea
+.vscode
+
+# node modules
+node_modules/
+
 # images
 *.png
 
-# py notebooks
+# notebooks
 *.ipynb
 
 # git / github stuff
+.git/
 .github/
 *.gitignore
+*.md
+LICENSE
+docs/
+previews/
 
 # misc
 *.log
+*.json
+*.bat
+*.sh

.env.example

@@ -1,16 +1,18 @@
+APPLICATION_ENV=production
 MONGO_URI=changeme
 NATIONAL_DATA_COLLECTION=changeme
-REGIONAL_DATA_COLLECTION=changeme
-PROVINCIAL_DATA_COLLECTION=changeme
 NATIONAL_TRENDS_COLLECTION=changeme
 NATIONAL_SERIES_COLLECTION=changeme
+REGIONAL_DATA_COLLECTION=changeme
 REGIONAL_TRENDS_COLLECTION=changeme
 REGIONAL_SERIES_COLLECTION=changeme
 REGIONAL_BREAKDOWN_COLLECTION=changeme
+PROVINCIAL_DATA_COLLECTION=changeme
 PROVINCIAL_BREAKDOWN_COLLECTION=changeme
 PROVINCIAL_TRENDS_COLLECTION=changeme
 PROVINCIAL_SERIES_COLLECTION=changeme
 VAX_COLLECTION=changeme
 VAX_SUMMARY_COLLECTION=changeme
 GH_WEBHOOK_SECRET=changeme
-SECRET_KEY=changeme
+API_KEY=changeme
+REDIS_URL=changeme

Procfile

@@ -1,2 +1,3 @@
 release: python -m flask create-collections
-web: gunicorn wsgi:app
+web: gunicorn wsgi:app
+worker: celery -A celery_worker.celery worker

app/__init__.py

@@ -4,20 +4,23 @@
 import os
 
 import click
+from celery import Celery
 from dotenv import load_dotenv
 from flask import Flask, request, render_template, send_from_directory
 from flask_babel import Babel
 from flask_compress import Compress
 from flask_pymongo import PyMongo
 from flask_sitemap import Sitemap
 
-from config import LANGUAGES, TRANSLATION_DIRNAME
+from config import config as app_config
+from constants import LANGUAGES, TRANSLATION_DIRNAME
 
 load_dotenv()
 mongo = PyMongo()
 babel = Babel()
 sitemap = Sitemap()
 compress = Compress()
+celery = Celery(__name__)
 
 
 @babel.localeselector
@@ -62,33 +65,30 @@ def favicon():
             "favicon.ico", mimetype="image/vnd.microsoft.icon")
 
 
+def get_environment():
+    """Return app environment"""
+    return os.environ.get('APPLICATION_ENV') or 'development'
+
+
 def create_app():
     """Create the flask application"""
+    env = get_environment()
     app = Flask(__name__)
-    translation_dir = os.path.join(app.root_path, TRANSLATION_DIRNAME)
-    app.config["BABEL_TRANSLATION_DIRECTORIES"] = translation_dir
-    app.config["SECRET_KEY"] = os.environ["SECRET_KEY"]
-    app.config["MONGO_URI"] = os.environ["MONGO_URI"]
-    app.config["SITEMAP_INCLUDE_RULES_WITHOUT_PARAMS"] = True
+    app.config.from_object(app_config[env])
+    app.config["BABEL_TRANSLATION_DIRECTORIES"] = os.path.join(
+        app.root_path, TRANSLATION_DIRNAME)
     compress.init_app(app)
     mongo.init_app(app)
     babel.init_app(app)
     sitemap.init_app(app)
     set_error_handlers(app)
     set_robots_txt_rule(app)
     set_favicon_rule(app)
-
-    @app.after_request
-    def add_header(r):
-        """
-        Add headers to both force latest IE rendering engine or Chrome Frame,
-        and also to cache the rendered page for 10 minutes.
-        """
-        r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-        r.headers["Pragma"] = "no-cache"
-        r.headers["Expires"] = "0"
-        r.headers["Cache-Control"] = "public, max-age=0"
-        return r
+    celery.config_from_object(app.config)
+    celery.conf.update(
+        broker_url=app.config['BROKER_URL'],
+        result_backend=app.config['RESULT_BACKEND']
+    )
 
     from .ui import pandemic, vaccines
     app.register_blueprint(pandemic)
@@ -125,6 +125,18 @@ def add_header(r):
         "vax_summary": create_vax_summary_collection
     }
 
+    @app.after_request
+    def add_header(r):
+        """
+        Add headers to both force latest IE rendering engine or Chrome Frame,
+        and also to cache the rendered page for 10 minutes.
+        """
+        r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+        r.headers["Pragma"] = "no-cache"
+        r.headers["Expires"] = "0"
+        r.headers["Cache-Control"] = "public, max-age=0"
+        return r
+
     @app.cli.command("create-collections")
     def populate_db():
         """Populate all the collections needed on mongoDB"""
@@ -135,7 +147,7 @@ def populate_db():
     @click.argument("collection_type")
     def populate_collection(collection_type):
         """
-        Populate a collection_type ob mongoDB.
+        Populate a collection_type on mongoDB.
         Choose one of the following:
         "national", "regional", "provincial", "latest_regional",
         "latest_provincial", "national_trends", "regional_trends",