Merged with release/2.0.11

.env_example

@@ -5,6 +5,8 @@ DJANGO_SECRET_KEY=
 PROJECT_FULL_NAME=gigamaps
 PROJECT_SHORT_NAME=gigamaps
 
+GIGAMAPS_LOG_LEVEL = DEBUG
+
 [email protected]
 SUPPORT_PHONE_NUMBER=1234567890
 
@@ -60,7 +62,6 @@ CACHE_CONTROL_MAX_AGE_FOR_FE=14400
 
 API_KEY_ADMIN_DASHBOARD_URL=http://localhost:9500/admin/api-keys
 
-
 EMAIL_URL=
 SERVER_EMAIL_SIGNATURE=Gigamaps
 [email protected]

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 120
 max-complexity = 30
-exclude = proco_data_migrations, migrations, config/settings, manage.py, Exceptions.py, venv, dailycheckapp_contact, realtime_dailycheckapp, realtime_unicef
+exclude = proco_data_migrations, migrations, config/settings, manage.py, Exceptions.py, venv
 ignore =
     ; PyFlakes errors
     ; F405 name may be undefined, or defined from star imports: module

Pipfile

@@ -79,6 +79,8 @@ djangorestframework-jwt = "==1.11.0"
 jsonfield = "==2.0.2"
 azure-search-documents = "==11.3.0"
 django-prometheus = "==2.2.0"
+celery-redbeat = "==2.2.0"
+flower = "==2.0.1"
 
 [requires]
 python_version = "3.8"

celery-dev.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -ex
+
+# export environment variables to make them available in ssh session
+for var in $(compgen -e); do
+    echo "export $var=${!var}" >> /etc/profile
+done
+
+export FLASK_APP=hello.py
+pipenv run python -m flask run --host 0.0.0.0 --port 8000 &
+
+if $ENABLED_FLOWER_METRICS; then
+    echo "Starting worker ..."
+    pipenv run celery --app=proco.taskapp worker --concurrency=2 --time-limit=300 --soft-time-limit=300 $* &
+
+    echo "Starting flower ..."
+    pipenv run celery --app=proco.taskapp flower
+else
+    echo "Starting worker ..."
+    pipenv run celery --app=proco.taskapp worker --concurrency=2 --time-limit=300 --soft-time-limit=300 $*
+fi

celery.sh

@@ -14,4 +14,14 @@ pipenv run python -m flask run --host 0.0.0.0 --port 8000 &
 
 # pipenv run celery -A proco.taskapp worker $*
 # --logfile=/code/celeryd-%n.log --loglevel=DEBUG
-pipenv run celery --app=proco.taskapp worker --concurrency=3 --time-limit=300 --soft-time-limit=60 $*
+
+if $ENABLED_FLOWER_METRICS; then
+    echo "Starting worker ..."
+    pipenv run celery --app=proco.taskapp worker --concurrency=3 --time-limit=300 --soft-time-limit=60 $* &
+
+    echo "Starting flower ..."
+    pipenv run celery --app=proco.taskapp flower
+else
+    echo "Starting worker ..."
+    pipenv run celery --app=proco.taskapp worker --concurrency=3 --time-limit=300 --soft-time-limit=60 $*
+fi

celerybeat.sh

@@ -14,4 +14,4 @@ pipenv run python -m flask run --host 0.0.0.0 --port 8000 &
 
 # pipenv run celery -A proco.taskapp beat $*
 # --logfile=/code/celeryd-%n.log --loglevel=DEBUG
-pipenv run celery --app=proco.taskapp beat $*
+pipenv run celery --app=proco.taskapp beat --scheduler=redbeat.RedBeatScheduler $*

config/settings/base.py

@@ -1,3 +1,6 @@
+import json
+import os
+import sys
 import warnings
 
 import environ
@@ -34,7 +37,7 @@
 # --------------------------------------------------------------------------
 
 DJANGO_APPS = [
-    'config.apps.CustomAdminConfig',
+    'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
@@ -73,12 +76,9 @@
     'proco.locations',
     'proco.connection_statistics',
     'proco.contact',
-    'proco.dailycheckapp_contact',
     'proco.background',
-    'proco.realtime_unicef',
-    'proco.realtime_dailycheckapp',
     'proco.proco_data_migrations',
-    'proco.data_sources'
+    'proco.data_sources',
 ]
 
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
@@ -342,15 +342,13 @@
     'schools',
     'background',
     'contact',
-    'dailycheckapp_contact',
     'accounts',
     'data_sources',
 )
 
 RANDOM_SCHOOLS_DEFAULT_AMOUNT = env('RANDOM_SCHOOLS_DEFAULT_AMOUNT', default=20000)
 
 CONTACT_MANAGERS = env.list('CONTACT_MANAGERS', default=['[email protected]'])
-DAILYCHECKAPP_CONTACT_MANAGERS = env.list('DAILYCHECKAPP_CONTACT_MANAGERS', default=['[email protected]'])
 
 CONSTANCE_REDIS_CONNECTION = env('REDIS_URL', default='redis://localhost:6379/0')
 CONSTANCE_ADDITIONAL_FIELDS = {
@@ -361,7 +359,6 @@
 }
 CONSTANCE_CONFIG = {
     'CONTACT_EMAIL': (env('CONTACT_EMAIL', default=''), 'Email to receive contact messages', 'email_input'),
-    'DAILYCHECKAPP_CONTACT_EMAIL': ('', 'Email to receive dailycheckapp_contact messages', 'email_input'),
 }
 
 # Cache control headers
@@ -436,4 +433,48 @@
 
 INVALIDATE_CACHE_HARD = env('INVALIDATE_CACHE_HARD', default='false')
 
+with open(os.path.join(BASE_DIR, 'proco', 'core', 'resources', 'filters.json')) as filters_json_file:
+    FILTERS_DATA = json.load(filters_json_file)
+
 # DATABASE_ROUTERS = ["proco.utils.read_db_router.StandbyRouter"]
+
+GIGAMAPS_LOG_LEVEL = env('GIGAMAPS_LOG_LEVEL', default='INFO')
+
+# LOGGING
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse',
+        },
+        'require_debug_true': {
+            '()': 'django.utils.log.RequireDebugTrue',
+        },
+        'hostname_filter': {
+            '()': 'proco.core.filters.HostInfoFilter',
+        },
+    },
+    'formatters': {
+        'verbose': {
+            'format': '%(hostname)s %(hostip)s %(asctime)s %(levelname)s %(pathname)s %(process)d '
+                      '%(processName)s %(thread)d: %(message)s'
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': GIGAMAPS_LOG_LEVEL,
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose',
+            'stream': sys.stderr,
+            'filters': ['hostname_filter'],
+        },
+    },
+    'loggers': {
+        'gigamaps': {
+            'level': GIGAMAPS_LOG_LEVEL,
+            'handlers': ['console'],
+            'filters': ['hostname_filter'],
+        },
+    },
+}

config/settings/dev.py

@@ -28,11 +28,8 @@
 
 try:
     DATABASES['read_only_database'] = env.db_url(var='READ_ONLY_DATABASE_URL')
-    DATABASES['realtime'] = env.db_url(var='REALTIME_DATABASE_URL')
-    DATABASES['dailycheckapp_realtime'] = env.db_url(var='REALTIME_DAILYCHECKAPP_DATABASE_URL')
 except ImproperlyConfigured:
-    DATABASES['realtime'] = DATABASES['default']
-    DATABASES['dailycheckapp_realtime'] = DATABASES['default']
+    pass
 
 # Email settings
 # --------------------------------------------------------------------------

config/settings/dev_test.py

@@ -0,0 +1,63 @@
+from kombu import Exchange, Queue  # NOQA
+
+from config.settings.base import *  # noqa: F403
+
+# Pytest speed improvements configuration
+# Disable debugging for test case execution
+DEBUG = False
+TEMPLATES[0]['OPTIONS']['debug'] = DEBUG
+
+SECRET_KEY = env('SECRET_KEY', default='test_key')
+
+ALLOWED_HOSTS = ['*']
+INTERNAL_IPS = []
+
+ADMINS = (
+    ('Dev Email', env('DEV_ADMIN_EMAIL', default='admin@localhost')),
+)
+MANAGERS = ADMINS
+
+
+# Database
+# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
+# --------------------------------------------------------------------------
+
+DATABASES = {
+    'default': env.db(default='postgis://localhost/proco'),
+}
+
+DATABASES['default']['CONN_MAX_AGE'] = 1000
+
+# Email settings
+# --------------------------------------------------------------------------
+
+# DEFAULT_FROM_EMAIL = '[email protected]'
+# SERVER_EMAIL = DEFAULT_FROM_EMAIL
+# EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
+
+if CELERY_ENABLED:
+    MAILING_USE_CELERY = False
+
+INTERNAL_IPS = ('127.0.0.1',)
+
+# Sentry config
+# -------------
+
+SENTRY_ENABLED = False
+
+
+# Mapbox
+# --------------
+
+MAPBOX_KEY = env('MAPBOX_KEY', default='')
+
+ANYMAIL['DEBUG_API_REQUESTS'] = False
+
+ENABLE_AZURE_COGNITIVE_SEARCH = False
+
+AZURE_CONFIG['COGNITIVE_SEARCH'] = {
+    'SEARCH_ENDPOINT': env('SEARCH_ENDPOINT', default='test.endpoint'),
+    'SEARCH_API_KEY': env('SEARCH_API_KEY', default='testsearchapikey'),
+    'COUNTRY_INDEX_NAME': env('COUNTRY_INDEX_NAME', default='giga_countries'),
+    'SCHOOL_INDEX_NAME': env('SCHOOL_INDEX_NAME', default='giga_schools'),
+}

config/settings/prod.py

@@ -23,8 +23,6 @@
 DATABASES = {
     'default': env.db(),
     'read_only_database': env.db_url(var='READ_ONLY_DATABASE_URL'),
-    'realtime': env.db_url(var='REALTIME_DATABASE_URL'),
-    'dailycheckapp_realtime': env.db_url(var='REALTIME_DAILYCHECKAPP_DATABASE_URL'),
 }
 
 # Template

config/urls.py

@@ -40,7 +40,6 @@ def trigger_error(request):
         path('statistics/', include('proco.connection_statistics.api_urls')),
         path('contact/', include('proco.contact.api_urls')),
         path('about_us/', include('proco.about_us.api_urls')),
-        path('dailycheckapp_contact/', include('proco.dailycheckapp_contact.api_urls')),
         path('accounts/', include('proco.accounts.api_urls')),
         path('sources/', include('proco.data_sources.api_urls')),
     ])),

docker/docker-compose.dev.yml

@@ -64,7 +64,7 @@ services:
     build:
       context: ..
       dockerfile: docker/Dockerfile-dev
-    command: bash -c "sleep 10 && pipenv run celery --app=proco.taskapp beat --loglevel=DEBUG"
+    command: bash -c "sleep 10 && pipenv run celery --app=proco.taskapp beat --scheduler=redbeat.RedBeatScheduler --loglevel=DEBUG"
     environment:
       DJANGO_SETTINGS_MODULE: config.settings.dev
       DATABASE_URL: postgis://test:test@db/proco
@@ -82,15 +82,22 @@ services:
     build:
       context: ..
       dockerfile: docker/Dockerfile-dev
-    command: bash -c "sleep 10 && pipenv run celery --app=proco.taskapp worker --loglevel=DEBUG --time-limit=300 --concurrency=2 --soft-time-limit=300"
+    command: bash -c "sleep 10 && /code/celery-dev.sh --loglevel=DEBUG --task-events"
     environment:
       DJANGO_SETTINGS_MODULE: config.settings.dev
       DATABASE_URL: postgis://test:test@db/proco
-#      CELERY_BROKER_URL: amqp://rabbitmq:rabbitmq@rabbitmq/
       REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/1
+      CELERY_RESULT_BACKEND_URL: redis://redis:6379/2
+      ENABLED_FLOWER_METRICS: true
+      FLOWER_BASIC_AUTH: hz195KIQ2Kvu8S9knla7lDZXIVX35mvj:TCynWCrvV5pCPpfjArcMtm40P39Od3FJ
+      FLOWER_PORT: 6543
+      FLOWER_DEBUG: false
     volumes:
       - "..:/code"
     depends_on:
       - db
 #      - rabbitmq
       - redis
+    ports:
+      - "6543:6543"