21769 - Furnishings job - stage 1 (email flow) (bcgov#2777)

* 21769 Signed-off-by: Hongjing Chen <[email protected]> * update furnishing CI Signed-off-by: Hongjing Chen <[email protected]> * add a unit test Signed-off-by: Hongjing Chen <[email protected]> * update tests Signed-off-by: Hongjing Chen <[email protected]> * update furnishing code model Signed-off-by: Hongjing Chen <[email protected]> * assign grouping_identifier Signed-off-by: Hongjing Chen <[email protected]> * update tests Signed-off-by: Hongjing Chen <[email protected]> * lots of refactoring Signed-off-by: Hongjing Chen <[email protected]> * remove unnecessary noqa Signed-off-by: Hongjing Chen <[email protected]> * rename & remove unused import Signed-off-by: Hongjing Chen <[email protected]> * update docstring Signed-off-by: Hongjing Chen <[email protected]> * remove processed_date value Signed-off-by: Hongjing Chen <[email protected]> --------- Signed-off-by: Hongjing Chen <[email protected]>
PaulGarewal · Jun 27, 2024 · e58ba1a · e58ba1a
1 parent 883671c
commit e58ba1a
Show file tree

Hide file tree

Showing 24 changed files with 800 additions and 69 deletions.
diff --git a/.github/workflows/furnishings-ci.yml b/.github/workflows/furnishings-ci.yml
@@ -55,6 +55,10 @@ jobs:
       DATABASE_PASSWORD: postgres
       DATABASE_HOST: localhost
       DATABASE_NAME: postgres
+      DATABASE_TEST_USERNAME: postgres
+      DATABASE_TEST_PASSWORD: postgres
+      DATABASE_TEST_NAME: postgres
+      DATABASE_TEST_HOST: localhost
       NATS_SERVERS: "nats://nats:4222"
       NATS_CLUSTER_ID: test-cluster
       NATS_CLIENT_NAME: entity.job.tester

diff --git a/jobs/furnishings/Makefile b/jobs/furnishings/Makefile
@@ -62,10 +62,10 @@ install-dev: ## Install local application
 ci: lint flake8 test ## CI flow
 
 pylint: ## Linting with pylint
-	. venv/bin/activate && pylint --rcfile=setup.cfg furnish.py
+	. venv/bin/activate && pylint --rcfile=setup.cfg src/$(PROJECT_NAME)
 
 flake8: ## Linting with flake8
-	. venv/bin/activate && flake8 furnish.py
+	. venv/bin/activate && flake8 src/$(PROJECT_NAME)
 
 lint: pylint flake8 ## run all lint type scripts
 

diff --git a/jobs/furnishings/devops/vaults.json b/jobs/furnishings/devops/vaults.json
@@ -10,7 +10,9 @@
         "application": [
             "postgres-legal",
             "launchdarkly",
-            "nats-filer"
+            "nats-filer",
+            "nats-emailer",
+            "entity-service-account"
         ]
     },
     {

diff --git a/jobs/furnishings/furnish.py b/jobs/furnishings/furnish.py
@@ -12,64 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Furnishings job."""
-import logging
-import os
+import asyncio
 
-import sentry_sdk  # noqa: I001, E501; pylint: disable=ungrouped-imports; conflicts with Flake8
-from flask import Flask
-from legal_api.models import db  # noqa: I001
-from legal_api.services.flags import Flags
-from sentry_sdk.integrations.logging import LoggingIntegration
+from legal_api.services.queue import QueueService
 
-import config  # pylint: disable=import-error
-from utils.logging import setup_logging  # pylint: disable=import-error
-# noqa: I003
-
-setup_logging(
-    os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf'))
-
-SENTRY_LOGGING = LoggingIntegration(
-    event_level=logging.ERROR  # send errors as events
-)
-
-flags = Flags()
-
-
-def create_app(run_mode=os.getenv('FLASK_ENV', 'production')):
-    """Return a configured Flask App using the Factory method."""
-    app = Flask(__name__)
-    app.config.from_object(config.CONFIGURATION[run_mode])
-    db.init_app(app)
-
-    # Configure Sentry
-    if app.config.get('SENTRY_DSN', None):
-        sentry_sdk.init(
-            dsn=app.config.get('SENTRY_DSN'),
-            integrations=[SENTRY_LOGGING]
-        )
-
-    if app.config.get('LD_SDK_KEY', None):
-        flags.init_app(app)
-
-    register_shellcontext(app)
-
-    return app
-
-
-def register_shellcontext(app):
-    """Register shell context objects."""
-    def shell_context():
-        """Shell context objects."""
-        return {'app': app}
-
-    app.shell_context_processor(shell_context)
+from furnishings.worker import create_app, run
 
 
 if __name__ == '__main__':
     application = create_app()
-    with application.app_context():
-        flag_on = flags.is_on('enable-involuntary-dissolution')
-        application.logger.debug(f'enable-involuntary-dissolution flag on: {flag_on}')
-        if flag_on:
-            # TODO: detailed implementation
-            application.logger.debug('Running furnishings job')
+    try:
+        event_loop = asyncio.get_event_loop()
+        queue_service = QueueService(app=application, loop=event_loop)
+        event_loop.run_until_complete(run(application, queue_service))
+    except Exception as err:  # pylint: disable=broad-except; Catching all errors from the frameworks
+        application.logger.error(err)  # pylint: disable=no-member
+        raise err
diff --git a/jobs/furnishings/requirements.txt b/jobs/furnishings/requirements.txt
@@ -5,6 +5,8 @@ Jinja2==2.11.3
 MarkupSafe==1.1.1
 Werkzeug==1.0.1
 aniso8601==9.0.1
+asyncio-nats-client==0.11.4
+asyncio-nats-streaming==0.4.0
 attrs==23.1.0
 blinker==1.4
 certifi==2020.12.5
@@ -14,6 +16,8 @@ flask-jwt-oidc==0.3.0
 gunicorn==20.1.0
 itsdangerous==1.1.0
 jsonschema==4.19.0
+nest_asyncio
+protobuf==3.15.8
 pyasn1==0.4.8
 pyrsistent==0.17.3
 python-dotenv==0.17.1

diff --git a/jobs/furnishings/requirements/dev.txt b/jobs/furnishings/requirements/dev.txt
@@ -1,7 +1,9 @@
 # Everything the developer needs outside of the production requirements
 
 # Testing
+freezegun
 pytest
+pytest-asyncio
 pytest-mock
 requests
 pyhamcrest

diff --git a/jobs/furnishings/requirements/prod.txt b/jobs/furnishings/requirements/prod.txt
@@ -3,7 +3,10 @@ Flask
 Flask-Script
 Flask-Moment
 Flask-RESTplus
+asyncio-nats-client
+asyncio-nats-streaming
 flask-jwt-oidc>=0.1.5
+nest_asyncio
 python-dotenv
 sentry-sdk[flask]
 werkzeug
diff --git a/jobs/furnishings/setup.py b/jobs/furnishings/setup.py
@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Installer and setup for this module."""
-
+from glob import glob
+from os.path import basename, splitext
 from setuptools import find_packages, setup
 
 setup(
     name='furnishings',
-    packages=find_packages()
+    packages=find_packages('src'),
+    package_dir={'': 'src'},
+    py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')]
 )
diff --git a/jobs/furnishings/src/furnishings/__init__.py b/jobs/furnishings/src/furnishings/__init__.py
@@ -0,0 +1,17 @@
+# Copyright © 2024 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The furnishings job.
+
+This module is the job worker for involuntary dissolution furnishing tasks.
+"""
diff --git a/jobs/furnishings/config.py → jobs/furnishings/src/furnishings/config.py b/jobs/furnishings/config.py → jobs/furnishings/src/furnishings/config.py
@@ -18,6 +18,7 @@
 
 from dotenv import find_dotenv, load_dotenv
 
+
 load_dotenv(find_dotenv())
 
 CONFIGURATION = {
@@ -41,18 +42,24 @@ def get_named_config(config_name: str = 'production'):
     return config
 
 
-class _Config(object):  # pylint: disable=too-few-public-methods
+class _Config:  # pylint: disable=too-few-public-methods
     """Base class configuration."""
 
     PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
 
     SENTRY_DSN = os.getenv('SENTRY_DSN', '')
     LD_SDK_KEY = os.getenv('LD_SDK_KEY', None)
 
+    AUTH_URL = os.getenv('AUTH_URL', None)
+    ACCOUNT_SVC_AUTH_URL = os.getenv('ACCOUNT_SVC_AUTH_URL', None)
+    ACCOUNT_SVC_CLIENT_ID = os.getenv('ACCOUNT_SVC_CLIENT_ID', None)
+    ACCOUNT_SVC_CLIENT_SECRET = os.getenv('ACCOUNT_SVC_CLIENT_SECRET', None)
+
     NATS_SERVERS = os.getenv('NATS_SERVERS', None)
     NATS_CLUSTER_ID = os.getenv('NATS_CLUSTER_ID', None)
     NATS_CLIENT_NAME = os.getenv('NATS_CLIENT_NAME', None)
     NATS_ENTITY_EVENTS_SUBJECT = os.getenv('NATS_ENTITY_EVENTS_SUBJECT', 'entity.events')
+    NATS_EMAILER_SUBJECT = os.getenv('NATS_EMAILER_SUBJECT', 'entity.email')
 
     SECRET_KEY = 'a secret'
 

diff --git a/jobs/furnishings/src/furnishings/stage_processors/__init__.py b/jobs/furnishings/src/furnishings/stage_processors/__init__.py
@@ -0,0 +1,17 @@
+# Copyright © 2024 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This module contains all of the furnishing processors.
+
+Processors hold the furnishing logic for each stage of involuntary dissolution.
+"""