diff --git a/jobs/update-colin-filings/config.py b/jobs/update-colin-filings/config.py index 8f6c7b1cf1..fd10197fdd 100644 --- a/jobs/update-colin-filings/config.py +++ b/jobs/update-colin-filings/config.py @@ -47,12 +47,13 @@ class _Config(object): # pylint: disable=too-few-public-methods PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) COLIN_URL = os.getenv('COLIN_URL', '') - LEGAL_URL = os.getenv('LEGAL_URL', '') + LEGAL_API_URL = os.getenv('LEGAL_API_URL', '') SENTRY_DSN = os.getenv('SENTRY_DSN', '') ACCOUNT_SVC_AUTH_URL = os.getenv('ACCOUNT_SVC_AUTH_URL', None) ACCOUNT_SVC_CLIENT_ID = os.getenv('ACCOUNT_SVC_CLIENT_ID', None) ACCOUNT_SVC_CLIENT_SECRET = os.getenv('ACCOUNT_SVC_CLIENT_SECRET', None) + ACCOUNT_SVC_TIMEOUT = os.getenv('ACCOUNT_SVC_TIMEOUT', 20) SECRET_KEY = 'a secret' @@ -74,7 +75,7 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods TESTING = True COLIN_URL = os.getenv('COLIN_URL_TEST', '') - LEGAL_URL = os.getenv('LEGAL_URL_TEST', '') + LEGAL_API_URL = os.getenv('LEGAL_API_URL_TEST', '') class ProdConfig(_Config): # pylint: disable=too-few-public-methods diff --git a/jobs/update-colin-filings/k8s/templates/cronjob.yaml b/jobs/update-colin-filings/k8s/templates/cronjob.yaml index ad7f9fa76d..69f13b78e3 100644 --- a/jobs/update-colin-filings/k8s/templates/cronjob.yaml +++ b/jobs/update-colin-filings/k8s/templates/cronjob.yaml @@ -52,10 +52,10 @@ objects: secretKeyRef: key: SENTRY_DSN name: ${NAME}-${TAG}-secret - - name: LEGAL_URL + - name: LEGAL_API_URL valueFrom: secretKeyRef: - key: LEGAL_URL + key: LEGAL_API_URL name: ${NAME}-${TAG}-secret - name: ACCOUNT_SVC_AUTH_URL valueFrom: diff --git a/jobs/update-colin-filings/update_colin_filings.py b/jobs/update-colin-filings/update_colin_filings.py index 76422a2437..1947e3eeae 100644 --- a/jobs/update-colin-filings/update_colin_filings.py +++ b/jobs/update-colin-filings/update_colin_filings.py @@ -21,7 +21,6 @@ import requests import sentry_sdk # noqa: I001; pylint: disable=ungrouped-imports; conflicts with Flake8 from flask import Flask -from legal_api.models import Business from legal_api.services.bootstrap import AccountService from sentry_sdk.integrations.logging import LoggingIntegration # noqa: I001 @@ -47,6 +46,10 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'production')): integrations=[SENTRY_LOGGING] ) + # Static class load the variables while importing the class for the first time, + # By then config is not loaded, so it never get the config value + AccountService.timeout = int(app.config.get('ACCOUNT_SVC_TIMEOUT')) + register_shellcontext(app) return app @@ -61,9 +64,10 @@ def shell_context(): app.shell_context_processor(shell_context) -def get_filings(app: Flask = None): +def get_filings(app: Flask, token): """Get a filing with filing_id.""" - req = requests.get(f'{app.config["LEGAL_URL"]}/internal/filings', + req = requests.get(f'{app.config["LEGAL_API_URL"]}/internal/filings', + headers={'Authorization': AccountService.BEARER + token}, timeout=AccountService.timeout) if not req or req.status_code != 200: app.logger.error(f'Failed to collect filings from legal-api. {req} {req.json()} {req.status_code}') @@ -78,10 +82,7 @@ def send_filing(app: Flask = None, filing: dict = None, filing_id: str = None): filing_type = filing['filing']['header'].get('name', None) identifier = filing['filing']['business'].get('identifier', None) - if identifier[:2] == Business.LegalTypes.COOP.value: - legal_type = Business.LegalTypes.COOP.value - else: - legal_type = filing['filing']['business'].get('legalType', Business.LegalTypes.BCOMP.value) + legal_type = filing['filing']['business'].get('legalType', None) req = None if legal_type and identifier and filing_type: @@ -102,8 +103,8 @@ def send_filing(app: Flask = None, filing: dict = None, filing_id: str = None): def update_colin_id(app: Flask = None, filing_id: str = None, colin_ids: list = None, token: dict = None): """Update the colin_id in the filings table.""" req = requests.patch( - f'{app.config["LEGAL_URL"]}/internal/filings/{filing_id}', - headers={'Authorization': f'Bearer {token}'}, + f'{app.config["LEGAL_API_URL"]}/internal/filings/{filing_id}', + headers={'Authorization': AccountService.BEARER + token}, json={'colinIds': colin_ids}, timeout=AccountService.timeout ) @@ -132,7 +133,7 @@ def run(): # get updater-job token token = AccountService.get_bearer_token() - filings = get_filings(app=application) + filings = get_filings(application, token) if not filings: # pylint: disable=no-member; false positive application.logger.debug('No completed filings to send to colin.') diff --git a/jobs/update-legal-filings/config.py b/jobs/update-legal-filings/config.py index 47eb047e58..fb72bfdb81 100644 --- a/jobs/update-legal-filings/config.py +++ b/jobs/update-legal-filings/config.py @@ -47,12 +47,13 @@ class _Config(object): # pylint: disable=too-few-public-methods PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) COLIN_URL = os.getenv('COLIN_URL', '') - LEGAL_URL = os.getenv('LEGAL_URL', '') + LEGAL_API_URL = os.getenv('LEGAL_API_URL', '') SENTRY_DSN = os.getenv('SENTRY_DSN', '') ACCOUNT_SVC_AUTH_URL = os.getenv('ACCOUNT_SVC_AUTH_URL', None) ACCOUNT_SVC_CLIENT_ID = os.getenv('ACCOUNT_SVC_CLIENT_ID', None) ACCOUNT_SVC_CLIENT_SECRET = os.getenv('ACCOUNT_SVC_CLIENT_SECRET', None) + ACCOUNT_SVC_TIMEOUT = os.getenv('ACCOUNT_SVC_TIMEOUT', 20) NATS_SERVERS = os.getenv('NATS_SERVERS', None) NATS_CLUSTER_ID = os.getenv('NATS_CLUSTER_ID', None) @@ -84,7 +85,7 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods TESTING = True COLIN_URL = os.getenv('COLIN_URL_TEST', '') - LEGAL_URL = os.getenv('LEGAL_URL_TEST', '') + LEGAL_API_URL = os.getenv('LEGAL_API_URL_TEST', '') # POSTGRESQL DB_USER = os.getenv('DATABASE_TEST_USERNAME', '') diff --git a/jobs/update-legal-filings/k8s/templates/cronjob.yaml b/jobs/update-legal-filings/k8s/templates/cronjob.yaml index edd300a071..65bb432943 100644 --- a/jobs/update-legal-filings/k8s/templates/cronjob.yaml +++ b/jobs/update-legal-filings/k8s/templates/cronjob.yaml @@ -52,11 +52,11 @@ objects: secretKeyRef: name: ${NAME}-${TAG}-secret key: SENTRY_DSN - - name: LEGAL_URL + - name: LEGAL_API_URL valueFrom: secretKeyRef: name: ${NAME}-${TAG}-secret - key: LEGAL_URL + key: LEGAL_API_URL - name: ACCOUNT_SVC_AUTH_URL valueFrom: secretKeyRef: diff --git a/jobs/update-legal-filings/requirements.txt b/jobs/update-legal-filings/requirements.txt index 08eed43a84..95e92e320b 100644 --- a/jobs/update-legal-filings/requirements.txt +++ b/jobs/update-legal-filings/requirements.txt @@ -23,6 +23,10 @@ rsa==4.7.2 sentry-sdk==1.20.0 six==1.15.0 urllib3==1.26.11 +asyncio-nats-client==0.11.4 +asyncio-nats-streaming==0.4.0 +nest_asyncio +protobuf==3.15.8 git+https://github.com/bcgov/lear.git#subdirectory=colin-api git+https://github.com/bcgov/lear.git#egg=legal_api&subdirectory=legal-api git+https://github.com/bcgov/business-schemas.git@2.5.12#egg=registry_schemas diff --git a/jobs/update-legal-filings/requirements/prod.txt b/jobs/update-legal-filings/requirements/prod.txt index b3101d8282..bddf5c8120 100644 --- a/jobs/update-legal-filings/requirements/prod.txt +++ b/jobs/update-legal-filings/requirements/prod.txt @@ -7,3 +7,6 @@ flask-jwt-oidc>=0.1.5 python-dotenv sentry-sdk[flask] werkzeug +asyncio-nats-client==0.11.4 +asyncio-nats-streaming==0.4.0 +nest_asyncio diff --git a/jobs/update-legal-filings/update_legal_filings.py b/jobs/update-legal-filings/update_legal_filings.py index 80070a8f6f..7915b779c3 100644 --- a/jobs/update-legal-filings/update_legal_filings.py +++ b/jobs/update-legal-filings/update_legal_filings.py @@ -58,6 +58,10 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'production')): register_shellcontext(app) + # Static class load the variables while importing the class for the first time, + # By then config is not loaded, so it never get the config value + AccountService.timeout = int(app.config.get('ACCOUNT_SVC_TIMEOUT')) + return app @@ -75,7 +79,7 @@ def check_for_manual_filings(application: Flask = None, token: dict = None): """Check for colin filings in oracle.""" id_list = [] colin_events = None - legal_url = application.config['LEGAL_URL'] + legal_url = application.config['LEGAL_API_URL'] colin_url = application.config['COLIN_URL'] corp_types = [Business.TypeCodes.COOP.value, Business.TypeCodes.BC_COMP.value, Business.TypeCodes.ULC_COMP.value, Business.TypeCodes.CCC_COMP.value] @@ -83,7 +87,9 @@ def check_for_manual_filings(application: Flask = None, token: dict = None): Business.TypeCodes.CCC_COMP.value] # get max colin event_id from legal - response = requests.get(f'{legal_url}/internal/filings/colin_id', timeout=AccountService.timeout) + response = requests.get(f'{legal_url}/internal/filings/colin_id', + headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'}, + timeout=AccountService.timeout) if response.status_code not in [200, 404]: application.logger.error(f'Error getting last updated colin id from \ legal: {response.status_code} {response.json()}') @@ -139,8 +145,10 @@ def check_for_manual_filings(application: Flask = None, token: dict = None): ) if response.status_code == 200: # check legal table - response = requests.get(f'{legal_url}/internal/filings/colin_id/{info["event_id"]}', - timeout=AccountService.timeout) + response = requests.get( + f'{legal_url}/internal/filings/colin_id/{info["event_id"]}', + headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'}, + timeout=AccountService.timeout) if response.status_code == 404: id_list.append(info) elif response.status_code != 200: @@ -209,7 +217,7 @@ def update_filings(application): # pylint: disable=redefined-outer-name, too-ma # call legal api with filing application.logger.debug(f'sending filing with event info: {event_info} to legal api.') response = requests.post( - f'{application.config["LEGAL_URL"]}/{event_info["corp_num"]}/filings', + f'{application.config["LEGAL_API_URL"]}/{event_info["corp_num"]}/filings', json=filing, headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'}, timeout=AccountService.timeout @@ -247,7 +255,7 @@ def update_filings(application): # pylint: disable=redefined-outer-name, too-ma # update max_event_id in legal_db application.logger.debug(f'setting last_event_id in legal_db to {max_event_id}') response = requests.post( - f'{application.config["LEGAL_URL"]}/internal/filings/colin_id/{max_event_id}', + f'{application.config["LEGAL_API_URL"]}/internal/filings/colin_id/{max_event_id}', headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'}, timeout=AccountService.timeout ) @@ -308,7 +316,7 @@ async def update_business_nos(application): # pylint: disable=redefined-outer-n # get identifiers with outstanding tax_ids application.logger.debug('Getting businesses with outstanding tax ids from legal api...') response = requests.get( - application.config['LEGAL_URL'] + '/internal/tax_ids', + application.config['LEGAL_API_URL'] + '/internal/tax_ids', headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'}, timeout=AccountService.timeout ) @@ -334,7 +342,7 @@ async def update_business_nos(application): # pylint: disable=redefined-outer-n # update lear with new tax ids from colin application.logger.debug(f'Updating tax ids for {tax_ids.keys()} in lear...') response = requests.post( - application.config['LEGAL_URL'] + '/internal/tax_ids', + application.config['LEGAL_API_URL'] + '/internal/tax_ids', json=tax_ids, headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'}, timeout=AccountService.timeout