From 10aa894c6a8f7569ab8eac4ece0442c7ed84b6ab Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Wed, 23 Oct 2024 15:32:44 -0400 Subject: [PATCH] temp: Retirement refactor example This commit shows an example of how we could use Stevedore drivers to handle retirement extension. The entrypoint name would become the "API" in our configuration, and each "API" extension class would provide a "get_instance" method which would take the retirement config dict and return a constructed instance of the API class, which would then be used a normal. This code is not complete, simply a real world example of how we could abstract out the existing 3rd party APIs using this method. --- scripts/user_retirement/requirements/base.in | 1 + scripts/user_retirement/setup.cfg | 12 +++ scripts/user_retirement/setup.py | 5 ++ scripts/user_retirement/utils/helpers.py | 86 ++++--------------- .../utils/thirdparty_apis/amplitude_api.py | 20 +++++ .../utils/thirdparty_apis/braze_api.py | 21 +++++ .../utils/thirdparty_apis/hubspot_api.py | 24 ++++++ .../utils/thirdparty_apis/salesforce_api.py | 25 ++++++ .../utils/thirdparty_apis/segment_api.py | 23 +++++ 9 files changed, 146 insertions(+), 71 deletions(-) create mode 100644 scripts/user_retirement/setup.cfg create mode 100644 scripts/user_retirement/setup.py diff --git a/scripts/user_retirement/requirements/base.in b/scripts/user_retirement/requirements/base.in index d0bdab9082a5..603875a7e7ee 100644 --- a/scripts/user_retirement/requirements/base.in +++ b/scripts/user_retirement/requirements/base.in @@ -10,4 +10,5 @@ jenkinsapi unicodecsv simplejson simple-salesforce +stevedore google-api-python-client diff --git a/scripts/user_retirement/setup.cfg b/scripts/user_retirement/setup.cfg new file mode 100644 index 000000000000..fc7e42cd407b --- /dev/null +++ b/scripts/user_retirement/setup.cfg @@ -0,0 +1,12 @@ +[options] +packages = utils.thirdparty_apis + +[options.entry_points] +# `retirement_driver` is the namespace chosen by our plugin manager +# this driver should register itself to `retirement_driver` +retirement_driver = + AMPLITUDE = utils.thirdparty_apis.amplitude_api:AmplitudeApi + BRAZE = utils.thirdparty_apis.braze_api:BrazeApi + HUBSPOT = utils.thirdparty_apis.hubspot_api:HubspotApi + SALESFORCE = utils.thirdparty_apis.salesforce_api:SalesforceApi + SEGMENT = utils.thirdparty_apis.segment_api:SegmentApi diff --git a/scripts/user_retirement/setup.py b/scripts/user_retirement/setup.py new file mode 100644 index 000000000000..229b2ebbbc90 --- /dev/null +++ b/scripts/user_retirement/setup.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from setuptools import setup + +setup() diff --git a/scripts/user_retirement/utils/helpers.py b/scripts/user_retirement/utils/helpers.py index 1bcbadb4b3c4..ac84000ee6fe 100644 --- a/scripts/user_retirement/utils/helpers.py +++ b/scripts/user_retirement/utils/helpers.py @@ -16,18 +16,10 @@ import yaml from six import text_type +from stevedore import driver from scripts.user_retirement.utils.edx_api import LmsApi # pylint: disable=wrong-import-position from scripts.user_retirement.utils.edx_api import CredentialsApi, EcommerceApi, LicenseManagerApi -from scripts.user_retirement.utils.thirdparty_apis.amplitude_api import \ - AmplitudeApi # pylint: disable=wrong-import-position -from scripts.user_retirement.utils.thirdparty_apis.braze_api import BrazeApi # pylint: disable=wrong-import-position -from scripts.user_retirement.utils.thirdparty_apis.hubspot_api import \ - HubspotAPI # pylint: disable=wrong-import-position -from scripts.user_retirement.utils.thirdparty_apis.salesforce_api import \ - SalesforceApi # pylint: disable=wrong-import-position -from scripts.user_retirement.utils.thirdparty_apis.segment_api import \ - SegmentApi # pylint: disable=wrong-import-position def _log(kind, message): @@ -152,69 +144,13 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config): lms_base_url = config['base_urls']['lms'] ecommerce_base_url = config['base_urls'].get('ecommerce', None) credentials_base_url = config['base_urls'].get('credentials', None) - segment_base_url = config['base_urls'].get('segment', None) license_manager_base_url = config['base_urls'].get('license_manager', None) client_id = config['client_id'] client_secret = config['client_secret'] - braze_api_key = config.get('braze_api_key', None) - braze_instance = config.get('braze_instance', None) - amplitude_api_key = config.get('amplitude_api_key', None) - amplitude_secret_key = config.get('amplitude_secret_key', None) - salesforce_user = config.get('salesforce_user', None) - salesforce_password = config.get('salesforce_password', None) - salesforce_token = config.get('salesforce_token', None) - salesforce_domain = config.get('salesforce_domain', None) - salesforce_assignee = config.get('salesforce_assignee', None) - segment_auth_token = config.get('segment_auth_token', None) - segment_workspace_slug = config.get('segment_workspace_slug', None) - hubspot_api_key = config.get('hubspot_api_key', None) - hubspot_aws_region = config.get('hubspot_aws_region', None) - hubspot_from_address = config.get('hubspot_from_address', None) - hubspot_alert_email = config.get('hubspot_alert_email', None) - - for state in config['retirement_pipeline']: - for service, service_url in ( - ('BRAZE', braze_api_key), - ('AMPLITUDE', amplitude_api_key), - ('ECOMMERCE', ecommerce_base_url), - ('CREDENTIALS', credentials_base_url), - ('SEGMENT', segment_base_url), - ('HUBSPOT', hubspot_api_key), - ): - if state[2] == service and service_url is None: - fail_func(fail_code, 'Service URL is not configured, but required for state {}'.format(state)) + # Load any Open edX service Apis that are configured config['LMS'] = LmsApi(lms_base_url, lms_base_url, client_id, client_secret) - if braze_api_key: - config['BRAZE'] = BrazeApi( - braze_api_key, - braze_instance, - ) - - if amplitude_api_key and amplitude_secret_key: - config['AMPLITUDE'] = AmplitudeApi( - amplitude_api_key, - amplitude_secret_key, - ) - - if salesforce_user and salesforce_password and salesforce_token: - config['SALESFORCE'] = SalesforceApi( - salesforce_user, - salesforce_password, - salesforce_token, - salesforce_domain, - salesforce_assignee - ) - - if hubspot_api_key: - config['HUBSPOT'] = HubspotAPI( - hubspot_api_key, - hubspot_aws_region, - hubspot_from_address, - hubspot_alert_email - ) - if ecommerce_base_url: config['ECOMMERCE'] = EcommerceApi(lms_base_url, ecommerce_base_url, client_id, client_secret) @@ -229,11 +165,19 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config): client_secret, ) - if segment_base_url: - config['SEGMENT'] = SegmentApi( - segment_base_url, - segment_auth_token, - segment_workspace_slug + # Load and configure any retirement plugin APIs that are installed + for state in config['retirement_pipeline']: + service_name = state[2] + + # Skip any states that have already loaded APIs + if service_name in config: + continue + + mgr = driver.DriverManager( + namespace="retirement_driver", + name=service_name ) + config[service_name] = mgr.driver.get_instance(config) + except Exception as exc: # pylint: disable=broad-except fail_func(fail_code, 'Unexpected error occurred!', exc) diff --git a/scripts/user_retirement/utils/thirdparty_apis/amplitude_api.py b/scripts/user_retirement/utils/thirdparty_apis/amplitude_api.py index ef930f602f0b..62cc59a640c9 100644 --- a/scripts/user_retirement/utils/thirdparty_apis/amplitude_api.py +++ b/scripts/user_retirement/utils/thirdparty_apis/amplitude_api.py @@ -36,6 +36,26 @@ def __init__(self, amplitude_api_key, amplitude_secret_key): self.base_url = "https://amplitude.com/" self.delete_user_path = "api/2/deletions/users" + @staticmethod + def get_instance(config): + """ + This function is used to get instance of AmplitudeApi. + + Returns: + AmplitudeApi: Returns instance of AmplitudeApi. + + Args: + config (dict): Configuration dictionary. + + Raises: + KeyError: If amplitude_api_key or amplitude_secret_key is not present in config. + """ + amplitude_api_key = config.get('amplitude_api_key', None) + amplitude_secret_key = config.get('amplitude_secret_key', None) + if not amplitude_api_key or not amplitude_secret_key: + raise KeyError("amplitude_api_key or amplitude_secret_key is not present in config.") + return AmplitudeApi(amplitude_api_key, amplitude_secret_key) + def auth(self): """ Returns auth credentials for Amplitude authorization. diff --git a/scripts/user_retirement/utils/thirdparty_apis/braze_api.py b/scripts/user_retirement/utils/thirdparty_apis/braze_api.py index 247ceccc6f17..534df1042251 100644 --- a/scripts/user_retirement/utils/thirdparty_apis/braze_api.py +++ b/scripts/user_retirement/utils/thirdparty_apis/braze_api.py @@ -30,6 +30,27 @@ def __init__(self, braze_api_key, braze_instance): # https://www.braze.com/docs/api/basics/#endpoints self.base_url = 'https://rest.{instance}.braze.com'.format(instance=braze_instance) + @staticmethod + def get_instance(config): + """ + This function is used to get instance of BrazeApi. + + Returns: + BrazeApi: Returns instance of BrazeApi. + + Args: + config (dict): Configuration dictionary. + + Raises: + KeyError: If braze_api_key or braze_instance is not present in config. + """ + braze_api_key = config.get('braze_api_key', None) + braze_instance = config.get('braze_instance', None) + if not braze_api_key or not braze_instance: + raise KeyError("braze_api_key or braze_instance is not present in config.") + + return BrazeApi(braze_api_key, braze_instance) + def auth_headers(self): """Returns authorization headers suitable for passing to the requests library""" return { diff --git a/scripts/user_retirement/utils/thirdparty_apis/hubspot_api.py b/scripts/user_retirement/utils/thirdparty_apis/hubspot_api.py index e357fd660264..ed55049aac4c 100644 --- a/scripts/user_retirement/utils/thirdparty_apis/hubspot_api.py +++ b/scripts/user_retirement/utils/thirdparty_apis/hubspot_api.py @@ -37,6 +37,30 @@ def __init__( self.from_address = from_address self.alert_email = alert_email + @staticmethod + def get_instance(config): + """ + This function is used to get instance of HubspotAPI. + + Returns: + HubspotAPI: Returns instance of HubspotAPI. + + Args: + config (dict): Configuration dictionary. + + Raises: + KeyError: If hubspot_api_key is not present in config. + """ + hubspot_api_key = config.get('hubspot_api_key', None) + hubspot_aws_region = config.get('hubspot_aws_region', None) + hubspot_from_address = config.get('hubspot_from_address', None) + hubspot_alert_email = config.get('hubspot_alert_email', None) + + if not hubspot_api_key or not hubspot_aws_region or not hubspot_from_address or not hubspot_alert_email: + raise KeyError("hubspot_api_key, hubspot_aws_region, hubspot_from_address, or hubspot_alert_email is not present in config.") + + return HubspotAPI(hubspot_api_key, hubspot_aws_region, hubspot_from_address, hubspot_alert_email) + @backoff.on_exception( backoff.expo, HubspotException, diff --git a/scripts/user_retirement/utils/thirdparty_apis/salesforce_api.py b/scripts/user_retirement/utils/thirdparty_apis/salesforce_api.py index 354e723739c8..88055a2c47bd 100644 --- a/scripts/user_retirement/utils/thirdparty_apis/salesforce_api.py +++ b/scripts/user_retirement/utils/thirdparty_apis/salesforce_api.py @@ -37,6 +37,31 @@ def __init__(self, username, password, security_token, domain, assignee_username if not self.assignee_id: raise Exception("Could not find Salesforce user with username " + assignee_username) + @staticmethod + def get_instance(config): + """ + This function is used to get instance of SalesforceApi. + + Returns: + SalesforceApi: Returns instance of SalesforceApi. + + Args: + config (dict): Configuration dictionary. + + Raises: + KeyError: If salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, or salesforce_assignee is not present in config. + """ + salesforce_username = config.get('salesforce_username', None) + salesforce_password = config.get('salesforce_password', None) + salesforce_security_token = config.get('salesforce_security_token', None) + salesforce_domain = config.get('salesforce_domain', None) + salesforce_assignee = config.get('salesforce_assignee', None) + + if not salesforce_username or not salesforce_password or not salesforce_security_token or not salesforce_domain or not salesforce_assignee: + raise KeyError("salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, or salesforce_assignee is not present in config.") + + return SalesforceApi(salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, salesforce_assignee) + @backoff.on_exception( backoff.expo, RequestsConnectionError, diff --git a/scripts/user_retirement/utils/thirdparty_apis/segment_api.py b/scripts/user_retirement/utils/thirdparty_apis/segment_api.py index 09df5d069392..b2cb25c82903 100644 --- a/scripts/user_retirement/utils/thirdparty_apis/segment_api.py +++ b/scripts/user_retirement/utils/thirdparty_apis/segment_api.py @@ -99,6 +99,29 @@ def __init__(self, base_url, auth_token, workspace_slug): self.auth_token = auth_token self.workspace_slug = workspace_slug + @staticmethod + def get_instance(config): + """ + This function is used to get instance of SegmentApi. + + Returns: + SegmentApi: Returns instance of SegmentApi. + + Args: + config (dict): Configuration dictionary. + + Raises: + KeyError: If segment_base_url, segment_auth_token, and segment_workspace_slug are not present in config. + """ + segment_base_url = config.get('segment_base_url', None) + segment_auth_token = config.get('segment_auth_token', None) + segment_workspace_slug = config.get('segment_workspace_slug', None) + + if not segment_base_url or not segment_auth_token or not segment_workspace_slug: + raise KeyError("segment_base_url, segment_auth_token or segment_workspace_slug is not present in config.") + + return SegmentApi(segment_base_url, segment_auth_token, segment_workspace_slug) + @_retry_segment_api() def _call_segment_post(self, url, params): """