temp: Retirement refactor example

scripts/user_retirement/requirements/base.in

@@ -10,4 +10,5 @@ jenkinsapi
 unicodecsv
 simplejson
 simple-salesforce
+stevedore
 google-api-python-client

scripts/user_retirement/setup.cfg

@@ -0,0 +1,12 @@
+[options]
+packages = utils.thirdparty_apis
+
+[options.entry_points]
+# `retirement_driver` is the namespace chosen by our plugin manager
+# this driver should register itself to `retirement_driver`
+retirement_driver =
+    AMPLITUDE = utils.thirdparty_apis.amplitude_api:AmplitudeApi
+    BRAZE = utils.thirdparty_apis.braze_api:BrazeApi
+    HUBSPOT = utils.thirdparty_apis.hubspot_api:HubspotApi
+    SALESFORCE = utils.thirdparty_apis.salesforce_api:SalesforceApi
+    SEGMENT = utils.thirdparty_apis.segment_api:SegmentApi

scripts/user_retirement/setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from setuptools import setup
+
+setup()

scripts/user_retirement/utils/helpers.py

@@ -16,18 +16,10 @@
 
 import yaml
 from six import text_type
+from stevedore import driver
 
 from scripts.user_retirement.utils.edx_api import LmsApi  # pylint: disable=wrong-import-position
 from scripts.user_retirement.utils.edx_api import CredentialsApi, EcommerceApi, LicenseManagerApi
-from scripts.user_retirement.utils.thirdparty_apis.amplitude_api import \
-    AmplitudeApi  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.braze_api import BrazeApi  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.hubspot_api import \
-    HubspotAPI  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.salesforce_api import \
-    SalesforceApi  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.segment_api import \
-    SegmentApi  # pylint: disable=wrong-import-position
 
 
 def _log(kind, message):
@@ -152,69 +144,13 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config):
         lms_base_url = config['base_urls']['lms']
         ecommerce_base_url = config['base_urls'].get('ecommerce', None)
         credentials_base_url = config['base_urls'].get('credentials', None)
-        segment_base_url = config['base_urls'].get('segment', None)
         license_manager_base_url = config['base_urls'].get('license_manager', None)
         client_id = config['client_id']
         client_secret = config['client_secret']
-        braze_api_key = config.get('braze_api_key', None)
-        braze_instance = config.get('braze_instance', None)
-        amplitude_api_key = config.get('amplitude_api_key', None)
-        amplitude_secret_key = config.get('amplitude_secret_key', None)
-        salesforce_user = config.get('salesforce_user', None)
-        salesforce_password = config.get('salesforce_password', None)
-        salesforce_token = config.get('salesforce_token', None)
-        salesforce_domain = config.get('salesforce_domain', None)
-        salesforce_assignee = config.get('salesforce_assignee', None)
-        segment_auth_token = config.get('segment_auth_token', None)
-        segment_workspace_slug = config.get('segment_workspace_slug', None)
-        hubspot_api_key = config.get('hubspot_api_key', None)
-        hubspot_aws_region = config.get('hubspot_aws_region', None)
-        hubspot_from_address = config.get('hubspot_from_address', None)
-        hubspot_alert_email = config.get('hubspot_alert_email', None)
-
-        for state in config['retirement_pipeline']:
-            for service, service_url in (
-                ('BRAZE', braze_api_key),
-                ('AMPLITUDE', amplitude_api_key),
-                ('ECOMMERCE', ecommerce_base_url),
-                ('CREDENTIALS', credentials_base_url),
-                ('SEGMENT', segment_base_url),
-                ('HUBSPOT', hubspot_api_key),
-            ):
-                if state[2] == service and service_url is None:
-                    fail_func(fail_code, 'Service URL is not configured, but required for state {}'.format(state))
 
+        # Load any Open edX service Apis that are configured
         config['LMS'] = LmsApi(lms_base_url, lms_base_url, client_id, client_secret)
 
-        if braze_api_key:
-            config['BRAZE'] = BrazeApi(
-                braze_api_key,
-                braze_instance,
-            )
-
-        if amplitude_api_key and amplitude_secret_key:
-            config['AMPLITUDE'] = AmplitudeApi(
-                amplitude_api_key,
-                amplitude_secret_key,
-            )
-
-        if salesforce_user and salesforce_password and salesforce_token:
-            config['SALESFORCE'] = SalesforceApi(
-                salesforce_user,
-                salesforce_password,
-                salesforce_token,
-                salesforce_domain,
-                salesforce_assignee
-            )
-
-        if hubspot_api_key:
-            config['HUBSPOT'] = HubspotAPI(
-                hubspot_api_key,
-                hubspot_aws_region,
-                hubspot_from_address,
-                hubspot_alert_email
-            )
-
         if ecommerce_base_url:
             config['ECOMMERCE'] = EcommerceApi(lms_base_url, ecommerce_base_url, client_id, client_secret)
 
@@ -229,11 +165,19 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config):
                 client_secret,
             )
 
-        if segment_base_url:
-            config['SEGMENT'] = SegmentApi(
-                segment_base_url,
-                segment_auth_token,
-                segment_workspace_slug
+        # Load and configure any retirement plugin APIs that are installed
+        for state in config['retirement_pipeline']:
+            service_name = state[2]
+
+            # Skip any states that have already loaded APIs
+            if service_name in config:
+                continue
+
+            mgr = driver.DriverManager(
+                namespace="retirement_driver",
+                name=service_name
             )
+            config[service_name] = mgr.driver.get_instance(config)
+
     except Exception as exc:  # pylint: disable=broad-except
         fail_func(fail_code, 'Unexpected error occurred!', exc)

scripts/user_retirement/utils/thirdparty_apis/amplitude_api.py

@@ -36,6 +36,26 @@ def __init__(self, amplitude_api_key, amplitude_secret_key):
         self.base_url = "https://amplitude.com/"
         self.delete_user_path = "api/2/deletions/users"
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of AmplitudeApi.
+
+        Returns:
+            AmplitudeApi: Returns instance of AmplitudeApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If amplitude_api_key or amplitude_secret_key is not present in config.
+        """
+        amplitude_api_key = config.get('amplitude_api_key', None)
+        amplitude_secret_key = config.get('amplitude_secret_key', None)
+        if not amplitude_api_key or not amplitude_secret_key:
+            raise KeyError("amplitude_api_key or amplitude_secret_key is not present in config.")
+        return AmplitudeApi(amplitude_api_key, amplitude_secret_key)
+
     def auth(self):
         """
         Returns auth credentials for Amplitude authorization.

scripts/user_retirement/utils/thirdparty_apis/braze_api.py

@@ -30,6 +30,27 @@ def __init__(self, braze_api_key, braze_instance):
         # https://www.braze.com/docs/api/basics/#endpoints
         self.base_url = 'https://rest.{instance}.braze.com'.format(instance=braze_instance)
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of BrazeApi.
+
+        Returns:
+            BrazeApi: Returns instance of BrazeApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If braze_api_key or braze_instance is not present in config.
+        """
+        braze_api_key = config.get('braze_api_key', None)
+        braze_instance = config.get('braze_instance', None)
+        if not braze_api_key or not braze_instance:
+            raise KeyError("braze_api_key or braze_instance is not present in config.")
+
+        return BrazeApi(braze_api_key, braze_instance)
+
     def auth_headers(self):
         """Returns authorization headers suitable for passing to the requests library"""
         return {

scripts/user_retirement/utils/thirdparty_apis/hubspot_api.py

@@ -37,6 +37,30 @@ def __init__(
         self.from_address = from_address
         self.alert_email = alert_email
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of HubspotAPI.
+
+        Returns:
+            HubspotAPI: Returns instance of HubspotAPI.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If hubspot_api_key is not present in config.
+        """
+        hubspot_api_key = config.get('hubspot_api_key', None)
+        hubspot_aws_region = config.get('hubspot_aws_region', None)
+        hubspot_from_address = config.get('hubspot_from_address', None)
+        hubspot_alert_email = config.get('hubspot_alert_email', None)
+
+        if not hubspot_api_key or not hubspot_aws_region or not hubspot_from_address or not hubspot_alert_email:
+            raise KeyError("hubspot_api_key, hubspot_aws_region, hubspot_from_address, or hubspot_alert_email is not present in config.")
+
+        return HubspotAPI(hubspot_api_key, hubspot_aws_region, hubspot_from_address, hubspot_alert_email)
+
     @backoff.on_exception(
         backoff.expo,
         HubspotException,

scripts/user_retirement/utils/thirdparty_apis/salesforce_api.py

@@ -37,6 +37,31 @@ def __init__(self, username, password, security_token, domain, assignee_username
         if not self.assignee_id:
             raise Exception("Could not find Salesforce user with username " + assignee_username)
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of SalesforceApi.
+
+        Returns:
+            SalesforceApi: Returns instance of SalesforceApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, or salesforce_assignee is not present in config.
+        """
+        salesforce_username = config.get('salesforce_username', None)
+        salesforce_password = config.get('salesforce_password', None)
+        salesforce_security_token = config.get('salesforce_security_token', None)
+        salesforce_domain = config.get('salesforce_domain', None)
+        salesforce_assignee = config.get('salesforce_assignee', None)
+
+        if not salesforce_username or not salesforce_password or not salesforce_security_token or not salesforce_domain or not salesforce_assignee:
+            raise KeyError("salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, or salesforce_assignee is not present in config.")
+
+        return SalesforceApi(salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, salesforce_assignee)
+
     @backoff.on_exception(
         backoff.expo,
         RequestsConnectionError,

scripts/user_retirement/utils/thirdparty_apis/segment_api.py

@@ -99,6 +99,29 @@ def __init__(self, base_url, auth_token, workspace_slug):
         self.auth_token = auth_token
         self.workspace_slug = workspace_slug
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of SegmentApi.
+
+        Returns:
+            SegmentApi: Returns instance of SegmentApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If segment_base_url, segment_auth_token, and segment_workspace_slug are not present in config.
+        """
+        segment_base_url = config.get('segment_base_url', None)
+        segment_auth_token = config.get('segment_auth_token', None)
+        segment_workspace_slug = config.get('segment_workspace_slug', None)
+
+        if not segment_base_url or not segment_auth_token or not segment_workspace_slug:
+            raise KeyError("segment_base_url, segment_auth_token or segment_workspace_slug is not present in config.")
+
+        return SegmentApi(segment_base_url, segment_auth_token, segment_workspace_slug)
+
     @_retry_segment_api()
     def _call_segment_post(self, url, params):
         """