-
Notifications
You must be signed in to change notification settings - Fork 3.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
temp: Retirement refactor example #35714
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -10,4 +10,5 @@ jenkinsapi | |
unicodecsv | ||
simplejson | ||
simple-salesforce | ||
stevedore | ||
google-api-python-client |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
[options] | ||
packages = utils.thirdparty_apis | ||
|
||
[options.entry_points] | ||
# `retirement_driver` is the namespace chosen by our plugin manager | ||
# this driver should register itself to `retirement_driver` | ||
retirement_driver = | ||
AMPLITUDE = utils.thirdparty_apis.amplitude_api:AmplitudeApi | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The entrypoint names become the "API" from our config, so keeping this
|
||
BRAZE = utils.thirdparty_apis.braze_api:BrazeApi | ||
HUBSPOT = utils.thirdparty_apis.hubspot_api:HubspotApi | ||
SALESFORCE = utils.thirdparty_apis.salesforce_api:SalesforceApi | ||
SEGMENT = utils.thirdparty_apis.segment_api:SegmentApi |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
#!/usr/bin/env python3 | ||
|
||
from setuptools import setup | ||
|
||
setup() |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,18 +16,10 @@ | |
|
||
import yaml | ||
from six import text_type | ||
from stevedore import driver | ||
|
||
from scripts.user_retirement.utils.edx_api import LmsApi # pylint: disable=wrong-import-position | ||
from scripts.user_retirement.utils.edx_api import CredentialsApi, EcommerceApi, LicenseManagerApi | ||
from scripts.user_retirement.utils.thirdparty_apis.amplitude_api import \ | ||
AmplitudeApi # pylint: disable=wrong-import-position | ||
from scripts.user_retirement.utils.thirdparty_apis.braze_api import BrazeApi # pylint: disable=wrong-import-position | ||
from scripts.user_retirement.utils.thirdparty_apis.hubspot_api import \ | ||
HubspotAPI # pylint: disable=wrong-import-position | ||
from scripts.user_retirement.utils.thirdparty_apis.salesforce_api import \ | ||
SalesforceApi # pylint: disable=wrong-import-position | ||
from scripts.user_retirement.utils.thirdparty_apis.segment_api import \ | ||
SegmentApi # pylint: disable=wrong-import-position | ||
|
||
|
||
def _log(kind, message): | ||
|
@@ -152,69 +144,13 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config): | |
lms_base_url = config['base_urls']['lms'] | ||
ecommerce_base_url = config['base_urls'].get('ecommerce', None) | ||
credentials_base_url = config['base_urls'].get('credentials', None) | ||
segment_base_url = config['base_urls'].get('segment', None) | ||
license_manager_base_url = config['base_urls'].get('license_manager', None) | ||
client_id = config['client_id'] | ||
client_secret = config['client_secret'] | ||
braze_api_key = config.get('braze_api_key', None) | ||
braze_instance = config.get('braze_instance', None) | ||
amplitude_api_key = config.get('amplitude_api_key', None) | ||
amplitude_secret_key = config.get('amplitude_secret_key', None) | ||
salesforce_user = config.get('salesforce_user', None) | ||
salesforce_password = config.get('salesforce_password', None) | ||
salesforce_token = config.get('salesforce_token', None) | ||
salesforce_domain = config.get('salesforce_domain', None) | ||
salesforce_assignee = config.get('salesforce_assignee', None) | ||
segment_auth_token = config.get('segment_auth_token', None) | ||
segment_workspace_slug = config.get('segment_workspace_slug', None) | ||
hubspot_api_key = config.get('hubspot_api_key', None) | ||
hubspot_aws_region = config.get('hubspot_aws_region', None) | ||
hubspot_from_address = config.get('hubspot_from_address', None) | ||
hubspot_alert_email = config.get('hubspot_alert_email', None) | ||
|
||
for state in config['retirement_pipeline']: | ||
for service, service_url in ( | ||
('BRAZE', braze_api_key), | ||
('AMPLITUDE', amplitude_api_key), | ||
('ECOMMERCE', ecommerce_base_url), | ||
('CREDENTIALS', credentials_base_url), | ||
('SEGMENT', segment_base_url), | ||
('HUBSPOT', hubspot_api_key), | ||
): | ||
if state[2] == service and service_url is None: | ||
fail_func(fail_code, 'Service URL is not configured, but required for state {}'.format(state)) | ||
|
||
# Load any Open edX service Apis that are configured | ||
config['LMS'] = LmsApi(lms_base_url, lms_base_url, client_id, client_secret) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Open question whether we want to make the 1st party APIs function the same way, but for the sake of simplicity I've left them for now. |
||
|
||
if braze_api_key: | ||
config['BRAZE'] = BrazeApi( | ||
braze_api_key, | ||
braze_instance, | ||
) | ||
|
||
if amplitude_api_key and amplitude_secret_key: | ||
config['AMPLITUDE'] = AmplitudeApi( | ||
amplitude_api_key, | ||
amplitude_secret_key, | ||
) | ||
|
||
if salesforce_user and salesforce_password and salesforce_token: | ||
config['SALESFORCE'] = SalesforceApi( | ||
salesforce_user, | ||
salesforce_password, | ||
salesforce_token, | ||
salesforce_domain, | ||
salesforce_assignee | ||
) | ||
|
||
if hubspot_api_key: | ||
config['HUBSPOT'] = HubspotAPI( | ||
hubspot_api_key, | ||
hubspot_aws_region, | ||
hubspot_from_address, | ||
hubspot_alert_email | ||
) | ||
|
||
if ecommerce_base_url: | ||
config['ECOMMERCE'] = EcommerceApi(lms_base_url, ecommerce_base_url, client_id, client_secret) | ||
|
||
|
@@ -229,11 +165,19 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config): | |
client_secret, | ||
) | ||
|
||
if segment_base_url: | ||
config['SEGMENT'] = SegmentApi( | ||
segment_base_url, | ||
segment_auth_token, | ||
segment_workspace_slug | ||
# Load and configure any retirement plugin APIs that are installed | ||
for state in config['retirement_pipeline']: | ||
service_name = state[2] | ||
|
||
# Skip any states that have already loaded APIs | ||
if service_name in config: | ||
continue | ||
|
||
mgr = driver.DriverManager( | ||
namespace="retirement_driver", | ||
name=service_name | ||
) | ||
config[service_name] = mgr.driver.get_instance(config) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These few lines are where the magic happens, if we don't already have an "API" configured (LMS, Ecomm, License Manager, etc would already be configured here) we take the value from our configuration and try to fetch a plugin that has a matching name in the If it exists, the given class will be have |
||
|
||
except Exception as exc: # pylint: disable=broad-except | ||
fail_func(fail_code, 'Unexpected error occurred!', exc) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -36,6 +36,26 @@ def __init__(self, amplitude_api_key, amplitude_secret_key): | |
self.base_url = "https://amplitude.com/" | ||
self.delete_user_path = "api/2/deletions/users" | ||
|
||
@staticmethod | ||
def get_instance(config): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Configuration for each plugin API gets pushed down to the class, which is really where it belongs. For simplicity and backwards compatibility I've left all of the configuration variables the same. This does involve passing the entire config object with potentially many sets of highly permissioned credentials into each plugin. That allows for things like plugins being able to contact the LMS or ecommerce, but isn't a security best practice. A potential improvement would be to move each plugin's configuration to a separate section and only pass that in. I've opted for backwards compatibility over all other considerations in this first pass. |
||
""" | ||
This function is used to get instance of AmplitudeApi. | ||
|
||
Returns: | ||
AmplitudeApi: Returns instance of AmplitudeApi. | ||
|
||
Args: | ||
config (dict): Configuration dictionary. | ||
|
||
Raises: | ||
KeyError: If amplitude_api_key or amplitude_secret_key is not present in config. | ||
""" | ||
amplitude_api_key = config.get('amplitude_api_key', None) | ||
amplitude_secret_key = config.get('amplitude_secret_key', None) | ||
if not amplitude_api_key or not amplitude_secret_key: | ||
raise KeyError("amplitude_api_key or amplitude_secret_key is not present in config.") | ||
return AmplitudeApi(amplitude_api_key, amplitude_secret_key) | ||
|
||
def auth(self): | ||
""" | ||
Returns auth credentials for Amplitude authorization. | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Any other python packages can be installed that have a setup.cfg like this to register their plugins with the system. They can still be turned on and off by adding or removing the steps in the retirement configuration file.