Tdl 14803 check api access in discovery mode

tap_zendesk/__init__.py

@@ -46,9 +46,9 @@ def request_metrics_patch(self, method, url, **kwargs):
 Session.request = request_metrics_patch
 # end patch
 
-def do_discover(client):
+def do_discover(client, config):
     LOGGER.info("Starting discover")
-    catalog = {"streams": discover_streams(client)}
+    catalog = {"streams": discover_streams(client, config)}
     json.dump(catalog, sys.stdout, indent=2)
     LOGGER.info("Finished discover")
 
@@ -199,7 +199,7 @@ def main():
         LOGGER.error("""No suitable authentication keys provided.""")
 
     if parsed_args.discover:
-        do_discover(client)
+        do_discover(client, parsed_args.config)
     elif parsed_args.catalog:
         state = parsed_args.state
         do_sync(client, parsed_args.catalog, state, parsed_args.config)

tap_zendesk/discover.py

@@ -1,7 +1,11 @@
 import os
 import json
 import singer
+import zenpy
 from tap_zendesk.streams import STREAMS
+from tap_zendesk.http import ZendeskForbiddenError
+
+LOGGER = singer.get_logger()
 
 def get_abs_path(path):
     return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
@@ -20,12 +24,37 @@ def load_shared_schema_refs():
 
     return shared_schema_refs
 
-def discover_streams(client):
+def discover_streams(client, config):
     streams = []
+    error_list = []
     refs = load_shared_schema_refs()
 
+
     for s in STREAMS.values():
-        s = s(client)
+        s = s(client, config)
         schema = singer.resolve_schema_references(s.load_schema(), refs)
+        try:
+            s.check_access()
+        except ZendeskForbiddenError as e:
+            error_list.append(s.name)
+        except zenpy.lib.exception.APIException as e:
+            err = json.loads(e.args[0]).get('error')
+
+            if isinstance(err, dict):
+                if err.get('message', None) == "You do not have access to this page. Please contact the account owner of this help desk for further help.":
+                    error_list.append(s.name)
+            elif json.loads(e.args[0]).get('description') == "You are missing the following required scopes: read":
+                error_list.append(s.name)
+            else:
+                raise e
+
         streams.append({'stream': s.name, 'tap_stream_id': s.name, 'schema': schema, 'metadata': s.load_metadata()})
+
+    if error_list:
+        streams_name = ", ".join(error_list)
+        message = "HTTP-error-code: 403, Error: You are missing the following required scopes: read. "\
+                    f"The account credentials supplied do not have read access for the following stream(s):  {streams_name}"
+        raise ZendeskForbiddenError(message)
+
+
     return streams

tap_zendesk/http.py

@@ -7,6 +7,95 @@
 LOGGER = singer.get_logger()
 
 
+class ZendeskError(Exception):
+    def __init__(self, message=None, response=None):
+        super().__init__(message)
+        self.message = message
+        self.response = response
+
+class ZendeskBackoffError(ZendeskError):
+    pass
+
+class ZendeskBadRequestError(ZendeskError):
+    pass
+
+class ZendeskUnauthorizedError(ZendeskError):
+    pass
+
+class ZendeskForbiddenError(ZendeskError):
+    pass
+
+class ZendeskNotFoundError(ZendeskError):
+    pass
+
+class ZendeskConflictError(ZendeskError):
+    pass
+
+class ZendeskUnprocessableEntityError(ZendeskError):
+    pass
+
+class ZendeskRateLimitError(ZendeskBackoffError):
+    pass
+
+class ZendeskInternalServerError(ZendeskError):
+    pass
+
+class ZendeskNotImplementedError(ZendeskError):
+    pass
+
+class ZendeskBadGatewayError(ZendeskError):
+    pass
+
+class ZendeskServiceUnavailableError(ZendeskBackoffError):
+    pass
+
+ERROR_CODE_EXCEPTION_MAPPING = {
+    400: {
+        "raise_exception": ZendeskBadRequestError,
+        "message": "A validation exception has occurred."
+    },
+    401: {
+        "raise_exception": ZendeskUnauthorizedError,
+        "message": "The access token provided is expired, revoked, malformed or invalid for other reasons."
+    },
+    403: {
+        "raise_exception": ZendeskForbiddenError,
+        "message": "You are missing the following required scopes: read"
+    },
+    404: {
+        "raise_exception": ZendeskNotFoundError,
+        "message": "There is no help desk configured at this address. This means that the address is available and that you can claim it at http://www.zendesk.com/signup"
+    },
+    409: {
+        "raise_exception": ZendeskConflictError,
+        "message": "The request does not match our state in some way."
+    },
+    422: {
+        "raise_exception": ZendeskUnprocessableEntityError,
+        "message": "The request content itself is not processable by the server."
+    },
+    429: {
+        "raise_exception": ZendeskRateLimitError,
+        "message": "The API rate limit for your organisation/application pairing has been exceeded."
+    },
+    500: {
+        "raise_exception": ZendeskInternalServerError,
+        "message": "The server encountered an unexpected condition which prevented" \
+            " it from fulfilling the request."
+    },
+    501: {
+        "raise_exception": ZendeskNotImplementedError,
+        "message": "The server does not support the functionality required to fulfill the request."
+    },
+    502: {
+        "raise_exception": ZendeskBadGatewayError,
+        "message": "Server received an invalid response."
+    },
+    503: {
+        "raise_exception": ZendeskServiceUnavailableError,
+        "message": "API service is currently unavailable."
+    }
+}
 def is_fatal(exception):
     status_code = exception.response.status_code
 
@@ -15,16 +104,36 @@ def is_fatal(exception):
         LOGGER.info("Caught HTTP 429, retrying request in %s seconds", sleep_time)
         sleep(sleep_time)
         return False
-
+    elif status_code == 503:
+        sleep_time = int(exception.response.headers['Retry-After'])
+        LOGGER.info("Caught HTTP 503, retrying request in %s seconds", sleep_time)
+        sleep(sleep_time)
+        return False
     return 400 <=status_code < 500
 
+def check_status(response):
+    # Forming a response message for raising custom exception
+    try:
+        response_json = response.json()
+    except Exception: # pylint: disable=broad-except
+        response_json = {}
+    if response.status_code != 200:
+        message = "HTTP-error-code: {}, Error: {}".format(
+            response.status_code,
+            response_json.get("errorMessages", [ERROR_CODE_EXCEPTION_MAPPING.get(
+                response.status_code, {}).get("message", "Unknown Error")])[0]
+        )
+        exc = ERROR_CODE_EXCEPTION_MAPPING.get(
+            response.status_code, {}).get("raise_exception", ZendeskError)
+        raise exc(message, response) from None
+
 @backoff.on_exception(backoff.expo,
-                      requests.exceptions.HTTPError,
+                      ZendeskBackoffError,
                       max_tries=10,
                       giveup=is_fatal)
 def call_api(url, params, headers):
     response = requests.get(url, params=params, headers=headers)
-    response.raise_for_status()
+    check_status(response)
     return response
 
 

tap_zendesk/streams.py

@@ -16,6 +16,7 @@
 LOGGER = singer.get_logger()
 KEY_PROPERTIES = ['id']
 
+START_DATE = datetime.datetime.strftime(datetime.datetime.utcnow() - datetime.timedelta(days=1), "%Y-%m-%dT00:00:00Z")
 CUSTOM_TYPES = {
     'text': 'string',
     'textarea': 'string',
@@ -114,13 +115,33 @@ def load_metadata(self):
     def is_selected(self):
         return self.stream is not None
 
+    def check_access(self):
+        '''
+        Check whether permission given to access stream resource or not.
+        '''
+        url = self.endpoint.format(self.config['subdomain'])
+
+        headers = {
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+        'Authorization': 'Bearer {}'.format(self.config['access_token'])
+        }
+
+        http.call_api(url, params={'page[size]': 1}, headers=headers)
+
+
 def raise_or_log_zenpy_apiexception(schema, stream, e):
     # There are multiple tiers of Zendesk accounts. Some of them have
     # access to `custom_fields` and some do not. This is the specific
     # error that appears to be return from the API call in the event that
     # it doesn't have access.
     if not isinstance(e, zenpy.lib.exception.APIException):
         raise ValueError("Called with a bad exception type") from e
+    #If read permission not available in oauth access_token, then it returns below error.
+    if json.loads(e.args[0]).get('description') == "You are missing the following required scopes: read":
+        LOGGER.warning("The account credentials supplied do not have access to `%s` custom fields.",
+                            stream)
+        return schema
     if json.loads(e.args[0])['error']['message'] == "You do not have access to this page. Please contact the account owner of this help desk for further help.":
         LOGGER.warning("The account credentials supplied do not have access to `%s` custom fields.",
                        stream)
@@ -158,6 +179,8 @@ def sync(self, state):
             self.update_bookmark(state, organization.updated_at)
             yield (self.stream, organization)
 
+    def check_access(self):
+        self.client.organizations.incremental(start_time=START_DATE)
 
 class Users(Stream):
     name = "users"
@@ -235,6 +258,8 @@ def sync(self, state):
             start = end - datetime.timedelta(seconds=1)
             end = start + datetime.timedelta(seconds=search_window_size)
 
+    def check_access(self):
+        self.client.search("", updated_after=START_DATE, updated_before='2000-01-02T00:00:00Z', type="user")
 
 class Tickets(Stream):
     name = "tickets"
@@ -338,6 +363,9 @@ def emit_sub_stream_metrics(sub_stream):
         emit_sub_stream_metrics(comments_stream)
         singer.write_state(state)
 
+    def check_access(self):
+        self.client.tickets.incremental(start_time=START_DATE, paginate_by_time=False)
+
 class TicketAudits(Stream):
     name = "ticket_audits"
     replication_method = "INCREMENTAL"
@@ -349,6 +377,9 @@ def sync(self, ticket_id):
             self.count += 1
             yield (self.stream, ticket_audit)
 
+    def check_access(self):
+        self.client.tickets.audits(ticket=None)
+
 class TicketMetrics(Stream):
     name = "ticket_metrics"
     replication_method = "INCREMENTAL"
@@ -359,6 +390,9 @@ def sync(self, ticket_id):
         self.count += 1
         yield (self.stream, ticket_metric)
 
+    def check_access(self):
+        self.client.tickets.metrics(ticket=None)
+
 class TicketComments(Stream):
     name = "ticket_comments"
     replication_method = "INCREMENTAL"
@@ -370,6 +404,9 @@ def sync(self, ticket_id):
             self.count += 1
             yield (self.stream, ticket_comment)
 
+    def check_access(self):
+        self.client.tickets.comments(ticket=None)
+
 class SatisfactionRatings(Stream):
     name = "satisfaction_ratings"
     replication_method = "INCREMENTAL"
@@ -475,6 +512,9 @@ def sync(self, state):
                 self.update_bookmark(state, form.updated_at)
                 yield (self.stream, form)
 
+    def check_access(self):
+        self.client.ticket_forms()
+
 class GroupMemberships(Stream):
     name = "group_memberships"
     replication_method = "INCREMENTAL"
@@ -512,6 +552,9 @@ def sync(self, state): # pylint: disable=unused-argument
         for policy in self.client.sla_policies():
             yield (self.stream, policy)
 
+    def check_access(self):
+        self.client.sla_policies()
+
 STREAMS = {
     "tickets": Tickets,
     "groups": Groups,