Skip to content

Commit

Permalink
source-zendesk-support: add logging & ticket_comments checkpoint
Browse files Browse the repository at this point in the history
For users with a lot of data, the `ticket_comments` stream can take a
while to backfill, especially given the 30 requests/60 seconds rate
limit for this endpoint. Checkpointing that stream will help if that
backfill complete over connector restarts, and it'll provide an
indicator in the UI that the connector is still processing records.

I've also added logging to catch if the `ticket_comments` cursor does
not increase & to log the status codes for non-200 responses to help
troubleshoot if the connector is stuck making the same failing request.
  • Loading branch information
Alex-Bair committed Oct 25, 2024
1 parent 4fa0a96 commit 3cf40b4
Showing 1 changed file with 12 additions and 0 deletions.
12 changes: 12 additions & 0 deletions source-zendesk-support/source_zendesk_support/streams.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ def should_retry(self, response: requests.Response) -> bool:
self.logger.error(f"Skipping stream {self.name}: Check permissions, error message: {error}.")
setattr(self, "raise_on_http_errors", False)
return False
if response.status_code != 200:
self.logger.warning(f"Received a {response.status_code} response.")
return super().should_retry(response)


Expand Down Expand Up @@ -671,6 +673,8 @@ class SourceZendeskSupportTicketEventsExportStream(SourceZendeskIncrementalExpor
@ param event_type : specific event_type to check ["Audit", "Change", "Comment", etc]
"""

state_checkpoint_interval = 1000

cursor_field = "created_at"
response_list_name: str = "ticket_events"
response_target_entity: str = "child_events"
Expand Down Expand Up @@ -707,6 +711,14 @@ def request_params(
Otherwise, returns the start time param from the stream's state/config and the sideload param.
"""
if next_page_token:
# Check if the next_page_token's start time is the the same or earlier than
# the previous request's / checkpointed state's start time.
next_page_start_time = int(next_page_token.get("start_time"))
checkpointed_start_time = self.check_stream_state(stream_state=stream_state)

if next_page_start_time <= checkpointed_start_time:
self.logger.warning(f"start_time query param {next_page_start_time} is less than or equal to than previous start_time param {checkpointed_start_time}. Check if the stream is stuck in a loop.")

return next_page_token

start_time = self.check_stream_state(stream_state=stream_state)
Expand Down

0 comments on commit 3cf40b4

Please sign in to comment.