From 8b0e007d4289d5bb212dc4507d25b2da3643a6ad Mon Sep 17 00:00:00 2001 From: Kyle Allan Date: Thu, 14 Mar 2019 18:41:10 +0000 Subject: [PATCH] log that we're skipping rows every so often --- tap_zendesk/streams.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 43efb25..8e1d314 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -301,6 +301,7 @@ def sync(self, state): bookmark = self.get_bookmark(state) satisfaction_ratings = self.client.satisfaction_ratings() + skipped_count = 0 for satisfaction_rating in satisfaction_ratings: if utils.strptime_with_tz(satisfaction_rating.updated_at) >= bookmark: # NB: We don't trust that the records come back ordered by @@ -308,6 +309,10 @@ def sync(self, state): # so we can't save state until we've seen all records self.update_bookmark(state, satisfaction_rating.updated_at) yield (self.stream, satisfaction_rating) + else: + skipped_count += 1 + if skipped_count % 10000 == 0: + LOGGER.info("Skipped 10,000 rows as they occur before bookmark: %s", bookmark) class Groups(Stream): name = "groups"