From 6893d31626623f1225ee2c3d336438bb2b3b3088 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Crdeshmukh15=E2=80=9D?=
 <“rutuja.deshmukh@qlik.com”>
Date: Wed, 30 Oct 2024 04:01:32 +0000
Subject: [PATCH] pagination continuation logic on refresh token

---
 tap_s3_csv/s3.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tap_s3_csv/s3.py b/tap_s3_csv/s3.py
index 272637b..fbf03c9 100644
--- a/tap_s3_csv/s3.py
+++ b/tap_s3_csv/s3.py
@@ -559,16 +559,24 @@ def create_s3_client():
     paginator = s3_client.get_paginator('list_objects_v2')
     LOGGER.info("in list_files_in_bucket.......2")
     pages = 0
+    continuation_token = None
 
     while True:
         try:
+            if continuation_token:
+                args['ContinuationToken'] = continuation_token
+
             for page in paginator.paginate(**args):
                 LOGGER.info("in list_files_in_bucket.......3")
                 pages += 1
                 LOGGER.debug("On page %s", pages)
                 s3_object_count += len(page.get('Contents', []))
                 yield from page.get('Contents', [])
-            break  # Break if pagination is successful
+                LOGGER.info("sleeping for 15 mins in list function")
+                # time.sleep(950)
+                continuation_token = page.get('NextContinuationToken')
+                # time.sleep(950)
+            break
         except ClientError as e:
             # Check if the error is due to an expired token
             if e.response['Error']['Code'] == 'ExpiredToken':
@@ -576,7 +584,7 @@ def create_s3_client():
                 refresh_session(config)
                 # Re-create the S3 client with new credentials
                 s3_client = create_s3_client()
-                paginator = s3_client.get_paginator('list_objects_v2')  # Recreate paginator with new client
+                paginator = s3_client.get_paginator('list_objects_v2')
             else:
                 LOGGER.error("Failed to list files: %s", e)
                 raise
@@ -598,6 +606,7 @@ def create_s3_resource():
     bucket = config['bucket']
     s3_bucket = s3_resource.Bucket(bucket)
     s3_object = s3_bucket.Object(s3_path)
+    # time.sleep(950)
 
     while True:
         try: