From b9e8e6f7203e455a3602d9d5bb4e17e7a0ad71d6 Mon Sep 17 00:00:00 2001 From: Luishfs Date: Wed, 25 Sep 2024 10:44:18 -0300 Subject: [PATCH] source-impact-native: switching snapshots pk to _meta/row_id after fixing the tombstone value with the correct class, the connector still broke due to tombstone failures. The newer bugs happened because there were still a pk required that was not being fed to the newer tombstone model. In order to fix that, i've removed all previous pk from snapshot models and fixated all pks to _meta/row_id ( the default strategy ) --- .../source_impact_native/models.py | 19 +++----- ...ests_test_snapshots__discover__stdout.json | 45 +++---------------- 2 files changed, 10 insertions(+), 54 deletions(-) diff --git a/source-impact-native/source_impact_native/models.py b/source-impact-native/source_impact_native/models.py index 977a74c83..3d4474c65 100644 --- a/source-impact-native/source_impact_native/models.py +++ b/source-impact-native/source_impact_native/models.py @@ -64,9 +64,7 @@ class Ads(BaseDocument, extra="allow"): START_DATE_INCREMENTAL: ClassVar[str] = None END_DATE: ClassVar[str] = "" REP_KEY: ClassVar[str] = "" - PRIMARY_KEY: ClassVar[str] = "/Id" - - Id: str + PRIMARY_KEY: ClassVar[str] = "/_meta/row_id" class Catalogs(BaseDocument, extra="allow"): NAME: ClassVar[str] = "Catalogs" @@ -74,9 +72,7 @@ class Catalogs(BaseDocument, extra="allow"): START_DATE_INCREMENTAL: ClassVar[str] = None END_DATE: ClassVar[str] = "" REP_KEY: ClassVar[str] = "" - PRIMARY_KEY: ClassVar[str] = "/Id" - - Id: str + PRIMARY_KEY: ClassVar[str] = "/_meta/row_id" class Invoices(BaseDocument, extra="allow"): NAME: ClassVar[str] = "Invoices" @@ -137,9 +133,8 @@ class Campaigns(BaseDocument, extra="allow"): START_DATE: ClassVar[str] = "" END_DATE: ClassVar[str] = "" REP_KEY: ClassVar[str] = "" - PRIMARY_KEY: ClassVar[str] = "/Id" + PRIMARY_KEY: ClassVar[str] = "/_meta/row_id" - Id: str class Reports(BaseDocument, extra="allow"): # No query parameters available, and no date replication key @@ -167,9 +162,7 @@ class PhoneNumbers(BaseDocument, extra="allow"): START_DATE: ClassVar[str] = "" END_DATE: ClassVar[str] = "" REP_KEY: ClassVar[str] = "DateCreated" - PRIMARY_KEY: ClassVar[str] = "/Id" - - Id: str + PRIMARY_KEY: ClassVar[str] = "/_meta/row_id" class PromoCodes(BaseDocument, extra="allow"): NAME: ClassVar[str] = "PromoCodes" @@ -218,9 +211,7 @@ class MediaPartnerGroups(BaseDocument, extra="allow"): START_DATE: ClassVar[str] = "" END_DATE: ClassVar[str] = "" REP_KEY: ClassVar[str] = "" - PRIMARY_KEY: ClassVar[str] = "/Id" - - Id: str + PRIMARY_KEY: ClassVar[str] = "/_meta/row_id" class Notes(BaseDocument, extra="allow"): NAME: ClassVar[str] = "Notes" diff --git a/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__discover__stdout.json b/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__discover__stdout.json index 4d5a21c62..51fdcab8d 100644 --- a/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__discover__stdout.json +++ b/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__discover__stdout.json @@ -592,21 +592,14 @@ "row_id": -1 }, "description": "Document metadata" - }, - "Id": { - "title": "Id", - "type": "string" } }, - "required": [ - "Id" - ], "title": "Campaigns", "type": "object", "x-infer-schema": true }, "key": [ - "/Id" + "/_meta/row_id" ] }, { @@ -653,21 +646,14 @@ "row_id": -1 }, "description": "Document metadata" - }, - "Id": { - "title": "Id", - "type": "string" } }, - "required": [ - "Id" - ], "title": "Ads", "type": "object", "x-infer-schema": true }, "key": [ - "/Id" + "/_meta/row_id" ] }, { @@ -714,21 +700,14 @@ "row_id": -1 }, "description": "Document metadata" - }, - "Id": { - "title": "Id", - "type": "string" } }, - "required": [ - "Id" - ], "title": "Catalogs", "type": "object", "x-infer-schema": true }, "key": [ - "/Id" + "/_meta/row_id" ] }, { @@ -829,21 +808,14 @@ "row_id": -1 }, "description": "Document metadata" - }, - "Id": { - "title": "Id", - "type": "string" } }, - "required": [ - "Id" - ], "title": "PhoneNumbers", "type": "object", "x-infer-schema": true }, "key": [ - "/Id" + "/_meta/row_id" ] }, { @@ -1134,21 +1106,14 @@ "row_id": -1 }, "description": "Document metadata" - }, - "Id": { - "title": "Id", - "type": "string" } }, - "required": [ - "Id" - ], "title": "MediaPartnerGroups", "type": "object", "x-infer-schema": true }, "key": [ - "/Id" + "/_meta/row_id" ] } ]