From 11c69b2c8f9b1ca317c63834668ebb01acc551a0 Mon Sep 17 00:00:00 2001
From: uzair khan <uzair.khan@juspay.in>
Date: Thu, 17 Oct 2024 15:51:04 +0530
Subject: [PATCH 1/2] feat(config): add transform before sending sdk logs

- change the merchant_id value that contains publishable_key to actual
merchant_id value
- an sdk_map.csv is used to keep map of publishable_key -> merchant_id
---
 config/vector.yaml | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/config/vector.yaml b/config/vector.yaml
index 54ff25cab5a..46128b24326 100644
--- a/config/vector.yaml
+++ b/config/vector.yaml
@@ -1,5 +1,16 @@
 acknowledgements:
   enabled: true
+enrichment_tables:
+  sdk_map:
+    type: file
+    file:
+      path: /etc/vector/config/sdk_map.csv
+      encoding:
+        type: csv
+    schema:
+      publishable_key: string
+      merchant_id: string
+
 
 api:
   enabled: true
@@ -69,6 +80,21 @@ transforms:
     key_field: "{{ .payment_id }}{{ .merchant_id }}"
     threshold: 1000
     window_secs: 60
+  
+  amend_sdk_logs:
+    type: remap
+    inputs:
+      - sdk_transformed
+    source: |
+      .before_transform = now()
+
+      merchant_id = .merchant_id
+      row = get_enrichment_table_record!("sdk_map", { "publishable_key" : merchant_id }, case_sensitive: true) 
+      .merchant_id = row.merchant_id
+
+      .after_transform = now()
+
+  
 
 sinks:
   opensearch_events_1:
@@ -178,7 +204,7 @@ sinks:
         - "path"
         - "source_type"
     inputs:
-      - "sdk_transformed"
+      - "amend_sdk_logs"
     bootstrap_servers: kafka0:29092
     topic: hyper-sdk-logs
     key_field: ".merchant_id"

From af5772a43a4dd532f3c62983cb17ddb03d19bdd7 Mon Sep 17 00:00:00 2001
From: uzair khan <uzair.khan@juspay.in>
Date: Thu, 17 Oct 2024 19:39:43 +0530
Subject: [PATCH 2/2] feat(config): add more opensearch log ignore fields

---
 config/vector.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/config/vector.yaml b/config/vector.yaml
index 46128b24326..8a4f37dd793 100644
--- a/config/vector.yaml
+++ b/config/vector.yaml
@@ -118,6 +118,9 @@ sinks:
         - offset
         - partition
         - topic
+        - clickhouse_database
+        - last_synced
+        - sign_flag
     bulk:
       index: "vector-{{ .topic }}"
 
@@ -142,6 +145,9 @@ sinks:
         - offset
         - partition
         - topic
+        - clickhouse_database
+        - last_synced
+        - sign_flag
     bulk:
       # Add a date suffixed index for better grouping
       index: "vector-{{ .topic }}-%Y-%m-%d"