Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(config): update vector config #6365

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(config): update vector config #6365

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11c69b2
feat(config): add transform before sending sdk logs
maverox Oct 17, 2024
af5772a
feat(config): add more opensearch log ignore fields
maverox Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion config/vector.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
acknowledgements:
enabled: true
enrichment_tables:
sdk_map:
type: file
file:
path: /etc/vector/config/sdk_map.csv
encoding:
type: csv
schema:
publishable_key: string
merchant_id: string


api:
enabled: true
Expand Down Expand Up @@ -69,6 +80,21 @@ transforms:
key_field: "{{ .payment_id }}{{ .merchant_id }}"
threshold: 1000
window_secs: 60

amend_sdk_logs:
type: remap
inputs:
- sdk_transformed
source: |
.before_transform = now()

merchant_id = .merchant_id
row = get_enrichment_table_record!("sdk_map", { "publishable_key" : merchant_id }, case_sensitive: true)
.merchant_id = row.merchant_id

.after_transform = now()



sinks:
opensearch_events_1:
Expand All @@ -92,6 +118,9 @@ sinks:
- offset
- partition
- topic
- clickhouse_database
- last_synced
- sign_flag
bulk:
index: "vector-{{ .topic }}"

Expand All @@ -116,6 +145,9 @@ sinks:
- offset
- partition
- topic
- clickhouse_database
- last_synced
- sign_flag
bulk:
# Add a date suffixed index for better grouping
index: "vector-{{ .topic }}-%Y-%m-%d"
Expand Down Expand Up @@ -178,7 +210,7 @@ sinks:
- "path"
- "source_type"
inputs:
- "sdk_transformed"
- "amend_sdk_logs"
bootstrap_servers: kafka0:29092
topic: hyper-sdk-logs
key_field: ".merchant_id"
Loading