changes consumer configuration

zendesk · May 30, 2022 · 084e79b · 084e79b
1 parent d85ab18
commit 084e79b
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,8 @@
 # Changelog
 
 ## Unreleased
+* [Racecar::Config] **breaking change** renames `max_bytes` configuration to `fetch_max_bytes` which is the correct name of the configuration that we pass to librdkafka.
+* [Racecar::ConsumerSet] removes `max_bytes_per_partition` hardcoded config and moves to [Racecar:::Config] with the correct naming `message_max_bytes`.
 
 ## v2.8.0
 * Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#283](https://github.com/zendesk/racecar/pull/286))

diff --git a/lib/racecar/config.rb b/lib/racecar/config.rb
@@ -63,7 +63,10 @@ class Config < KingKonf::Config
     float :message_timeout, default: 5*60
 
     desc "Maximum amount of data the broker shall return for a Fetch request"
-    integer :max_bytes, default: 10485760
+    integer :fetch_max_bytes, default: 10485760
+
+    desc "Maximum request message size"
+    integer :message_max_bytes, default: 1048576
 
     desc "A prefix used when generating consumer group names"
     string :group_id_prefix

diff --git a/lib/racecar/consumer_set.rb b/lib/racecar/consumer_set.rb
@@ -214,7 +214,8 @@ def rdkafka_config(subscription)
         "bootstrap.servers"       => @config.brokers.join(","),
         "client.id"               => @config.client_id,
         "enable.partition.eof"    => false,
-        "fetch.max.bytes"         => @config.max_bytes,
+        "message.max.bytes"       => @config.message_max_bytes,
+        "fetch.max.bytes"         => @config.fetch_max_bytes,
         "fetch.min.bytes"         => @config.fetch_min_bytes,
         "fetch.wait.max.ms"       => @config.max_wait_time_ms,
         "group.id"                => @config.group_id,