diff --git a/.gitignore b/.gitignore index 516ec934..68fece43 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ _rel/ *.log relx docker/ -TAGS \ No newline at end of file +TAGS +.vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 820e4874..c73416cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ # Changelog +- 3.19.1 + - Made brod-cli to work on OTP 26. [PR#582](https://github.com/kafka4beam/brod/pull/582) + - `--ssl` option is now mandatory if TLS is to be used (previously it can be derived from `--cacertfile` option) + - TLS version defaults to 1.2, added `--ssl-versions` to support explictly setting TLS 1.3 + - 3.19.0 - Forward unhandled messages in topic/group consumer processes to handle_info/2 callbacks - in order to support arbitrary message passing [PR#580](https://github.com/kafka4beam/brod/pull/580) + in order to support arbitrary message passing [PR#580](https://github.com/kafka4beam/brod/pull/580) - 3.18.0 - Add transactional APIs. [PR#549](https://github.com/kafka4beam/brod/pull/549) diff --git a/guides/examples/Authentication.md b/guides/examples/Authentication.md index e593514b..a3065655 100644 --- a/guides/examples/Authentication.md +++ b/guides/examples/Authentication.md @@ -60,6 +60,7 @@ For more info see the Erlang Ecosystem Foundation's [server certificate verifica , { depth, 3 } , { customize_hostname_check, [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]} + , {version, ['tlsv1.3', 'tlsv1.2']} ]} , { sasl, {plain, "GFRW5BSQHKEH0TSG", "GrL3CNTkLhsvtBr8srGn0VilMpgDb4lPD"}} ] diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml index 272f0b4f..34d28561 100644 --- a/scripts/docker-compose.yml +++ b/scripts/docker-compose.yml @@ -1,35 +1,17 @@ version: "2" services: - pause: - image: "gcr.io/google_containers/pause-amd64:3.0" - networks: - - pausenet - ports: - - "2181:2181" - - "9092:9092" - - "9093:9093" - - "9094:9094" - - "9095:9095" - - "9192:9192" - - "9193:9193" - - "9194:9194" - - "9195:9195" - container_name: pause zookeeper: - depends_on: - - pause image: "zmstone/kafka:${KAFKA_VERSION}" container_name: zookeeper command: run zookeeper - network_mode: service:pause + network_mode: host kafka_1: depends_on: - - pause - zookeeper image: "zmstone/kafka:${KAFKA_VERSION}" container_name: "kafka-1" - network_mode: service:pause + network_mode: host environment: BROKER_ID: 0 PLAINTEXT_PORT: 9092 @@ -40,11 +22,10 @@ services: ZOOKEEPER_CONNECT: "localhost:2181" kafka_2: depends_on: - - pause - zookeeper image: "zmstone/kafka:${KAFKA_VERSION}" container_name: "kafka-2" - network_mode: service:pause + network_mode: host environment: BROKER_ID: 1 PLAINTEXT_PORT: 9192 @@ -53,6 +34,3 @@ services: SASL_PLAINTEXT_PORT: 9195 ADVERTISED_HOSTNAME: localhost ZOOKEEPER_CONNECT: "localhost:2181" - -networks: - pausenet: diff --git a/scripts/setup-test-env.sh b/scripts/setup-test-env.sh index b39fe63e..b877ef2f 100755 --- a/scripts/setup-test-env.sh +++ b/scripts/setup-test-env.sh @@ -18,7 +18,7 @@ function docker_compose { fi } -VERSION=${KAFKA_VERSION:-1.1} +VERSION=${KAFKA_VERSION:-2.4} if [ -z $VERSION ]; then VERSION=$1; fi case $VERSION in diff --git a/src/brod_cli.erl b/src/brod_cli.erl index b28b6302..e3f3cef8 100644 --- a/src/brod_cli.erl +++ b/src/brod_cli.erl @@ -61,6 +61,8 @@ commands: %% NOTE: bad indentation at the first line is intended -define(COMMAND_COMMON_OPTIONS, " --ssl Use TLS, validate server using trusted CAs + --ssl-versions= Specify SSL versions. Comma separated versions, + e.g. 1.3,1.2 --cacertfile= Use TLS, validate server using the given certificate --certfile= Client certificate in case client authentication is enabled in brokers @@ -365,6 +367,7 @@ main(Command, Doc, Args, Stop, LogLevel) -> C1 : E1 ?BIND_STACKTRACE(Stack1) -> ?GET_STACKTRACE(Stack1), verbose("~p:~p\n~p\n", [C1, E1, Stack1]), + io:format(user, "~p~n", [{C1, E1, Stack1}]), ?STOP(Stop) end, case LogLevel =:= ?LOG_LEVEL_QUIET of @@ -1125,20 +1128,25 @@ parse_offset_time(T) -> int(T). parse_connection_config(Args) -> SslBool = parse(Args, "--ssl", fun parse_boolean/1), + SslVersions = parse(Args, "--ssl-versions", fun parse_ssl_versions/1), CaCertFile = parse(Args, "--cacertfile", fun parse_file/1), CertFile = parse(Args, "--certfile", fun parse_file/1), KeyFile = parse(Args, "--keyfile", fun parse_file/1), FilterPred = fun({_, V}) -> V =/= ?undef end, SslOpt = - case CaCertFile of - ?undef -> - SslBool; - _ -> - Files = + case SslBool of + true -> + Opts = [{cacertfile, CaCertFile}, {certfile, CertFile}, - {keyfile, KeyFile}], - lists:filter(FilterPred, Files) + {keyfile, KeyFile}, + {versions, SslVersions}, + %% TODO: verify_peer if cacertfile is provided + {verify, verify_none} + ], + lists:filter(FilterPred, Opts); + false -> + false end, SaslPlain = parse(Args, "--sasl-plain", fun parse_file/1), SaslScram256 = parse(Args, "--scram256", fun parse_file/1), @@ -1157,12 +1165,31 @@ parse_boolean(true) -> true; parse_boolean(false) -> false; parse_boolean("true") -> true; parse_boolean("false") -> false; -parse_boolean(?undef) -> ?undef. +parse_boolean(?undef) -> false. parse_cg_ids("") -> []; parse_cg_ids("all") -> all; parse_cg_ids(Str) -> [bin(I) || I <- string:tokens(Str, ",")]. +parse_ssl_versions(?undef) -> + parse_ssl_versions(""); +parse_ssl_versions(Versions) -> + case lists:map(fun parse_ssl_version/1, string:tokens(Versions, ", ")) of + [] -> + ['tlsv1.2']; + Vsns -> + Vsns + end. + +parse_ssl_version("1.2") -> + 'tlsv1.2'; +parse_ssl_version("1.3") -> + 'tlsv1.3'; +parse_ssl_version("1.1") -> + 'tlsv1.1'; +parse_ssl_version(Other) -> + error({unsupported_tls_version, Other}). + parse_file(?undef) -> ?undef; parse_file(Path) -> diff --git a/test/brod_cli_tests.erl b/test/brod_cli_tests.erl index a9615283..4d382fb2 100644 --- a/test/brod_cli_tests.erl +++ b/test/brod_cli_tests.erl @@ -37,9 +37,12 @@ meta_test() -> ssl_test() -> run(["meta", "-b", "localhost:9093", "-L", - "--cacertfile", "priv/ssl/ca.crt", - "--keyfile", "priv/ssl/client.key", - "--certfile", "priv/ssl/client.crt"]). + "--ssl", + "--cacertfile", "test/data/ssl/ca.pem", + "--keyfile", "test/data/ssl/client-key.pem", + "--certfile", "test/data/ssl/client-crt.pem", + "--ssl-versions", "1.2,1.1" + ]). offset_test() -> Args = ["offset", "-b", "localhost", "-t", "test-topic", "-p", "0"], @@ -74,9 +77,10 @@ test_sasl() -> Output = cmd(["send", "--brokers", "localhost:9194,localhost:9094", "-t", "test-topic", "-p", "0", - "--cacertfile", "priv/ssl/ca.crt", - "--keyfile", "priv/ssl/client.key", - "--certfile", "priv/ssl/client.crt", + "--ssl", + "--cacertfile", "test/data/ssl/ca.pem", + "--keyfile", "test/data/ssl/client-key.pem", + "--certfile", "test/data/ssl/client-crt.pem", "--sasl-plain", "sasl.testdata", "-k", K, "-v", V]), ?assertEqual(<<"">>, Output), diff --git a/test/brod_client_SUITE.erl b/test/brod_client_SUITE.erl index dfef00aa..eff92257 100644 --- a/test/brod_client_SUITE.erl +++ b/test/brod_client_SUITE.erl @@ -385,11 +385,13 @@ t_magic_version(Config) when is_list(Config) -> auth(_Host, _Sock, _Mod, _ClientId, _Timeout, _Opts) -> ok. ssl_options() -> - PrivDir = code:priv_dir(brod), - Fname = fun(Name) -> filename:join([PrivDir, ssl, Name]) end, - [ {cacertfile, Fname("ca.crt")} - , {keyfile, Fname("client.key")} - , {certfile, Fname("client.crt")} + LibDir = code:lib_dir(brod), + Fname = fun(Name) -> filename:join([LibDir, test, data, ssl, Name]) end, + [ {cacertfile, Fname("ca.pem")} + , {keyfile, Fname("client-key.pem")} + , {certfile, Fname("client-crt.pem")} + , {versions, ['tlsv1.2']} + , {verify, verify_none} ]. produce_and_consume_message(Host, Client, ClientConfig) -> diff --git a/priv/ssl/ca.crt b/test/data/ssl/ca.pem similarity index 100% rename from priv/ssl/ca.crt rename to test/data/ssl/ca.pem diff --git a/priv/ssl/client.crt b/test/data/ssl/client-crt.pem similarity index 100% rename from priv/ssl/client.crt rename to test/data/ssl/client-crt.pem diff --git a/priv/ssl/client.key b/test/data/ssl/client-key.pem similarity index 100% rename from priv/ssl/client.key rename to test/data/ssl/client-key.pem