From 5df6d6b59e79f80e0e4ef47df2b69198f49929fb Mon Sep 17 00:00:00 2001 From: Eray Ates Date: Wed, 2 Oct 2024 13:06:47 +0200 Subject: [PATCH] feat: wkafka handler ui Signed-off-by: Eray Ates --- Makefile | 40 +--- README.md | 4 +- client.go | 2 +- config.go | 6 +- consumer.go | 14 +- handler/gen/wkafka/wkafka.pb.go | 219 ++++++++++++------ .../wkafka/wkafkaconnect/wkafka.connect.go | 36 ++- handler/handler.go | 38 +-- handler/proto/wkafka/wkafka.proto | 8 +- 9 files changed, 230 insertions(+), 137 deletions(-) diff --git a/Makefile b/Makefile index 1ef7d9d..fbada0d 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,10 @@ -LOCAL_BIN_DIR := $(PWD)/bin - -## golangci configuration -GOLANGCI_CONFIG_URL := https://raw.githubusercontent.com/worldline-go/guide/main/lint/.golangci.yml -GOLANGCI_LINT_VERSION := v1.55.2 - .DEFAULT_GOAL := help +.PHONY: generate +generate: ## Generate gRPC code + cd handler/proto && buf generate --path wkafka/wkafka.proto + @echo "> Successfully generated gRPC code" + .PHONY: env env: ## Start env docker compose -p wkafka -f env/docker-compose.yml up -d @@ -31,21 +30,10 @@ example: ## Run example ci-run: ## Run CI in local with act act -j sonarcloud -.golangci.yml: - @$(MAKE) golangci - -.PHONY: golangci -golangci: ## Download .golangci.yml file - @curl --insecure -o .golangci.yml -L'#' $(GOLANGCI_CONFIG_URL) - -bin/golangci-lint-$(GOLANGCI_LINT_VERSION): - @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(LOCAL_BIN_DIR) $(GOLANGCI_LINT_VERSION) - @mv $(LOCAL_BIN_DIR)/golangci-lint $(LOCAL_BIN_DIR)/golangci-lint-$(GOLANGCI_LINT_VERSION) - .PHONY: lint -lint: .golangci.yml bin/golangci-lint-$(GOLANGCI_LINT_VERSION) ## Lint Go files - @$(LOCAL_BIN_DIR)/golangci-lint-$(GOLANGCI_LINT_VERSION) --version - @GOPATH="$(shell dirname $(PWD))" $(LOCAL_BIN_DIR)/golangci-lint-$(GOLANGCI_LINT_VERSION) run ./... +lint: ## Lint Go files + golangci-lint--version + GOPATH="$(shell dirname $(PWD))" golangci-lint run ./... .PHONY: test test: ## Run unit tests @@ -64,18 +52,6 @@ coverage: ## Run unit tests with coverage @go test -v -race -cover -coverpkg=./... -coverprofile=coverage.out -covermode=atomic ./... @go tool cover -func=coverage.out -.PHONY: html -html: ## Show html coverage result - @go tool cover -html=./coverage.out - -.PHONY: html-gen -html-gen: ## Export html coverage result - @go tool cover -html=./coverage.out -o ./coverage.html - -.PHONY: html-wsl -html-wsl: html-gen ## Open html coverage result in wsl - @explorer.exe `wslpath -w ./coverage.html` || true - .PHONY: help help: ## Display this help screen @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index b0dfde5..24c59d4 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ max_poll_records: 0 # if this value is more than max_poll_records then max_poll_records will be used batch_count: 100 dlq: - disable: false # disable dead letter queue + disabled: false # disable dead letter queue topic: "" # dead letter topic name, it can be assigned in the kafka config's format_dlq_topic retry_interval: "10s" # retry time interval of the message if can't be processed, default is 10s start_offset: 0 # -1 to start end of the offsets @@ -121,6 +121,8 @@ Send record to dead letter queue, use __WrapErrDLQ__ function with to wrap the e Editing the skip map and use our handler to initialize server mux. ```go +// import github.com/worldline-go/wkafka/handler + mux := http.NewServeMux() mux.Handle(handler.New(client)) diff --git a/client.go b/client.go index 84c81e2..264bd1e 100644 --- a/client.go +++ b/client.go @@ -54,7 +54,7 @@ func New(ctx context.Context, cfg Config, opts ...Option) (*Client, error) { return nil, fmt.Errorf("validate config: %w", err) } - if !o.ConsumerConfig.DLQ.Disable { + if !o.ConsumerConfig.DLQ.Disabled { o.ConsumerDLQEnabled = true } } diff --git a/config.go b/config.go index 015077f..52c1738 100644 --- a/config.go +++ b/config.go @@ -48,13 +48,13 @@ func configApply(c ConsumerPreConfig, consumerConfig *ConsumerConfig, progName s consumerConfig.GroupID = c.PrefixGroupID + consumerConfig.GroupID } - if !consumerConfig.DLQ.Disable && consumerConfig.DLQ.Topic == "" && c.FormatDLQTopic == "" { - consumerConfig.DLQ.Disable = true + if !consumerConfig.DLQ.Disabled && consumerConfig.DLQ.Topic == "" && c.FormatDLQTopic == "" { + consumerConfig.DLQ.Disabled = true logger.Warn("dlq is disabled because topic and format_dlq_topic is not set") } // add default topic name for DLQ - if !consumerConfig.DLQ.Disable { + if !consumerConfig.DLQ.Disabled { if consumerConfig.DLQ.Topic == "" { if c.FormatDLQTopic == "" { return fmt.Errorf("format_dlq_topic is required if dlq topic is not set") diff --git a/consumer.go b/consumer.go index 111e58d..98f55bb 100644 --- a/consumer.go +++ b/consumer.go @@ -53,11 +53,11 @@ type ConsumerConfig struct { } type DLQConfig struct { - // Disable is a flag to disable DLQ. + // Disabled is a flag to disable DLQ. // - Default is false. // - If topic is not set, it will be generated from format_dlq_topic. // - If topic and format_dlq_topic is not set, dlq will be disabled! - Disable bool `cfg:"disable"` + Disabled bool `cfg:"disabled"` // RetryInterval is a time interval to retry again of DLQ messages. // - Default is 10 seconds. RetryInterval time.Duration `cfg:"retry_interval"` @@ -158,7 +158,7 @@ func WithCallbackBatch[T any](fn func(ctx context.Context, msg []T) error) CallB Meter: o.Meter, } - if o.ConsumerConfig.DLQ.Disable { + if o.ConsumerConfig.DLQ.Disabled { return nil } @@ -166,7 +166,7 @@ func WithCallbackBatch[T any](fn func(ctx context.Context, msg []T) error) CallB Decode: decode, ProcessDLQ: dlqProcessBatch(fn), Cfg: o.ConsumerConfig, - Skip: newSkipper(&o.Client.consumerMutex, o.ConsumerConfig.DLQ.Disable), + Skip: newSkipper(&o.Client.consumerMutex, o.ConsumerConfig.DLQ.Disabled), IsDLQ: true, Logger: o.Client.logger, PartitionHandler: o.Client.partitionHandlerDLQ, @@ -195,7 +195,7 @@ func WithCallback[T any](fn func(ctx context.Context, msg T) error) CallBackFunc Meter: o.Meter, } - if o.ConsumerConfig.DLQ.Disable { + if o.ConsumerConfig.DLQ.Disabled { return nil } @@ -203,7 +203,7 @@ func WithCallback[T any](fn func(ctx context.Context, msg T) error) CallBackFunc ProcessDLQ: fn, Decode: decode, Cfg: o.ConsumerConfig, - Skip: newSkipper(&o.Client.consumerMutex, o.ConsumerConfig.DLQ.Disable), + Skip: newSkipper(&o.Client.consumerMutex, o.ConsumerConfig.DLQ.Disabled), IsDLQ: true, Logger: o.Client.logger, PartitionHandler: o.Client.partitionHandlerDLQ, @@ -226,7 +226,7 @@ func getDecodeProduceDLQ[T any](o *optionConsumer) (func(raw []byte, r *kgo.Reco } var produceDLQ func(ctx context.Context, err *DLQError, records []*kgo.Record) error - if !o.ConsumerConfig.DLQ.Disable { + if !o.ConsumerConfig.DLQ.Disabled { produceDLQ = producerDLQ(o.ConsumerConfig.DLQ.Topic, o.Client.clientID, o.Client.ProduceRaw) } diff --git a/handler/gen/wkafka/wkafka.pb.go b/handler/gen/wkafka/wkafka.pb.go index 4c495f5..19547ed 100644 --- a/handler/gen/wkafka/wkafka.pb.go +++ b/handler/gen/wkafka/wkafka.pb.go @@ -9,6 +9,7 @@ package wkafka import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" ) @@ -71,9 +72,8 @@ type CreateSkipRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Topics map[string]*Topic `protobuf:"bytes,1,rep,name=topics,proto3" json:"topics,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - Option SkipOption `protobuf:"varint,2,opt,name=option,proto3,enum=wkafka.SkipOption" json:"option,omitempty"` - EnableMainTopics bool `protobuf:"varint,3,opt,name=enable_main_topics,json=enableMainTopics,proto3" json:"enable_main_topics,omitempty"` + Topics map[string]*Topic `protobuf:"bytes,1,rep,name=topics,proto3" json:"topics,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Option SkipOption `protobuf:"varint,2,opt,name=option,proto3,enum=wkafka.SkipOption" json:"option,omitempty"` } func (x *CreateSkipRequest) Reset() { @@ -122,13 +122,6 @@ func (x *CreateSkipRequest) GetOption() SkipOption { return SkipOption_APPEND } -func (x *CreateSkipRequest) GetEnableMainTopics() bool { - if x != nil { - return x.EnableMainTopics - } - return false -} - type Topic struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -278,59 +271,118 @@ func (x *Response) GetMessage() string { return "" } +type SkipListResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Skip map[string]*Topic `protobuf:"bytes,1,rep,name=skip,proto3" json:"skip,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *SkipListResponse) Reset() { + *x = SkipListResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_wkafka_wkafka_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SkipListResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SkipListResponse) ProtoMessage() {} + +func (x *SkipListResponse) ProtoReflect() protoreflect.Message { + mi := &file_wkafka_wkafka_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SkipListResponse.ProtoReflect.Descriptor instead. +func (*SkipListResponse) Descriptor() ([]byte, []int) { + return file_wkafka_wkafka_proto_rawDescGZIP(), []int{4} +} + +func (x *SkipListResponse) GetSkip() map[string]*Topic { + if x != nil { + return x.Skip + } + return nil +} + var File_wkafka_wkafka_proto protoreflect.FileDescriptor var file_wkafka_wkafka_proto_rawDesc = []byte{ 0x0a, 0x13, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2f, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x22, 0xf6, 0x01, - 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x06, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, - 0x6f, 0x70, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x74, 0x6f, 0x70, 0x69, - 0x63, 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x53, 0x6b, 0x69, 0x70, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, - 0x0a, 0x12, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x74, 0x6f, - 0x70, 0x69, 0x63, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x65, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x4d, 0x61, 0x69, 0x6e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x1a, 0x48, 0x0a, 0x0b, - 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x77, - 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x98, 0x01, 0x0a, 0x05, 0x54, 0x6f, 0x70, 0x69, 0x63, - 0x12, 0x3d, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x54, 0x6f, - 0x70, 0x69, 0x63, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, - 0x50, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x50, 0x61, 0x72, - 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x3d, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, - 0x0a, 0x07, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, - 0x07, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x65, 0x66, 0x6f, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, - 0x22, 0x24, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2a, 0x25, 0x0a, 0x0a, 0x53, 0x6b, 0x69, 0x70, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, - 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x01, 0x32, 0x44, 0x0a, - 0x0d, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x33, - 0x0a, 0x04, 0x53, 0x6b, 0x69, 0x70, 0x12, 0x19, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x10, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x84, 0x01, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x77, 0x6b, 0x61, 0x66, - 0x6b, 0x61, 0x42, 0x0b, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, - 0x01, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x77, 0x6f, - 0x72, 0x6c, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x67, 0x6f, 0x2f, 0x77, 0x6b, 0x61, 0x66, 0x6b, - 0x61, 0x2f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x77, 0x6b, - 0x61, 0x66, 0x6b, 0x61, 0xa2, 0x02, 0x03, 0x57, 0x58, 0x58, 0xaa, 0x02, 0x06, 0x57, 0x6b, 0x61, - 0x66, 0x6b, 0x61, 0xca, 0x02, 0x06, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0xe2, 0x02, 0x12, 0x57, - 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0xea, 0x02, 0x06, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x1a, 0x1b, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, + 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc8, 0x01, 0x0a, 0x11, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x3d, 0x0a, 0x06, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x25, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x53, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, 0x6f, 0x70, 0x69, + 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x12, + 0x2a, 0x0a, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x12, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x53, 0x6b, 0x69, 0x70, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x48, 0x0a, 0x0b, 0x54, + 0x6f, 0x70, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x77, 0x6b, + 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x98, 0x01, 0x0a, 0x05, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x12, + 0x3d, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x54, 0x6f, 0x70, + 0x69, 0x63, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x50, + 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x50, 0x61, 0x72, 0x74, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x3d, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, + 0x07, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x07, + 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x22, + 0x24, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x92, 0x01, 0x0a, 0x10, 0x53, 0x6b, 0x69, 0x70, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x73, 0x6b, + 0x69, 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, + 0x61, 0x2e, 0x53, 0x6b, 0x69, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x73, 0x6b, + 0x69, 0x70, 0x1a, 0x46, 0x0a, 0x09, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x23, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0d, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0x25, 0x0a, 0x0a, 0x53, 0x6b, + 0x69, 0x70, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, + 0x4e, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, + 0x01, 0x32, 0x82, 0x01, 0x0a, 0x0d, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x53, 0x6b, 0x69, 0x70, 0x12, 0x19, 0x2e, 0x77, 0x6b, + 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x08, 0x53, 0x6b, 0x69, 0x70, + 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x77, + 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x2e, 0x53, 0x6b, 0x69, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x84, 0x01, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x77, + 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x42, 0x0b, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x67, 0x6f, 0x2f, 0x77, 0x6b, + 0x61, 0x66, 0x6b, 0x61, 0x2f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x2f, 0x67, 0x65, 0x6e, + 0x2f, 0x77, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0xa2, 0x02, 0x03, 0x57, 0x58, 0x58, 0xaa, 0x02, 0x06, + 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0xca, 0x02, 0x06, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0xe2, + 0x02, 0x12, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x06, 0x57, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -346,29 +398,36 @@ func file_wkafka_wkafka_proto_rawDescGZIP() []byte { } var file_wkafka_wkafka_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_wkafka_wkafka_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_wkafka_wkafka_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_wkafka_wkafka_proto_goTypes = []any{ (SkipOption)(0), // 0: wkafka.SkipOption (*CreateSkipRequest)(nil), // 1: wkafka.CreateSkipRequest (*Topic)(nil), // 2: wkafka.Topic (*Partition)(nil), // 3: wkafka.Partition (*Response)(nil), // 4: wkafka.Response - nil, // 5: wkafka.CreateSkipRequest.TopicsEntry - nil, // 6: wkafka.Topic.PartitionsEntry + (*SkipListResponse)(nil), // 5: wkafka.SkipListResponse + nil, // 6: wkafka.CreateSkipRequest.TopicsEntry + nil, // 7: wkafka.Topic.PartitionsEntry + nil, // 8: wkafka.SkipListResponse.SkipEntry + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_wkafka_wkafka_proto_depIdxs = []int32{ - 5, // 0: wkafka.CreateSkipRequest.topics:type_name -> wkafka.CreateSkipRequest.TopicsEntry + 6, // 0: wkafka.CreateSkipRequest.topics:type_name -> wkafka.CreateSkipRequest.TopicsEntry 0, // 1: wkafka.CreateSkipRequest.option:type_name -> wkafka.SkipOption - 6, // 2: wkafka.Topic.partitions:type_name -> wkafka.Topic.PartitionsEntry - 2, // 3: wkafka.CreateSkipRequest.TopicsEntry.value:type_name -> wkafka.Topic - 3, // 4: wkafka.Topic.PartitionsEntry.value:type_name -> wkafka.Partition - 1, // 5: wkafka.WkafkaService.Skip:input_type -> wkafka.CreateSkipRequest - 4, // 6: wkafka.WkafkaService.Skip:output_type -> wkafka.Response - 6, // [6:7] is the sub-list for method output_type - 5, // [5:6] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 7, // 2: wkafka.Topic.partitions:type_name -> wkafka.Topic.PartitionsEntry + 8, // 3: wkafka.SkipListResponse.skip:type_name -> wkafka.SkipListResponse.SkipEntry + 2, // 4: wkafka.CreateSkipRequest.TopicsEntry.value:type_name -> wkafka.Topic + 3, // 5: wkafka.Topic.PartitionsEntry.value:type_name -> wkafka.Partition + 2, // 6: wkafka.SkipListResponse.SkipEntry.value:type_name -> wkafka.Topic + 1, // 7: wkafka.WkafkaService.Skip:input_type -> wkafka.CreateSkipRequest + 9, // 8: wkafka.WkafkaService.SkipList:input_type -> google.protobuf.Empty + 4, // 9: wkafka.WkafkaService.Skip:output_type -> wkafka.Response + 5, // 10: wkafka.WkafkaService.SkipList:output_type -> wkafka.SkipListResponse + 9, // [9:11] is the sub-list for method output_type + 7, // [7:9] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_wkafka_wkafka_proto_init() } @@ -425,6 +484,18 @@ func file_wkafka_wkafka_proto_init() { return nil } } + file_wkafka_wkafka_proto_msgTypes[4].Exporter = func(v any, i int) any { + switch v := v.(*SkipListResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -432,7 +503,7 @@ func file_wkafka_wkafka_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_wkafka_wkafka_proto_rawDesc, NumEnums: 1, - NumMessages: 6, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/handler/gen/wkafka/wkafkaconnect/wkafka.connect.go b/handler/gen/wkafka/wkafkaconnect/wkafka.connect.go index 883f5b5..dda85c3 100644 --- a/handler/gen/wkafka/wkafkaconnect/wkafka.connect.go +++ b/handler/gen/wkafka/wkafkaconnect/wkafka.connect.go @@ -9,6 +9,7 @@ import ( context "context" errors "errors" wkafka "github.com/worldline-go/wkafka/handler/gen/wkafka" + emptypb "google.golang.org/protobuf/types/known/emptypb" http "net/http" strings "strings" ) @@ -35,17 +36,21 @@ const ( const ( // WkafkaServiceSkipProcedure is the fully-qualified name of the WkafkaService's Skip RPC. WkafkaServiceSkipProcedure = "/wkafka.WkafkaService/Skip" + // WkafkaServiceSkipListProcedure is the fully-qualified name of the WkafkaService's SkipList RPC. + WkafkaServiceSkipListProcedure = "/wkafka.WkafkaService/SkipList" ) // These variables are the protoreflect.Descriptor objects for the RPCs defined in this package. var ( - wkafkaServiceServiceDescriptor = wkafka.File_wkafka_wkafka_proto.Services().ByName("WkafkaService") - wkafkaServiceSkipMethodDescriptor = wkafkaServiceServiceDescriptor.Methods().ByName("Skip") + wkafkaServiceServiceDescriptor = wkafka.File_wkafka_wkafka_proto.Services().ByName("WkafkaService") + wkafkaServiceSkipMethodDescriptor = wkafkaServiceServiceDescriptor.Methods().ByName("Skip") + wkafkaServiceSkipListMethodDescriptor = wkafkaServiceServiceDescriptor.Methods().ByName("SkipList") ) // WkafkaServiceClient is a client for the wkafka.WkafkaService service. type WkafkaServiceClient interface { Skip(context.Context, *connect.Request[wkafka.CreateSkipRequest]) (*connect.Response[wkafka.Response], error) + SkipList(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[wkafka.SkipListResponse], error) } // NewWkafkaServiceClient constructs a client for the wkafka.WkafkaService service. By default, it @@ -64,12 +69,19 @@ func NewWkafkaServiceClient(httpClient connect.HTTPClient, baseURL string, opts connect.WithSchema(wkafkaServiceSkipMethodDescriptor), connect.WithClientOptions(opts...), ), + skipList: connect.NewClient[emptypb.Empty, wkafka.SkipListResponse]( + httpClient, + baseURL+WkafkaServiceSkipListProcedure, + connect.WithSchema(wkafkaServiceSkipListMethodDescriptor), + connect.WithClientOptions(opts...), + ), } } // wkafkaServiceClient implements WkafkaServiceClient. type wkafkaServiceClient struct { - skip *connect.Client[wkafka.CreateSkipRequest, wkafka.Response] + skip *connect.Client[wkafka.CreateSkipRequest, wkafka.Response] + skipList *connect.Client[emptypb.Empty, wkafka.SkipListResponse] } // Skip calls wkafka.WkafkaService.Skip. @@ -77,9 +89,15 @@ func (c *wkafkaServiceClient) Skip(ctx context.Context, req *connect.Request[wka return c.skip.CallUnary(ctx, req) } +// SkipList calls wkafka.WkafkaService.SkipList. +func (c *wkafkaServiceClient) SkipList(ctx context.Context, req *connect.Request[emptypb.Empty]) (*connect.Response[wkafka.SkipListResponse], error) { + return c.skipList.CallUnary(ctx, req) +} + // WkafkaServiceHandler is an implementation of the wkafka.WkafkaService service. type WkafkaServiceHandler interface { Skip(context.Context, *connect.Request[wkafka.CreateSkipRequest]) (*connect.Response[wkafka.Response], error) + SkipList(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[wkafka.SkipListResponse], error) } // NewWkafkaServiceHandler builds an HTTP handler from the service implementation. It returns the @@ -94,10 +112,18 @@ func NewWkafkaServiceHandler(svc WkafkaServiceHandler, opts ...connect.HandlerOp connect.WithSchema(wkafkaServiceSkipMethodDescriptor), connect.WithHandlerOptions(opts...), ) + wkafkaServiceSkipListHandler := connect.NewUnaryHandler( + WkafkaServiceSkipListProcedure, + svc.SkipList, + connect.WithSchema(wkafkaServiceSkipListMethodDescriptor), + connect.WithHandlerOptions(opts...), + ) return "/wkafka.WkafkaService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case WkafkaServiceSkipProcedure: wkafkaServiceSkipHandler.ServeHTTP(w, r) + case WkafkaServiceSkipListProcedure: + wkafkaServiceSkipListHandler.ServeHTTP(w, r) default: http.NotFound(w, r) } @@ -110,3 +136,7 @@ type UnimplementedWkafkaServiceHandler struct{} func (UnimplementedWkafkaServiceHandler) Skip(context.Context, *connect.Request[wkafka.CreateSkipRequest]) (*connect.Response[wkafka.Response], error) { return nil, connect.NewError(connect.CodeUnimplemented, errors.New("wkafka.WkafkaService.Skip is not implemented")) } + +func (UnimplementedWkafkaServiceHandler) SkipList(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[wkafka.SkipListResponse], error) { + return nil, connect.NewError(connect.CodeUnimplemented, errors.New("wkafka.WkafkaService.SkipList is not implemented")) +} diff --git a/handler/handler.go b/handler/handler.go index 46d5768..5821c21 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -4,9 +4,9 @@ import ( "context" "fmt" "net/http" - "slices" "connectrpc.com/connect" + "google.golang.org/protobuf/types/known/emptypb" "github.com/worldline-go/wkafka" wkafkahandler "github.com/worldline-go/wkafka/handler/gen/wkafka" @@ -73,25 +73,33 @@ func convertSkipMap(skip map[string]*wkafkahandler.Topic) wkafka.SkipMap { return m } -func (h *Handler) Skip(ctx context.Context, req *connect.Request[wkafkahandler.CreateSkipRequest]) (*connect.Response[wkafkahandler.Response], error) { - topics := req.Msg.GetTopics() - h.Logger.Debug("skip topics", "topics", topics) - - if !req.Msg.GetEnableMainTopics() { - dlqTopics := h.Client.DLQTopics() +func (h *Handler) SkipList(ctx context.Context, req *connect.Request[emptypb.Empty]) (*connect.Response[wkafkahandler.SkipListResponse], error) { + var skipList map[string]*wkafkahandler.Topic + h.Client.Skip(func(m wkafka.SkipMap) wkafka.SkipMap { + skipList = make(map[string]*wkafkahandler.Topic, len(m)) + for k, v := range m { + p := make(map[int32]*wkafkahandler.Partition, len(v)) + for k, v := range v { + p[k] = &wkafkahandler.Partition{ + Before: v.Before, + Offsets: v.Offsets, + } + } - deleteTopicsInList := make([]string, 0) - for k := range topics { - if !slices.Contains(dlqTopics, k) { - deleteTopicsInList = append(deleteTopicsInList, k) + skipList[k] = &wkafkahandler.Topic{ + Partitions: p, } } - for _, k := range deleteTopicsInList { - delete(topics, k) - } - } + return m + }) + + return connect.NewResponse(&wkafkahandler.SkipListResponse{ + Skip: skipList, + }), nil +} +func (h *Handler) Skip(ctx context.Context, req *connect.Request[wkafkahandler.CreateSkipRequest]) (*connect.Response[wkafkahandler.Response], error) { switch req.Msg.GetOption() { case wkafkahandler.SkipOption_APPEND: h.Client.Skip(wkafka.SkipAppend(convertSkipMap(req.Msg.GetTopics()))) diff --git a/handler/proto/wkafka/wkafka.proto b/handler/proto/wkafka/wkafka.proto index 03cebb8..cc28862 100644 --- a/handler/proto/wkafka/wkafka.proto +++ b/handler/proto/wkafka/wkafka.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package wkafka; +import "google/protobuf/empty.proto"; + enum SkipOption { APPEND = 0; REPLACE = 1; @@ -10,7 +12,6 @@ enum SkipOption { message CreateSkipRequest { map topics = 1; SkipOption option = 2; - bool enable_main_topics = 3; } message Topic { @@ -26,6 +27,11 @@ message Response { string message = 1; } +message SkipListResponse { + map skip = 1; +} + service WkafkaService { rpc Skip (CreateSkipRequest) returns (Response); + rpc SkipList (google.protobuf.Empty) returns (SkipListResponse); }