From ad43c63303d9a3ec8fadffa2077dfad6ce9469fc Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 18 Mar 2023 00:10:08 +0800 Subject: [PATCH 001/280] [fix][test] Move MetadataStoreStatsTest to flaky group to avoid blocking merge. (#19828) --- .../org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java index 70e58c6b079f5..f4deddf0cec9e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java @@ -39,7 +39,7 @@ import org.testng.annotations.Test; -@Test(groups = "broker") +@Test(groups = "flaky") public class MetadataStoreStatsTest extends BrokerTestBase { @BeforeMethod(alwaysRun = true) From e0098ee0c34643371f53ee921bdc361e163f6387 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Sat, 18 Mar 2023 00:39:38 +0800 Subject: [PATCH 002/280] [improve][fn] Support configure compression type (#19470) --- .../common/functions/ProducerConfig.java | 2 + pulsar-function-go/pb/Function.pb.go | 654 ++++++++++-------- pulsar-function-go/pf/instance.go | 14 +- .../instance/JavaInstanceRunnable.java | 4 +- .../pulsar/functions/sink/PulsarSink.java | 6 +- .../instance/src/main/python/Function_pb2.py | 131 +++- .../src/main/python/python_instance.py | 13 +- .../proto/src/main/proto/Function.proto | 9 + .../functions/utils/FunctionCommon.java | 39 ++ .../functions/utils/FunctionConfigUtils.java | 8 + .../functions/utils/SourceConfigUtils.java | 8 + .../utils/FunctionConfigUtilsTest.java | 3 + .../utils/SourceConfigUtilsTest.java | 2 + 13 files changed, 578 insertions(+), 315 deletions(-) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java index f6af7a663fa0c..25ca2ad79c877 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java @@ -23,6 +23,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import org.apache.pulsar.client.api.CompressionType; /** * Configuration of the producer inside the function. @@ -38,4 +39,5 @@ public class ProducerConfig { private Boolean useThreadLocalProducers; private CryptoConfig cryptoConfig; private String batchBuilder; + private CompressionType compressionType; } diff --git a/pulsar-function-go/pb/Function.pb.go b/pulsar-function-go/pb/Function.pb.go index bee1af7cd0c22..bad1afe6658da 100644 --- a/pulsar-function-go/pb/Function.pb.go +++ b/pulsar-function-go/pb/Function.pb.go @@ -191,6 +191,61 @@ func (SubscriptionPosition) EnumDescriptor() ([]byte, []int) { return file_Function_proto_rawDescGZIP(), []int{2} } +type CompressionType int32 + +const ( + CompressionType_LZ4 CompressionType = 0 + CompressionType_NONE CompressionType = 1 + CompressionType_ZLIB CompressionType = 2 + CompressionType_ZSTD CompressionType = 3 + CompressionType_SNAPPY CompressionType = 4 +) + +// Enum value maps for CompressionType. +var ( + CompressionType_name = map[int32]string{ + 0: "LZ4", + 1: "NONE", + 2: "ZLIB", + 3: "ZSTD", + 4: "SNAPPY", + } + CompressionType_value = map[string]int32{ + "LZ4": 0, + "NONE": 1, + "ZLIB": 2, + "ZSTD": 3, + "SNAPPY": 4, + } +) + +func (x CompressionType) Enum() *CompressionType { + p := new(CompressionType) + *p = x + return p +} + +func (x CompressionType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CompressionType) Descriptor() protoreflect.EnumDescriptor { + return file_Function_proto_enumTypes[3].Descriptor() +} + +func (CompressionType) Type() protoreflect.EnumType { + return &file_Function_proto_enumTypes[3] +} + +func (x CompressionType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CompressionType.Descriptor instead. +func (CompressionType) EnumDescriptor() ([]byte, []int) { + return file_Function_proto_rawDescGZIP(), []int{3} +} + type FunctionState int32 const ( @@ -221,11 +276,11 @@ func (x FunctionState) String() string { } func (FunctionState) Descriptor() protoreflect.EnumDescriptor { - return file_Function_proto_enumTypes[3].Descriptor() + return file_Function_proto_enumTypes[4].Descriptor() } func (FunctionState) Type() protoreflect.EnumType { - return &file_Function_proto_enumTypes[3] + return &file_Function_proto_enumTypes[4] } func (x FunctionState) Number() protoreflect.EnumNumber { @@ -234,7 +289,7 @@ func (x FunctionState) Number() protoreflect.EnumNumber { // Deprecated: Use FunctionState.Descriptor instead. func (FunctionState) EnumDescriptor() ([]byte, []int) { - return file_Function_proto_rawDescGZIP(), []int{3} + return file_Function_proto_rawDescGZIP(), []int{4} } type FunctionDetails_Runtime int32 @@ -270,11 +325,11 @@ func (x FunctionDetails_Runtime) String() string { } func (FunctionDetails_Runtime) Descriptor() protoreflect.EnumDescriptor { - return file_Function_proto_enumTypes[4].Descriptor() + return file_Function_proto_enumTypes[5].Descriptor() } func (FunctionDetails_Runtime) Type() protoreflect.EnumType { - return &file_Function_proto_enumTypes[4] + return &file_Function_proto_enumTypes[5] } func (x FunctionDetails_Runtime) Number() protoreflect.EnumNumber { @@ -322,11 +377,11 @@ func (x FunctionDetails_ComponentType) String() string { } func (FunctionDetails_ComponentType) Descriptor() protoreflect.EnumDescriptor { - return file_Function_proto_enumTypes[5].Descriptor() + return file_Function_proto_enumTypes[6].Descriptor() } func (FunctionDetails_ComponentType) Type() protoreflect.EnumType { - return &file_Function_proto_enumTypes[5] + return &file_Function_proto_enumTypes[6] } func (x FunctionDetails_ComponentType) Number() protoreflect.EnumNumber { @@ -374,11 +429,11 @@ func (x CryptoSpec_FailureAction) String() string { } func (CryptoSpec_FailureAction) Descriptor() protoreflect.EnumDescriptor { - return file_Function_proto_enumTypes[6].Descriptor() + return file_Function_proto_enumTypes[7].Descriptor() } func (CryptoSpec_FailureAction) Type() protoreflect.EnumType { - return &file_Function_proto_enumTypes[6] + return &file_Function_proto_enumTypes[7] } func (x CryptoSpec_FailureAction) Number() protoreflect.EnumNumber { @@ -524,6 +579,7 @@ type FunctionDetails struct { Runtime FunctionDetails_Runtime `protobuf:"varint,8,opt,name=runtime,proto3,enum=proto.FunctionDetails_Runtime" json:"runtime,omitempty"` // Deprecated since, see https://github.com/apache/pulsar/issues/15560 // + // Deprecated: Do not use. AutoAck bool `protobuf:"varint,9,opt,name=autoAck,proto3" json:"autoAck,omitempty"` Parallelism int32 `protobuf:"varint,10,opt,name=parallelism,proto3" json:"parallelism,omitempty"` Source *SourceSpec `protobuf:"bytes,11,opt,name=source,proto3" json:"source,omitempty"` @@ -844,11 +900,12 @@ type ProducerSpec struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - MaxPendingMessages int32 `protobuf:"varint,1,opt,name=maxPendingMessages,proto3" json:"maxPendingMessages,omitempty"` - MaxPendingMessagesAcrossPartitions int32 `protobuf:"varint,2,opt,name=maxPendingMessagesAcrossPartitions,proto3" json:"maxPendingMessagesAcrossPartitions,omitempty"` - UseThreadLocalProducers bool `protobuf:"varint,3,opt,name=useThreadLocalProducers,proto3" json:"useThreadLocalProducers,omitempty"` - CryptoSpec *CryptoSpec `protobuf:"bytes,4,opt,name=cryptoSpec,proto3" json:"cryptoSpec,omitempty"` - BatchBuilder string `protobuf:"bytes,5,opt,name=batchBuilder,proto3" json:"batchBuilder,omitempty"` + MaxPendingMessages int32 `protobuf:"varint,1,opt,name=maxPendingMessages,proto3" json:"maxPendingMessages,omitempty"` + MaxPendingMessagesAcrossPartitions int32 `protobuf:"varint,2,opt,name=maxPendingMessagesAcrossPartitions,proto3" json:"maxPendingMessagesAcrossPartitions,omitempty"` + UseThreadLocalProducers bool `protobuf:"varint,3,opt,name=useThreadLocalProducers,proto3" json:"useThreadLocalProducers,omitempty"` + CryptoSpec *CryptoSpec `protobuf:"bytes,4,opt,name=cryptoSpec,proto3" json:"cryptoSpec,omitempty"` + BatchBuilder string `protobuf:"bytes,5,opt,name=batchBuilder,proto3" json:"batchBuilder,omitempty"` + CompressionType CompressionType `protobuf:"varint,6,opt,name=compressionType,proto3,enum=proto.CompressionType" json:"compressionType,omitempty"` } func (x *ProducerSpec) Reset() { @@ -918,6 +975,13 @@ func (x *ProducerSpec) GetBatchBuilder() string { return "" } +func (x *ProducerSpec) GetCompressionType() CompressionType { + if x != nil { + return x.CompressionType + } + return CompressionType_LZ4 +} + type CryptoSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1017,8 +1081,7 @@ type SourceSpec struct { // // Deprecated: Do not use. TopicsToSerDeClassName map[string]string `protobuf:"bytes,4,rep,name=topicsToSerDeClassName,proto3" json:"topicsToSerDeClassName,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - //* - // + // * InputSpecs map[string]*ConsumerSpec `protobuf:"bytes,10,rep,name=inputSpecs,proto3" json:"inputSpecs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` TimeoutMs uint64 `protobuf:"varint,6,opt,name=timeoutMs,proto3" json:"timeoutMs,omitempty"` // Deprecated: Do not use. @@ -1030,6 +1093,7 @@ type SourceSpec struct { CleanupSubscription bool `protobuf:"varint,11,opt,name=cleanupSubscription,proto3" json:"cleanupSubscription,omitempty"` SubscriptionPosition SubscriptionPosition `protobuf:"varint,12,opt,name=subscriptionPosition,proto3,enum=proto.SubscriptionPosition" json:"subscriptionPosition,omitempty"` NegativeAckRedeliveryDelayMs uint64 `protobuf:"varint,13,opt,name=negativeAckRedeliveryDelayMs,proto3" json:"negativeAckRedeliveryDelayMs,omitempty"` + SkipToLatest bool `protobuf:"varint,14,opt,name=skipToLatest,proto3" json:"skipToLatest,omitempty"` } func (x *SourceSpec) Reset() { @@ -1157,6 +1221,13 @@ func (x *SourceSpec) GetNegativeAckRedeliveryDelayMs() uint64 { return 0 } +func (x *SourceSpec) GetSkipToLatest() bool { + if x != nil { + return x.SkipToLatest + } + return false +} + type SinkSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1173,7 +1244,7 @@ type SinkSpec struct { // If specified, this will refer to an archive that is // already present in the server Builtin string `protobuf:"bytes,6,opt,name=builtin,proto3" json:"builtin,omitempty"` - //* + // * // Builtin schema type or custom schema class name SchemaType string `protobuf:"bytes,7,opt,name=schemaType,proto3" json:"schemaType,omitempty"` ForwardSourceMessageProperty bool `protobuf:"varint,8,opt,name=forwardSourceMessageProperty,proto3" json:"forwardSourceMessageProperty,omitempty"` @@ -1350,12 +1421,13 @@ type FunctionMetaData struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - FunctionDetails *FunctionDetails `protobuf:"bytes,1,opt,name=functionDetails,proto3" json:"functionDetails,omitempty"` - PackageLocation *PackageLocationMetaData `protobuf:"bytes,2,opt,name=packageLocation,proto3" json:"packageLocation,omitempty"` - Version uint64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` - CreateTime uint64 `protobuf:"varint,4,opt,name=createTime,proto3" json:"createTime,omitempty"` - InstanceStates map[int32]FunctionState `protobuf:"bytes,5,rep,name=instanceStates,proto3" json:"instanceStates,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=proto.FunctionState"` - FunctionAuthSpec *FunctionAuthenticationSpec `protobuf:"bytes,6,opt,name=functionAuthSpec,proto3" json:"functionAuthSpec,omitempty"` + FunctionDetails *FunctionDetails `protobuf:"bytes,1,opt,name=functionDetails,proto3" json:"functionDetails,omitempty"` + PackageLocation *PackageLocationMetaData `protobuf:"bytes,2,opt,name=packageLocation,proto3" json:"packageLocation,omitempty"` + Version uint64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + CreateTime uint64 `protobuf:"varint,4,opt,name=createTime,proto3" json:"createTime,omitempty"` + InstanceStates map[int32]FunctionState `protobuf:"bytes,5,rep,name=instanceStates,proto3" json:"instanceStates,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=proto.FunctionState"` + FunctionAuthSpec *FunctionAuthenticationSpec `protobuf:"bytes,6,opt,name=functionAuthSpec,proto3" json:"functionAuthSpec,omitempty"` + TransformFunctionPackageLocation *PackageLocationMetaData `protobuf:"bytes,7,opt,name=transformFunctionPackageLocation,proto3" json:"transformFunctionPackageLocation,omitempty"` } func (x *FunctionMetaData) Reset() { @@ -1432,18 +1504,25 @@ func (x *FunctionMetaData) GetFunctionAuthSpec() *FunctionAuthenticationSpec { return nil } +func (x *FunctionMetaData) GetTransformFunctionPackageLocation() *PackageLocationMetaData { + if x != nil { + return x.TransformFunctionPackageLocation + } + return nil +} + type FunctionAuthenticationSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - //* + // * // function authentication related data that the function authentication provider // needs to cache/distribute to all workers support function authentication. // Depending on the function authentication provider implementation, this can be the actual auth credentials // or a pointer to the auth credentials that this function should use Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` - //* + // * // classname of the function auth provicer this data is relevant to Provider string `protobuf:"bytes,2,opt,name=provider,proto3" json:"provider,omitempty"` } @@ -1776,7 +1855,7 @@ var file_Function_proto_rawDesc = []byte{ 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9f, 0x02, 0x0a, 0x0c, 0x50, 0x72, 0x6f, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x02, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x2e, 0x0a, 0x12, 0x6d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, @@ -1794,202 +1873,220 @@ var file_Function_proto_rawDesc = []byte{ 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0a, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x22, 0x0a, 0x0c, 0x62, 0x61, 0x74, 0x63, 0x68, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x61, - 0x74, 0x63, 0x68, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x22, 0xc1, 0x03, 0x0a, 0x0a, 0x43, - 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x18, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73, - 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73, - 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, - 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a, 0x19, 0x70, - 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x19, - 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x1b, 0x70, 0x72, 0x6f, - 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, - 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, - 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x1b, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x61, 0x0a, 0x1b, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, - 0x53, 0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, - 0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, - 0x3d, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x08, 0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x49, - 0x53, 0x43, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x53, 0x55, - 0x4d, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0a, 0x22, 0xd1, - 0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, - 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, - 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x79, - 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x73, - 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75, - 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x10, - 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x69, 0x0a, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44, - 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, - 0x70, 0x65, 0x63, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44, - 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, - 0x02, 0x18, 0x01, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, - 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70, - 0x65, 0x63, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x1c, - 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x28, 0x0a, 0x0d, - 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50, - 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, - 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, - 0x12, 0x2a, 0x0a, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x75, 0x62, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x13, - 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6c, 0x65, 0x61, 0x6e, - 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4f, - 0x0a, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, - 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x42, 0x0a, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x63, 0x6b, 0x52, 0x65, - 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x4d, 0x73, 0x18, - 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, - 0x63, 0x6b, 0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, - 0x79, 0x4d, 0x73, 0x1a, 0x49, 0x0a, 0x1b, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, - 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x52, - 0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, - 0x6d, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0x95, 0x05, 0x0a, 0x08, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x12, + 0x74, 0x63, 0x68, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x40, 0x0a, 0x0f, 0x63, 0x6f, + 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, 0x70, + 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0f, 0x63, 0x6f, 0x6d, + 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0xc1, 0x03, 0x0a, + 0x0a, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x18, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, + 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, + 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, + 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a, + 0x19, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x19, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x1b, 0x70, + 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69, + 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, + 0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x1b, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, + 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x61, + 0x0a, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, + 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, + 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, + 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x3d, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, + 0x44, 0x49, 0x53, 0x43, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, + 0x53, 0x55, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0a, + 0x22, 0xf5, 0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, - 0x70, 0x69, 0x63, 0x12, 0x37, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, - 0x70, 0x65, 0x63, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0c, - 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x26, 0x0a, 0x0e, - 0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x12, 0x1e, - 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x42, - 0x0a, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x79, 0x12, 0x51, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x53, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, - 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, - 0x65, 0x63, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x43, - 0x0a, 0x15, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x67, 0x0a, 0x17, 0x50, 0x61, - 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, - 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b, - 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, - 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, - 0x61, 0x6d, 0x65, 0x22, 0xd5, 0x03, 0x0a, 0x10, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, 0x0f, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x48, 0x0a, 0x0f, 0x70, 0x61, - 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, - 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, - 0x61, 0x74, 0x61, 0x52, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x53, - 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x2e, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x73, 0x12, 0x4d, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, - 0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, - 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x53, 0x70, - 0x65, 0x63, 0x1a, 0x57, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a, 0x1a, 0x46, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a, - 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x08, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x0a, 0x41, 0x73, - 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, - 0x64, 0x2a, 0x5b, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x47, - 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x54, 0x4c, - 0x45, 0x41, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x41, - 0x54, 0x4d, 0x4f, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, - 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, 0x45, 0x4c, 0x59, 0x5f, 0x4f, 0x4e, 0x43, 0x45, - 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0x3c, - 0x0a, 0x10, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, - 0x0a, 0x08, 0x46, 0x41, 0x49, 0x4c, 0x4f, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, - 0x4b, 0x45, 0x59, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x30, 0x0a, 0x14, - 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54, 0x10, 0x00, - 0x12, 0x0c, 0x0a, 0x08, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x01, 0x2a, 0x29, - 0x0a, 0x0d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, - 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, - 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, 0x01, 0x42, 0x2d, 0x0a, 0x21, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x70, 0x75, 0x6c, 0x73, 0x61, 0x72, 0x2e, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x08, - 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, + 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x69, 0x0a, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, + 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, + 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, + 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, + 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x53, 0x70, 0x65, 0x63, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, + 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x28, + 0x0a, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63, + 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, + 0x74, 0x69, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, + 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x75, + 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30, + 0x0a, 0x13, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6c, 0x65, + 0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x4f, 0x0a, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x73, 0x75, 0x62, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x42, 0x0a, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x63, 0x6b, + 0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x4d, + 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, + 0x65, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, + 0x6c, 0x61, 0x79, 0x4d, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x54, 0x6f, 0x4c, + 0x61, 0x74, 0x65, 0x73, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, + 0x70, 0x54, 0x6f, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x1a, 0x49, 0x0a, 0x1b, 0x54, 0x6f, 0x70, + 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, + 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x52, 0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, + 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x95, 0x05, 0x0a, 0x08, 0x53, 0x69, 0x6e, + 0x6b, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a, + 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x37, 0x0a, 0x0c, 0x70, 0x72, 0x6f, + 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, + 0x53, 0x70, 0x65, 0x63, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, + 0x65, 0x63, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, + 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x44, + 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, + 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, + 0x6c, 0x74, 0x69, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x42, 0x0a, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, 0x66, 0x6f, 0x72, 0x77, + 0x61, 0x72, 0x64, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x51, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, + 0x70, 0x65, 0x63, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x12, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, + 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x12, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x1a, 0x43, 0x0a, 0x15, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x43, 0x6f, 0x6e, + 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x67, 0x0a, 0x17, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x70, + 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a, + 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, + 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc1, 0x04, 0x0a, 0x10, 0x46, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40, + 0x0a, 0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, + 0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, + 0x12, 0x48, 0x0a, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, + 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x53, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, + 0x61, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, + 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x4d, 0x0a, 0x10, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x41, 0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x12, 0x6a, 0x0a, 0x20, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x63, + 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, + 0x74, 0x61, 0x52, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x57, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a, + 0x1a, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, + 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x08, 0x49, + 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, + 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x0a, + 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x69, 0x6e, + 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x69, + 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x49, 0x64, 0x2a, 0x5b, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, + 0x67, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x0c, 0x41, + 0x54, 0x4c, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a, + 0x0b, 0x41, 0x54, 0x4d, 0x4f, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x14, + 0x0a, 0x10, 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, 0x45, 0x4c, 0x59, 0x5f, 0x4f, 0x4e, + 0x43, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, + 0x2a, 0x3c, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x00, + 0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x49, 0x4c, 0x4f, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0e, + 0x0a, 0x0a, 0x4b, 0x45, 0x59, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x30, + 0x0a, 0x14, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, + 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54, + 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x01, + 0x2a, 0x44, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x5a, 0x34, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, + 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x4c, 0x49, 0x42, 0x10, 0x02, + 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x53, 0x54, 0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x4e, + 0x41, 0x50, 0x50, 0x59, 0x10, 0x04, 0x2a, 0x29, 0x0a, 0x0d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, + 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, + 0x01, 0x42, 0x2d, 0x0a, 0x21, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x70, 0x75, 0x6c, 0x73, 0x61, 0x72, 0x2e, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x08, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2004,74 +2101,77 @@ func file_Function_proto_rawDescGZIP() []byte { return file_Function_proto_rawDescData } -var file_Function_proto_enumTypes = make([]protoimpl.EnumInfo, 7) +var file_Function_proto_enumTypes = make([]protoimpl.EnumInfo, 8) var file_Function_proto_msgTypes = make([]protoimpl.MessageInfo, 21) var file_Function_proto_goTypes = []interface{}{ (ProcessingGuarantees)(0), // 0: proto.ProcessingGuarantees (SubscriptionType)(0), // 1: proto.SubscriptionType (SubscriptionPosition)(0), // 2: proto.SubscriptionPosition - (FunctionState)(0), // 3: proto.FunctionState - (FunctionDetails_Runtime)(0), // 4: proto.FunctionDetails.Runtime - (FunctionDetails_ComponentType)(0), // 5: proto.FunctionDetails.ComponentType - (CryptoSpec_FailureAction)(0), // 6: proto.CryptoSpec.FailureAction - (*Resources)(nil), // 7: proto.Resources - (*RetryDetails)(nil), // 8: proto.RetryDetails - (*FunctionDetails)(nil), // 9: proto.FunctionDetails - (*ConsumerSpec)(nil), // 10: proto.ConsumerSpec - (*ProducerSpec)(nil), // 11: proto.ProducerSpec - (*CryptoSpec)(nil), // 12: proto.CryptoSpec - (*SourceSpec)(nil), // 13: proto.SourceSpec - (*SinkSpec)(nil), // 14: proto.SinkSpec - (*PackageLocationMetaData)(nil), // 15: proto.PackageLocationMetaData - (*FunctionMetaData)(nil), // 16: proto.FunctionMetaData - (*FunctionAuthenticationSpec)(nil), // 17: proto.FunctionAuthenticationSpec - (*Instance)(nil), // 18: proto.Instance - (*Assignment)(nil), // 19: proto.Assignment - (*ConsumerSpec_ReceiverQueueSize)(nil), // 20: proto.ConsumerSpec.ReceiverQueueSize - nil, // 21: proto.ConsumerSpec.SchemaPropertiesEntry - nil, // 22: proto.ConsumerSpec.ConsumerPropertiesEntry - nil, // 23: proto.SourceSpec.TopicsToSerDeClassNameEntry - nil, // 24: proto.SourceSpec.InputSpecsEntry - nil, // 25: proto.SinkSpec.SchemaPropertiesEntry - nil, // 26: proto.SinkSpec.ConsumerPropertiesEntry - nil, // 27: proto.FunctionMetaData.InstanceStatesEntry + (CompressionType)(0), // 3: proto.CompressionType + (FunctionState)(0), // 4: proto.FunctionState + (FunctionDetails_Runtime)(0), // 5: proto.FunctionDetails.Runtime + (FunctionDetails_ComponentType)(0), // 6: proto.FunctionDetails.ComponentType + (CryptoSpec_FailureAction)(0), // 7: proto.CryptoSpec.FailureAction + (*Resources)(nil), // 8: proto.Resources + (*RetryDetails)(nil), // 9: proto.RetryDetails + (*FunctionDetails)(nil), // 10: proto.FunctionDetails + (*ConsumerSpec)(nil), // 11: proto.ConsumerSpec + (*ProducerSpec)(nil), // 12: proto.ProducerSpec + (*CryptoSpec)(nil), // 13: proto.CryptoSpec + (*SourceSpec)(nil), // 14: proto.SourceSpec + (*SinkSpec)(nil), // 15: proto.SinkSpec + (*PackageLocationMetaData)(nil), // 16: proto.PackageLocationMetaData + (*FunctionMetaData)(nil), // 17: proto.FunctionMetaData + (*FunctionAuthenticationSpec)(nil), // 18: proto.FunctionAuthenticationSpec + (*Instance)(nil), // 19: proto.Instance + (*Assignment)(nil), // 20: proto.Assignment + (*ConsumerSpec_ReceiverQueueSize)(nil), // 21: proto.ConsumerSpec.ReceiverQueueSize + nil, // 22: proto.ConsumerSpec.SchemaPropertiesEntry + nil, // 23: proto.ConsumerSpec.ConsumerPropertiesEntry + nil, // 24: proto.SourceSpec.TopicsToSerDeClassNameEntry + nil, // 25: proto.SourceSpec.InputSpecsEntry + nil, // 26: proto.SinkSpec.SchemaPropertiesEntry + nil, // 27: proto.SinkSpec.ConsumerPropertiesEntry + nil, // 28: proto.FunctionMetaData.InstanceStatesEntry } var file_Function_proto_depIdxs = []int32{ 0, // 0: proto.FunctionDetails.processingGuarantees:type_name -> proto.ProcessingGuarantees - 4, // 1: proto.FunctionDetails.runtime:type_name -> proto.FunctionDetails.Runtime - 13, // 2: proto.FunctionDetails.source:type_name -> proto.SourceSpec - 14, // 3: proto.FunctionDetails.sink:type_name -> proto.SinkSpec - 7, // 4: proto.FunctionDetails.resources:type_name -> proto.Resources - 8, // 5: proto.FunctionDetails.retryDetails:type_name -> proto.RetryDetails - 5, // 6: proto.FunctionDetails.componentType:type_name -> proto.FunctionDetails.ComponentType + 5, // 1: proto.FunctionDetails.runtime:type_name -> proto.FunctionDetails.Runtime + 14, // 2: proto.FunctionDetails.source:type_name -> proto.SourceSpec + 15, // 3: proto.FunctionDetails.sink:type_name -> proto.SinkSpec + 8, // 4: proto.FunctionDetails.resources:type_name -> proto.Resources + 9, // 5: proto.FunctionDetails.retryDetails:type_name -> proto.RetryDetails + 6, // 6: proto.FunctionDetails.componentType:type_name -> proto.FunctionDetails.ComponentType 2, // 7: proto.FunctionDetails.subscriptionPosition:type_name -> proto.SubscriptionPosition - 20, // 8: proto.ConsumerSpec.receiverQueueSize:type_name -> proto.ConsumerSpec.ReceiverQueueSize - 21, // 9: proto.ConsumerSpec.schemaProperties:type_name -> proto.ConsumerSpec.SchemaPropertiesEntry - 22, // 10: proto.ConsumerSpec.consumerProperties:type_name -> proto.ConsumerSpec.ConsumerPropertiesEntry - 12, // 11: proto.ConsumerSpec.cryptoSpec:type_name -> proto.CryptoSpec - 12, // 12: proto.ProducerSpec.cryptoSpec:type_name -> proto.CryptoSpec - 6, // 13: proto.CryptoSpec.producerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction - 6, // 14: proto.CryptoSpec.consumerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction - 1, // 15: proto.SourceSpec.subscriptionType:type_name -> proto.SubscriptionType - 23, // 16: proto.SourceSpec.topicsToSerDeClassName:type_name -> proto.SourceSpec.TopicsToSerDeClassNameEntry - 24, // 17: proto.SourceSpec.inputSpecs:type_name -> proto.SourceSpec.InputSpecsEntry - 2, // 18: proto.SourceSpec.subscriptionPosition:type_name -> proto.SubscriptionPosition - 11, // 19: proto.SinkSpec.producerSpec:type_name -> proto.ProducerSpec - 25, // 20: proto.SinkSpec.schemaProperties:type_name -> proto.SinkSpec.SchemaPropertiesEntry - 26, // 21: proto.SinkSpec.consumerProperties:type_name -> proto.SinkSpec.ConsumerPropertiesEntry - 9, // 22: proto.FunctionMetaData.functionDetails:type_name -> proto.FunctionDetails - 15, // 23: proto.FunctionMetaData.packageLocation:type_name -> proto.PackageLocationMetaData - 27, // 24: proto.FunctionMetaData.instanceStates:type_name -> proto.FunctionMetaData.InstanceStatesEntry - 17, // 25: proto.FunctionMetaData.functionAuthSpec:type_name -> proto.FunctionAuthenticationSpec - 16, // 26: proto.Instance.functionMetaData:type_name -> proto.FunctionMetaData - 18, // 27: proto.Assignment.instance:type_name -> proto.Instance - 10, // 28: proto.SourceSpec.InputSpecsEntry.value:type_name -> proto.ConsumerSpec - 3, // 29: proto.FunctionMetaData.InstanceStatesEntry.value:type_name -> proto.FunctionState - 30, // [30:30] is the sub-list for method output_type - 30, // [30:30] is the sub-list for method input_type - 30, // [30:30] is the sub-list for extension type_name - 30, // [30:30] is the sub-list for extension extendee - 0, // [0:30] is the sub-list for field type_name + 21, // 8: proto.ConsumerSpec.receiverQueueSize:type_name -> proto.ConsumerSpec.ReceiverQueueSize + 22, // 9: proto.ConsumerSpec.schemaProperties:type_name -> proto.ConsumerSpec.SchemaPropertiesEntry + 23, // 10: proto.ConsumerSpec.consumerProperties:type_name -> proto.ConsumerSpec.ConsumerPropertiesEntry + 13, // 11: proto.ConsumerSpec.cryptoSpec:type_name -> proto.CryptoSpec + 13, // 12: proto.ProducerSpec.cryptoSpec:type_name -> proto.CryptoSpec + 3, // 13: proto.ProducerSpec.compressionType:type_name -> proto.CompressionType + 7, // 14: proto.CryptoSpec.producerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction + 7, // 15: proto.CryptoSpec.consumerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction + 1, // 16: proto.SourceSpec.subscriptionType:type_name -> proto.SubscriptionType + 24, // 17: proto.SourceSpec.topicsToSerDeClassName:type_name -> proto.SourceSpec.TopicsToSerDeClassNameEntry + 25, // 18: proto.SourceSpec.inputSpecs:type_name -> proto.SourceSpec.InputSpecsEntry + 2, // 19: proto.SourceSpec.subscriptionPosition:type_name -> proto.SubscriptionPosition + 12, // 20: proto.SinkSpec.producerSpec:type_name -> proto.ProducerSpec + 26, // 21: proto.SinkSpec.schemaProperties:type_name -> proto.SinkSpec.SchemaPropertiesEntry + 27, // 22: proto.SinkSpec.consumerProperties:type_name -> proto.SinkSpec.ConsumerPropertiesEntry + 10, // 23: proto.FunctionMetaData.functionDetails:type_name -> proto.FunctionDetails + 16, // 24: proto.FunctionMetaData.packageLocation:type_name -> proto.PackageLocationMetaData + 28, // 25: proto.FunctionMetaData.instanceStates:type_name -> proto.FunctionMetaData.InstanceStatesEntry + 18, // 26: proto.FunctionMetaData.functionAuthSpec:type_name -> proto.FunctionAuthenticationSpec + 16, // 27: proto.FunctionMetaData.transformFunctionPackageLocation:type_name -> proto.PackageLocationMetaData + 17, // 28: proto.Instance.functionMetaData:type_name -> proto.FunctionMetaData + 19, // 29: proto.Assignment.instance:type_name -> proto.Instance + 11, // 30: proto.SourceSpec.InputSpecsEntry.value:type_name -> proto.ConsumerSpec + 4, // 31: proto.FunctionMetaData.InstanceStatesEntry.value:type_name -> proto.FunctionState + 32, // [32:32] is the sub-list for method output_type + 32, // [32:32] is the sub-list for method input_type + 32, // [32:32] is the sub-list for extension type_name + 32, // [32:32] is the sub-list for extension extendee + 0, // [0:32] is the sub-list for field type_name } func init() { file_Function_proto_init() } @@ -2254,7 +2354,7 @@ func file_Function_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_Function_proto_rawDesc, - NumEnums: 7, + NumEnums: 8, NumMessages: 21, NumExtensions: 0, NumServices: 0, diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 5e05c070e65f9..5d17cfe0c333a 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -225,7 +225,19 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) { batchBuilderType := pulsar.DefaultBatchBuilder + compressionType := pulsar.LZ4 if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil { + switch gi.context.instanceConf.funcDetails.Sink.ProducerSpec.CompressionType { + case pb.CompressionType_NONE: + compressionType = pulsar.NoCompression + case pb.CompressionType_ZLIB: + compressionType = pulsar.ZLib + case pb.CompressionType_ZSTD: + compressionType = pulsar.ZSTD + default: + compressionType = pulsar.LZ4 // go doesn't support SNAPPY yet + } + batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder if batchBuilder != "" { if batchBuilder == "KEY_BASED" { @@ -237,7 +249,7 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) { producer, err := gi.client.CreateProducer(pulsar.ProducerOptions{ Topic: topicName, Properties: properties, - CompressionType: pulsar.LZ4, + CompressionType: compressionType, BatchingMaxPublishDelay: time.Millisecond * 10, BatcherBuilderType: batchBuilderType, SendTimeout: 0, diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 0dbfa0945caa7..e2ad9e4c989d1 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -904,7 +904,9 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { .maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions()) .batchBuilder(conf.getBatchBuilder()) .useThreadLocalProducers(conf.getUseThreadLocalProducers()) - .cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec())); + .cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec())) + .compressionType(FunctionCommon.convertFromFunctionDetailsCompressionType( + conf.getCompressionType())); pulsarSinkConfig.setProducerConfig(builder.build()); } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 41ec4d99e71b6..8add0a78c5fff 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -108,7 +108,6 @@ public Producer createProducer(PulsarClient client, String topic, String prod .blockIfQueueFull(true) .enableBatching(true) .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS) - .compressionType(CompressionType.LZ4) .hashingScheme(HashingScheme.Murmur3_32Hash) // .messageRoutingMode(MessageRoutingMode.CustomPartition) .messageRouter(FunctionResultRouter.of()) @@ -121,6 +120,11 @@ public Producer createProducer(PulsarClient client, String topic, String prod } if (pulsarSinkConfig.getProducerConfig() != null) { ProducerConfig producerConfig = pulsarSinkConfig.getProducerConfig(); + if (producerConfig.getCompressionType() != null) { + builder.compressionType(producerConfig.getCompressionType()); + } else { + builder.compressionType(CompressionType.LZ4); + } if (producerConfig.getMaxPendingMessages() != 0) { builder.maxPendingMessages(producerConfig.getMaxPendingMessages()); } diff --git a/pulsar-functions/instance/src/main/python/Function_pb2.py b/pulsar-functions/instance/src/main/python/Function_pb2.py index eebfe8589d5db..118a6a1cd8967 100644 --- a/pulsar-functions/instance/src/main/python/Function_pb2.py +++ b/pulsar-functions/instance/src/main/python/Function_pb2.py @@ -39,7 +39,7 @@ syntax='proto3', serialized_options=b'\n!org.apache.pulsar.functions.protoB\010Function', create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0e\x46unction.proto\x12\x05proto\"3\n\tResources\x12\x0b\n\x03\x63pu\x18\x01 \x01(\x01\x12\x0b\n\x03ram\x18\x02 \x01(\x03\x12\x0c\n\x04\x64isk\x18\x03 \x01(\x03\"B\n\x0cRetryDetails\x12\x19\n\x11maxMessageRetries\x18\x01 \x01(\x05\x12\x17\n\x0f\x64\x65\x61\x64LetterTopic\x18\x02 \x01(\t\"\xa6\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12\x10\n\x08logTopic\x18\x05 \x01(\t\x12\x39\n\x14processingGuarantees\x18\x06 \x01(\x0e\x32\x1b.proto.ProcessingGuarantees\x12\x12\n\nuserConfig\x18\x07 \x01(\t\x12\x12\n\nsecretsMap\x18\x10 \x01(\t\x12/\n\x07runtime\x18\x08 \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x13\n\x07\x61utoAck\x18\t \x01(\x08\x42\x02\x18\x01\x12\x13\n\x0bparallelism\x18\n \x01(\x05\x12!\n\x06source\x18\x0b \x01(\x0b\x32\x11.proto.SourceSpec\x12\x1d\n\x04sink\x18\x0c \x01(\x0b\x32\x0f.proto.SinkSpec\x12#\n\tresources\x18\r \x01(\x0b\x32\x10.proto.Resources\x12\x12\n\npackageUrl\x18\x0e \x01(\t\x12)\n\x0cretryDetails\x18\x0f \x01(\x0b\x32\x13.proto.RetryDetails\x12\x14\n\x0cruntimeFlags\x18\x11 \x01(\t\x12;\n\rcomponentType\x18\x12 \x01(\x0e\x32$.proto.FunctionDetails.ComponentType\x12\x1c\n\x14\x63ustomRuntimeOptions\x18\x13 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x14 \x01(\t\x12\x16\n\x0eretainOrdering\x18\x15 \x01(\x08\x12\x19\n\x11retainKeyOrdering\x18\x16 \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x17 \x01(\x0e\x32\x1b.proto.SubscriptionPosition\"\'\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\x12\x06\n\x02GO\x10\x03\"@\n\rComponentType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x46UNCTION\x10\x01\x12\n\n\x06SOURCE\x10\x02\x12\x08\n\x04SINK\x10\x03\"\xf7\x03\n\x0c\x43onsumerSpec\x12\x12\n\nschemaType\x18\x01 \x01(\t\x12\x16\n\x0eserdeClassName\x18\x02 \x01(\t\x12\x16\n\x0eisRegexPattern\x18\x03 \x01(\x08\x12@\n\x11receiverQueueSize\x18\x04 \x01(\x0b\x32%.proto.ConsumerSpec.ReceiverQueueSize\x12\x43\n\x10schemaProperties\x18\x05 \x03(\x0b\x32).proto.ConsumerSpec.SchemaPropertiesEntry\x12G\n\x12\x63onsumerProperties\x18\x06 \x03(\x0b\x32+.proto.ConsumerSpec.ConsumerPropertiesEntry\x12%\n\ncryptoSpec\x18\x07 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0cpoolMessages\x18\x08 \x01(\x08\x1a\"\n\x11ReceiverQueueSize\x12\r\n\x05value\x18\x01 \x01(\x05\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb4\x01\n\x0cProducerSpec\x12\x1a\n\x12maxPendingMessages\x18\x01 \x01(\x05\x12*\n\"maxPendingMessagesAcrossPartitions\x18\x02 \x01(\x05\x12\x1f\n\x17useThreadLocalProducers\x18\x03 \x01(\x08\x12%\n\ncryptoSpec\x18\x04 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0c\x62\x61tchBuilder\x18\x05 \x01(\t\"\xbb\x02\n\nCryptoSpec\x12 \n\x18\x63ryptoKeyReaderClassName\x18\x01 \x01(\t\x12\x1d\n\x15\x63ryptoKeyReaderConfig\x18\x02 \x01(\t\x12!\n\x19producerEncryptionKeyName\x18\x03 \x03(\t\x12\x44\n\x1bproducerCryptoFailureAction\x18\x04 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\x12\x44\n\x1b\x63onsumerCryptoFailureAction\x18\x05 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\"=\n\rFailureAction\x12\x08\n\x04\x46\x41IL\x10\x00\x12\x0b\n\x07\x44ISCARD\x10\x01\x12\x0b\n\x07\x43ONSUME\x10\x02\x12\x08\n\x04SEND\x10\n\"\xe2\x04\n\nSourceSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\x31\n\x10subscriptionType\x18\x03 \x01(\x0e\x32\x17.proto.SubscriptionType\x12Q\n\x16topicsToSerDeClassName\x18\x04 \x03(\x0b\x32-.proto.SourceSpec.TopicsToSerDeClassNameEntryB\x02\x18\x01\x12\x35\n\ninputSpecs\x18\n \x03(\x0b\x32!.proto.SourceSpec.InputSpecsEntry\x12\x11\n\ttimeoutMs\x18\x06 \x01(\x04\x12\x19\n\rtopicsPattern\x18\x07 \x01(\tB\x02\x18\x01\x12\x0f\n\x07\x62uiltin\x18\x08 \x01(\t\x12\x18\n\x10subscriptionName\x18\t \x01(\t\x12\x1b\n\x13\x63leanupSubscription\x18\x0b \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x0c \x01(\x0e\x32\x1b.proto.SubscriptionPosition\x12$\n\x1cnegativeAckRedeliveryDelayMs\x18\r \x01(\x04\x1a=\n\x1bTopicsToSerDeClassNameEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x46\n\x0fInputSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.proto.ConsumerSpec:\x02\x38\x01\"\xdc\x03\n\x08SinkSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\r\n\x05topic\x18\x03 \x01(\t\x12)\n\x0cproducerSpec\x18\x0b \x01(\x0b\x32\x13.proto.ProducerSpec\x12\x16\n\x0eserDeClassName\x18\x04 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x06 \x01(\t\x12\x12\n\nschemaType\x18\x07 \x01(\t\x12$\n\x1c\x66orwardSourceMessageProperty\x18\x08 \x01(\x08\x12?\n\x10schemaProperties\x18\t \x03(\x0b\x32%.proto.SinkSpec.SchemaPropertiesEntry\x12\x43\n\x12\x63onsumerProperties\x18\n \x03(\x0b\x32\'.proto.SinkSpec.ConsumerPropertiesEntry\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"H\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\x12\x18\n\x10originalFileName\x18\x02 \x01(\t\"\xf0\x02\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\x12\x43\n\x0einstanceStates\x18\x05 \x03(\x0b\x32+.proto.FunctionMetaData.InstanceStatesEntry\x12;\n\x10\x66unctionAuthSpec\x18\x06 \x01(\x0b\x32!.proto.FunctionAuthenticationSpec\x1aK\n\x13InstanceStatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12#\n\x05value\x18\x02 \x01(\x0e\x32\x14.proto.FunctionState:\x02\x38\x01\"<\n\x1a\x46unctionAuthenticationSpec\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08provider\x18\x02 \x01(\t\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\t*[\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\x12\n\n\x06MANUAL\x10\x03*<\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\x0c\n\x08\x46\x41ILOVER\x10\x01\x12\x0e\n\nKEY_SHARED\x10\x02*0\n\x14SubscriptionPosition\x12\n\n\x06LATEST\x10\x00\x12\x0c\n\x08\x45\x41RLIEST\x10\x01*)\n\rFunctionState\x12\x0b\n\x07RUNNING\x10\x00\x12\x0b\n\x07STOPPED\x10\x01\x42-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3' + serialized_pb=b'\n\x0e\x46unction.proto\x12\x05proto\"3\n\tResources\x12\x0b\n\x03\x63pu\x18\x01 \x01(\x01\x12\x0b\n\x03ram\x18\x02 \x01(\x03\x12\x0c\n\x04\x64isk\x18\x03 \x01(\x03\"B\n\x0cRetryDetails\x12\x19\n\x11maxMessageRetries\x18\x01 \x01(\x05\x12\x17\n\x0f\x64\x65\x61\x64LetterTopic\x18\x02 \x01(\t\"\xa6\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12\x10\n\x08logTopic\x18\x05 \x01(\t\x12\x39\n\x14processingGuarantees\x18\x06 \x01(\x0e\x32\x1b.proto.ProcessingGuarantees\x12\x12\n\nuserConfig\x18\x07 \x01(\t\x12\x12\n\nsecretsMap\x18\x10 \x01(\t\x12/\n\x07runtime\x18\x08 \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x13\n\x07\x61utoAck\x18\t \x01(\x08\x42\x02\x18\x01\x12\x13\n\x0bparallelism\x18\n \x01(\x05\x12!\n\x06source\x18\x0b \x01(\x0b\x32\x11.proto.SourceSpec\x12\x1d\n\x04sink\x18\x0c \x01(\x0b\x32\x0f.proto.SinkSpec\x12#\n\tresources\x18\r \x01(\x0b\x32\x10.proto.Resources\x12\x12\n\npackageUrl\x18\x0e \x01(\t\x12)\n\x0cretryDetails\x18\x0f \x01(\x0b\x32\x13.proto.RetryDetails\x12\x14\n\x0cruntimeFlags\x18\x11 \x01(\t\x12;\n\rcomponentType\x18\x12 \x01(\x0e\x32$.proto.FunctionDetails.ComponentType\x12\x1c\n\x14\x63ustomRuntimeOptions\x18\x13 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x14 \x01(\t\x12\x16\n\x0eretainOrdering\x18\x15 \x01(\x08\x12\x19\n\x11retainKeyOrdering\x18\x16 \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x17 \x01(\x0e\x32\x1b.proto.SubscriptionPosition\"\'\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\x12\x06\n\x02GO\x10\x03\"@\n\rComponentType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x46UNCTION\x10\x01\x12\n\n\x06SOURCE\x10\x02\x12\x08\n\x04SINK\x10\x03\"\xf7\x03\n\x0c\x43onsumerSpec\x12\x12\n\nschemaType\x18\x01 \x01(\t\x12\x16\n\x0eserdeClassName\x18\x02 \x01(\t\x12\x16\n\x0eisRegexPattern\x18\x03 \x01(\x08\x12@\n\x11receiverQueueSize\x18\x04 \x01(\x0b\x32%.proto.ConsumerSpec.ReceiverQueueSize\x12\x43\n\x10schemaProperties\x18\x05 \x03(\x0b\x32).proto.ConsumerSpec.SchemaPropertiesEntry\x12G\n\x12\x63onsumerProperties\x18\x06 \x03(\x0b\x32+.proto.ConsumerSpec.ConsumerPropertiesEntry\x12%\n\ncryptoSpec\x18\x07 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0cpoolMessages\x18\x08 \x01(\x08\x1a\"\n\x11ReceiverQueueSize\x12\r\n\x05value\x18\x01 \x01(\x05\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xe5\x01\n\x0cProducerSpec\x12\x1a\n\x12maxPendingMessages\x18\x01 \x01(\x05\x12*\n\"maxPendingMessagesAcrossPartitions\x18\x02 \x01(\x05\x12\x1f\n\x17useThreadLocalProducers\x18\x03 \x01(\x08\x12%\n\ncryptoSpec\x18\x04 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0c\x62\x61tchBuilder\x18\x05 \x01(\t\x12/\n\x0f\x63ompressionType\x18\x06 \x01(\x0e\x32\x16.proto.CompressionType\"\xbb\x02\n\nCryptoSpec\x12 \n\x18\x63ryptoKeyReaderClassName\x18\x01 \x01(\t\x12\x1d\n\x15\x63ryptoKeyReaderConfig\x18\x02 \x01(\t\x12!\n\x19producerEncryptionKeyName\x18\x03 \x03(\t\x12\x44\n\x1bproducerCryptoFailureAction\x18\x04 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\x12\x44\n\x1b\x63onsumerCryptoFailureAction\x18\x05 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\"=\n\rFailureAction\x12\x08\n\x04\x46\x41IL\x10\x00\x12\x0b\n\x07\x44ISCARD\x10\x01\x12\x0b\n\x07\x43ONSUME\x10\x02\x12\x08\n\x04SEND\x10\n\"\xe2\x04\n\nSourceSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\x31\n\x10subscriptionType\x18\x03 \x01(\x0e\x32\x17.proto.SubscriptionType\x12Q\n\x16topicsToSerDeClassName\x18\x04 \x03(\x0b\x32-.proto.SourceSpec.TopicsToSerDeClassNameEntryB\x02\x18\x01\x12\x35\n\ninputSpecs\x18\n \x03(\x0b\x32!.proto.SourceSpec.InputSpecsEntry\x12\x11\n\ttimeoutMs\x18\x06 \x01(\x04\x12\x19\n\rtopicsPattern\x18\x07 \x01(\tB\x02\x18\x01\x12\x0f\n\x07\x62uiltin\x18\x08 \x01(\t\x12\x18\n\x10subscriptionName\x18\t \x01(\t\x12\x1b\n\x13\x63leanupSubscription\x18\x0b \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x0c \x01(\x0e\x32\x1b.proto.SubscriptionPosition\x12$\n\x1cnegativeAckRedeliveryDelayMs\x18\r \x01(\x04\x1a=\n\x1bTopicsToSerDeClassNameEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x46\n\x0fInputSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.proto.ConsumerSpec:\x02\x38\x01\"\xdc\x03\n\x08SinkSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\r\n\x05topic\x18\x03 \x01(\t\x12)\n\x0cproducerSpec\x18\x0b \x01(\x0b\x32\x13.proto.ProducerSpec\x12\x16\n\x0eserDeClassName\x18\x04 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x06 \x01(\t\x12\x12\n\nschemaType\x18\x07 \x01(\t\x12$\n\x1c\x66orwardSourceMessageProperty\x18\x08 \x01(\x08\x12?\n\x10schemaProperties\x18\t \x03(\x0b\x32%.proto.SinkSpec.SchemaPropertiesEntry\x12\x43\n\x12\x63onsumerProperties\x18\n \x03(\x0b\x32\'.proto.SinkSpec.ConsumerPropertiesEntry\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"H\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\x12\x18\n\x10originalFileName\x18\x02 \x01(\t\"\xba\x03\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\x12\x43\n\x0einstanceStates\x18\x05 \x03(\x0b\x32+.proto.FunctionMetaData.InstanceStatesEntry\x12;\n\x10\x66unctionAuthSpec\x18\x06 \x01(\x0b\x32!.proto.FunctionAuthenticationSpec\x12H\n transformFunctionPackageLocation\x18\x07 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x1aK\n\x13InstanceStatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12#\n\x05value\x18\x02 \x01(\x0e\x32\x14.proto.FunctionState:\x02\x38\x01\"<\n\x1a\x46unctionAuthenticationSpec\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08provider\x18\x02 \x01(\t\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\t*[\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\x12\n\n\x06MANUAL\x10\x03*<\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\x0c\n\x08\x46\x41ILOVER\x10\x01\x12\x0e\n\nKEY_SHARED\x10\x02*0\n\x14SubscriptionPosition\x12\n\n\x06LATEST\x10\x00\x12\x0c\n\x08\x45\x41RLIEST\x10\x01*D\n\x0f\x43ompressionType\x12\x07\n\x03LZ4\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x08\n\x04ZLIB\x10\x02\x12\x08\n\x04ZSTD\x10\x03\x12\n\n\x06SNAPPY\x10\x04*)\n\rFunctionState\x12\x0b\n\x07RUNNING\x10\x00\x12\x0b\n\x07STOPPED\x10\x01\x42-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3' ) _PROCESSINGGUARANTEES = _descriptor.EnumDescriptor( @@ -72,8 +72,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3711, - serialized_end=3802, + serialized_start=3834, + serialized_end=3925, ) _sym_db.RegisterEnumDescriptor(_PROCESSINGGUARANTEES) @@ -103,8 +103,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3804, - serialized_end=3864, + serialized_start=3927, + serialized_end=3987, ) _sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONTYPE) @@ -129,12 +129,53 @@ ], containing_type=None, serialized_options=None, - serialized_start=3866, - serialized_end=3914, + serialized_start=3989, + serialized_end=4037, ) _sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONPOSITION) SubscriptionPosition = enum_type_wrapper.EnumTypeWrapper(_SUBSCRIPTIONPOSITION) +_COMPRESSIONTYPE = _descriptor.EnumDescriptor( + name='CompressionType', + full_name='proto.CompressionType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='LZ4', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='NONE', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='ZLIB', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='ZSTD', index=3, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='SNAPPY', index=4, number=4, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=4039, + serialized_end=4107, +) +_sym_db.RegisterEnumDescriptor(_COMPRESSIONTYPE) + +CompressionType = enum_type_wrapper.EnumTypeWrapper(_COMPRESSIONTYPE) _FUNCTIONSTATE = _descriptor.EnumDescriptor( name='FunctionState', full_name='proto.FunctionState', @@ -155,8 +196,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3916, - serialized_end=3957, + serialized_start=4109, + serialized_end=4150, ) _sym_db.RegisterEnumDescriptor(_FUNCTIONSTATE) @@ -170,6 +211,11 @@ KEY_SHARED = 2 LATEST = 0 EARLIEST = 1 +LZ4 = 0 +NONE = 1 +ZLIB = 2 +ZSTD = 3 +SNAPPY = 4 RUNNING = 0 STOPPED = 1 @@ -269,8 +315,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=1899, - serialized_end=1960, + serialized_start=1948, + serialized_end=2009, ) _sym_db.RegisterEnumDescriptor(_CRYPTOSPEC_FAILUREACTION) @@ -779,6 +825,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='compressionType', full_name='proto.ProducerSpec.compressionType', index=5, + number=6, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -792,7 +845,7 @@ oneofs=[ ], serialized_start=1462, - serialized_end=1642, + serialized_end=1691, ) @@ -852,8 +905,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1645, - serialized_end=1960, + serialized_start=1694, + serialized_end=2009, ) @@ -891,8 +944,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2440, - serialized_end=2501, + serialized_start=2489, + serialized_end=2550, ) _SOURCESPEC_INPUTSPECSENTRY = _descriptor.Descriptor( @@ -929,8 +982,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2503, - serialized_end=2573, + serialized_start=2552, + serialized_end=2622, ) _SOURCESPEC = _descriptor.Descriptor( @@ -1044,8 +1097,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1963, - serialized_end=2573, + serialized_start=2012, + serialized_end=2622, ) @@ -1222,8 +1275,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2576, - serialized_end=3052, + serialized_start=2625, + serialized_end=3101, ) @@ -1261,8 +1314,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=3054, - serialized_end=3126, + serialized_start=3103, + serialized_end=3175, ) @@ -1300,8 +1353,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=3422, - serialized_end=3497, + serialized_start=3545, + serialized_end=3620, ) _FUNCTIONMETADATA = _descriptor.Descriptor( @@ -1354,6 +1407,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='transformFunctionPackageLocation', full_name='proto.FunctionMetaData.transformFunctionPackageLocation', index=6, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -1366,8 +1426,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=3129, - serialized_end=3497, + serialized_start=3178, + serialized_end=3620, ) @@ -1405,8 +1465,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=3499, - serialized_end=3559, + serialized_start=3622, + serialized_end=3682, ) @@ -1444,8 +1504,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=3561, - serialized_end=3642, + serialized_start=3684, + serialized_end=3765, ) @@ -1483,8 +1543,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=3644, - serialized_end=3709, + serialized_start=3767, + serialized_end=3832, ) _FUNCTIONDETAILS.fields_by_name['processingGuarantees'].enum_type = _PROCESSINGGUARANTEES @@ -1505,6 +1565,7 @@ _CONSUMERSPEC.fields_by_name['consumerProperties'].message_type = _CONSUMERSPEC_CONSUMERPROPERTIESENTRY _CONSUMERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC _PRODUCERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC +_PRODUCERSPEC.fields_by_name['compressionType'].enum_type = _COMPRESSIONTYPE _CRYPTOSPEC.fields_by_name['producerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION _CRYPTOSPEC.fields_by_name['consumerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION _CRYPTOSPEC_FAILUREACTION.containing_type = _CRYPTOSPEC @@ -1526,6 +1587,7 @@ _FUNCTIONMETADATA.fields_by_name['packageLocation'].message_type = _PACKAGELOCATIONMETADATA _FUNCTIONMETADATA.fields_by_name['instanceStates'].message_type = _FUNCTIONMETADATA_INSTANCESTATESENTRY _FUNCTIONMETADATA.fields_by_name['functionAuthSpec'].message_type = _FUNCTIONAUTHENTICATIONSPEC +_FUNCTIONMETADATA.fields_by_name['transformFunctionPackageLocation'].message_type = _PACKAGELOCATIONMETADATA _INSTANCE.fields_by_name['functionMetaData'].message_type = _FUNCTIONMETADATA _ASSIGNMENT.fields_by_name['instance'].message_type = _INSTANCE DESCRIPTOR.message_types_by_name['Resources'] = _RESOURCES @@ -1544,6 +1606,7 @@ DESCRIPTOR.enum_types_by_name['ProcessingGuarantees'] = _PROCESSINGGUARANTEES DESCRIPTOR.enum_types_by_name['SubscriptionType'] = _SUBSCRIPTIONTYPE DESCRIPTOR.enum_types_by_name['SubscriptionPosition'] = _SUBSCRIPTIONPOSITION +DESCRIPTOR.enum_types_by_name['CompressionType'] = _COMPRESSIONTYPE DESCRIPTOR.enum_types_by_name['FunctionState'] = _FUNCTIONSTATE _sym_db.RegisterFileDescriptor(DESCRIPTOR) diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index ac723232f4839..c679288789536 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -352,6 +352,17 @@ def setup_producer(self): if crypto_key_reader is not None: encryption_key = self.instance_config.function_details.sink.producerSpec.cryptoSpec.producerEncryptionKeyName[0] + compression_type = pulsar.CompressionType.LZ4 + if self.instance_config.function_details.sink.producerSpec.compressionType is not None: + if self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("NONE"): + compression_type = pulsar.CompressionType.NONE + elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("ZLIB"): + compression_type = pulsar.CompressionType.ZLib + elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("ZSTD"): + compression_type = pulsar.CompressionType.ZSTD + elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("SNAPPY"): + compression_type = pulsar.CompressionType.SNAPPY + self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), schema=self.output_schema, @@ -359,7 +370,7 @@ def setup_producer(self): batching_enabled=True, batching_type=batch_type, batching_max_publish_delay_ms=10, - compression_type=pulsar.CompressionType.LZ4, + compression_type=compression_type, # set send timeout to be infinity to prevent potential deadlock with consumer # that might happen when consumer is blocked due to unacked messages send_timeout_millis=0, diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto index 101d45bc59cd7..de3f03a39008c 100644 --- a/pulsar-functions/proto/src/main/proto/Function.proto +++ b/pulsar-functions/proto/src/main/proto/Function.proto @@ -41,6 +41,14 @@ enum SubscriptionPosition { EARLIEST = 1; } +enum CompressionType { + LZ4 = 0; + NONE = 1; + ZLIB = 2; + ZSTD = 3; + SNAPPY = 4; +} + message Resources { double cpu = 1; int64 ram = 2; @@ -112,6 +120,7 @@ message ProducerSpec { bool useThreadLocalProducers = 3; CryptoSpec cryptoSpec = 4; string batchBuilder = 5; + CompressionType compressionType = 6; } message CryptoSpec { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index bda99a39478a3..d3ce9d93a2d36 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -44,6 +44,7 @@ import lombok.extern.slf4j.Slf4j; import net.jodah.typetools.TypeResolver; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -568,4 +569,42 @@ public static SubscriptionInitialPosition convertFromFunctionDetailsSubscription return SubscriptionInitialPosition.Latest; } } + + public static CompressionType convertFromFunctionDetailsCompressionType( + org.apache.pulsar.functions.proto.Function.CompressionType compressionType) { + if (compressionType == null) { + return CompressionType.LZ4; + } + switch (compressionType) { + case NONE: + return CompressionType.NONE; + case ZLIB: + return CompressionType.ZLIB; + case ZSTD: + return CompressionType.ZSTD; + case SNAPPY: + return CompressionType.SNAPPY; + default: + return CompressionType.LZ4; + } + } + + public static org.apache.pulsar.functions.proto.Function.CompressionType convertFromCompressionType( + CompressionType compressionType) { + if (compressionType == null) { + return org.apache.pulsar.functions.proto.Function.CompressionType.LZ4; + } + switch (compressionType) { + case NONE: + return org.apache.pulsar.functions.proto.Function.CompressionType.NONE; + case ZLIB: + return org.apache.pulsar.functions.proto.Function.CompressionType.ZLIB; + case ZSTD: + return org.apache.pulsar.functions.proto.Function.CompressionType.ZSTD; + case SNAPPY: + return org.apache.pulsar.functions.proto.Function.CompressionType.SNAPPY; + default: + return org.apache.pulsar.functions.proto.Function.CompressionType.LZ4; + } + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index d02fe5f788b5a..d20d50df1bdd1 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -24,6 +24,8 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.pulsar.common.functions.Utils.BUILTIN; import static org.apache.pulsar.common.util.ClassLoaderUtils.loadJar; +import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType; +import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -271,6 +273,11 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu if (producerConf.getBatchBuilder() != null) { pbldr.setBatchBuilder(producerConf.getBatchBuilder()); } + if (producerConf.getCompressionType() != null) { + pbldr.setCompressionType(convertFromCompressionType(producerConf.getCompressionType())); + } else { + pbldr.setCompressionType(Function.CompressionType.LZ4); + } sinkSpecBuilder.setProducerSpec(pbldr.build()); } functionDetailsBuilder.setSink(sinkSpecBuilder); @@ -471,6 +478,7 @@ public static FunctionConfig convertFromDetails(FunctionDetails functionDetails) producerConfig.setBatchBuilder(spec.getBatchBuilder()); } producerConfig.setUseThreadLocalProducers(spec.getUseThreadLocalProducers()); + producerConfig.setCompressionType(convertFromFunctionDetailsCompressionType(spec.getCompressionType())); functionConfig.setProducerConfig(producerConfig); } if (!isEmpty(functionDetails.getLogTopic())) { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java index 24d4259a74a29..ec0c5c444ccba 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java @@ -19,6 +19,8 @@ package org.apache.pulsar.functions.utils; import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType; +import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType; import static org.apache.pulsar.functions.utils.FunctionCommon.convertProcessingGuarantee; import static org.apache.pulsar.functions.utils.FunctionCommon.getSourceType; import com.fasterxml.jackson.core.type.TypeReference; @@ -164,6 +166,11 @@ public static FunctionDetails convert(SourceConfig sourceConfig, ExtractedSource if (conf.getBatchBuilder() != null) { pbldr.setBatchBuilder(conf.getBatchBuilder()); } + if (conf.getCompressionType() != null) { + pbldr.setCompressionType(convertFromCompressionType(conf.getCompressionType())); + } else { + pbldr.setCompressionType(Function.CompressionType.LZ4); + } sinkSpecBuilder.setProducerSpec(pbldr.build()); } @@ -264,6 +271,7 @@ public static SourceConfig convertFromDetails(FunctionDetails functionDetails) { producerConfig.setBatchBuilder(spec.getBatchBuilder()); } producerConfig.setUseThreadLocalProducers(spec.getUseThreadLocalProducers()); + producerConfig.setCompressionType(convertFromFunctionDetailsCompressionType(spec.getCompressionType())); sourceConfig.setProducerConfig(producerConfig); } if (functionDetails.hasResources()) { diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java index 4ba5138f7c6b9..8b4470d8c76c7 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java @@ -20,6 +20,7 @@ import com.google.gson.Gson; +import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; @@ -96,6 +97,7 @@ public void testConvertBackFidelity() { producerConfig.setMaxPendingMessagesAcrossPartitions(1000); producerConfig.setUseThreadLocalProducers(true); producerConfig.setBatchBuilder("DEFAULT"); + producerConfig.setCompressionType(CompressionType.ZLIB); functionConfig.setProducerConfig(producerConfig); Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); @@ -137,6 +139,7 @@ public void testConvertWindow() { producerConfig.setMaxPendingMessagesAcrossPartitions(1000); producerConfig.setUseThreadLocalProducers(true); producerConfig.setBatchBuilder("KEY_BASED"); + producerConfig.setCompressionType(CompressionType.SNAPPY); functionConfig.setProducerConfig(producerConfig); Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java index 63485d7993fad..49313dbf02c62 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java @@ -22,6 +22,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.Accessors; +import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.functions.Resources; @@ -370,6 +371,7 @@ private SourceConfig createSourceConfig() { producerConfig.setMaxPendingMessagesAcrossPartitions(1000); producerConfig.setUseThreadLocalProducers(true); producerConfig.setBatchBuilder("DEFAULT"); + producerConfig.setCompressionType(CompressionType.ZSTD); sourceConfig.setProducerConfig(producerConfig); sourceConfig.setConfigs(configs); From 4ab4463fee5f773cc29104598c707ed25f827f37 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Sat, 18 Mar 2023 17:44:45 +0800 Subject: [PATCH 003/280] [improve][broker] PIP-192: Add metrics for unload operation (#19749) PIP: https://github.com/apache/pulsar/issues/16691 ### Motivation Raising a PR to implement https://github.com/apache/pulsar/issues/16691. We need to support metrics for unload/transfer operations in Load Manager Extension. ### Modifications In this PR: * Change the `findBundlesForUnloading` method return type from `UnloadDecision` to `Set`. * The `UnloadDecision` no longer contains all unload objects. Each unload object has its own reason. * Add units test to verify the unload counter. --- .../extensions/ExtensibleLoadManagerImpl.java | 18 +- .../extensions/manager/UnloadManager.java | 25 +- .../extensions/models/UnloadCounter.java | 72 +++-- .../extensions/models/UnloadDecision.java | 62 +---- .../AntiAffinityGroupPolicyHelper.java | 3 - .../scheduler/NamespaceUnloadStrategy.java | 7 +- .../extensions/scheduler/TransferShedder.java | 93 ++++--- .../extensions/scheduler/UnloadScheduler.java | 129 ++++++--- .../ExtensibleLoadManagerImplTest.java | 38 +-- .../extensions/manager/UnloadManagerTest.java | 51 +++- .../scheduler/TransferShedderTest.java | 260 ++++++++---------- .../scheduler/UnloadSchedulerTest.java | 51 +++- 12 files changed, 440 insertions(+), 369 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 716be3718bf19..486c32153589f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -20,6 +20,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; @@ -204,8 +206,8 @@ public void start() throws PulsarServerException { }); this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar); this.brokerRegistry.start(); - this.unloadManager = new UnloadManager(); this.splitManager = new SplitManager(splitCounter); + this.unloadManager = new UnloadManager(unloadCounter); this.serviceUnitStateChannel.listen(unloadManager); this.serviceUnitStateChannel.listen(splitManager); this.leaderElectionService.start(); @@ -265,7 +267,8 @@ public void start() throws PulsarServerException { interval, TimeUnit.MILLISECONDS); this.unloadScheduler = new UnloadScheduler( - pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel); + pulsar, pulsar.getLoadManagerExecutor(), unloadManager, + context, serviceUnitStateChannel, antiAffinityGroupPolicyHelper, unloadCounter, unloadMetrics); this.unloadScheduler.start(); this.splitScheduler = new SplitScheduler( pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context); @@ -401,16 +404,21 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, log.warn(msg); throw new IllegalArgumentException(msg); } - return unloadAsync(new Unload(sourceBroker, bundle.toString(), destinationBroker), + Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker); + UnloadDecision unloadDecision = + new UnloadDecision(unload, Success, Admin); + return unloadAsync(unloadDecision, conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); }); } - private CompletableFuture unloadAsync(Unload unload, + private CompletableFuture unloadAsync(UnloadDecision unloadDecision, long timeout, TimeUnit timeoutUnit) { + Unload unload = unloadDecision.getUnload(); CompletableFuture future = serviceUnitStateChannel.publishUnloadEventAsync(unload); - return unloadManager.waitAsync(future, unload.serviceUnit(), timeout, timeoutUnit); + return unloadManager.waitAsync(future, unload.serviceUnit(), unloadDecision, timeout, timeoutUnit) + .thenRun(() -> unloadCounter.updateUnloadBrokerCount(1)); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java index ead6384daba8d..2dde0c4708e41 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.manager; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -25,6 +27,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; /** * Unload manager. @@ -32,9 +36,11 @@ @Slf4j public class UnloadManager implements StateChangeListener { + private final UnloadCounter counter; private final Map> inFlightUnloadRequest; - public UnloadManager() { + public UnloadManager(UnloadCounter counter) { + this.counter = counter; this.inFlightUnloadRequest = new ConcurrentHashMap<>(); } @@ -43,14 +49,8 @@ private void complete(String serviceUnit, Throwable ex) { if (!future.isDone()) { if (ex != null) { future.completeExceptionally(ex); - if (log.isDebugEnabled()) { - log.debug("Complete exceptionally unload bundle: {}", serviceUnit, ex); - } } else { future.complete(null); - if (log.isDebugEnabled()) { - log.debug("Complete unload bundle: {}", serviceUnit); - } } } return null; @@ -59,6 +59,7 @@ private void complete(String serviceUnit, Throwable ex) { public CompletableFuture waitAsync(CompletableFuture eventPubFuture, String bundle, + UnloadDecision decision, long timeout, TimeUnit timeoutUnit) { @@ -74,7 +75,15 @@ public CompletableFuture waitAsync(CompletableFuture eventPubFuture, } }); return future; - })); + })).whenComplete((__, ex) -> { + if (ex != null) { + counter.update(Failure, Unknown); + log.warn("Failed to unload bundle: {}", bundle, ex); + return; + } + log.info("Complete unload bundle: {}", bundle); + counter.update(decision); + }); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java index e2a51b1248967..37483b58b53e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; @@ -30,11 +31,13 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.mutable.MutableLong; +import java.util.concurrent.atomic.AtomicLong; +import lombok.Getter; import org.apache.pulsar.common.stats.Metrics; /** @@ -45,36 +48,63 @@ public class UnloadCounter { long unloadBrokerCount = 0; long unloadBundleCount = 0; - final Map> breakdownCounters; + @Getter + @VisibleForTesting + final Map> breakdownCounters; + @Getter + @VisibleForTesting double loadAvg; + @Getter + @VisibleForTesting double loadStd; + private volatile long updatedAt = 0; + public UnloadCounter() { breakdownCounters = Map.of( Success, Map.of( - Overloaded, new MutableLong(), - Underloaded, new MutableLong()), + Overloaded, new AtomicLong(), + Underloaded, new AtomicLong(), + Admin, new AtomicLong()), Skip, Map.of( - Balanced, new MutableLong(), - NoBundles, new MutableLong(), - CoolDown, new MutableLong(), - OutDatedData, new MutableLong(), - NoLoadData, new MutableLong(), - NoBrokers, new MutableLong(), - Unknown, new MutableLong()), + Balanced, new AtomicLong(), + NoBundles, new AtomicLong(), + CoolDown, new AtomicLong(), + OutDatedData, new AtomicLong(), + NoLoadData, new AtomicLong(), + NoBrokers, new AtomicLong(), + Unknown, new AtomicLong()), Failure, Map.of( - Unknown, new MutableLong()) + Unknown, new AtomicLong()) ); } public void update(UnloadDecision decision) { - var unloads = decision.getUnloads(); - unloadBrokerCount += unloads.keySet().size(); - unloadBundleCount += unloads.values().size(); - breakdownCounters.get(decision.getLabel()).get(decision.getReason()).increment(); - loadAvg = decision.loadAvg; - loadStd = decision.loadStd; + if (decision.getLabel() == Success) { + unloadBundleCount++; + } + breakdownCounters.get(decision.getLabel()).get(decision.getReason()).incrementAndGet(); + updatedAt = System.currentTimeMillis(); + } + + public void update(UnloadDecision.Label label, UnloadDecision.Reason reason) { + if (label == Success) { + unloadBundleCount++; + } + breakdownCounters.get(label).get(reason).incrementAndGet(); + updatedAt = System.currentTimeMillis(); + } + + public void updateLoadData(double loadAvg, double loadStd) { + this.loadAvg = loadAvg; + this.loadStd = loadStd; + updatedAt = System.currentTimeMillis(); + } + + public void updateUnloadBrokerCount(int unloadBrokerCount) { + this.unloadBrokerCount += unloadBrokerCount; + updatedAt = System.currentTimeMillis(); } public List toMetrics(String advertisedBrokerAddress) { @@ -125,4 +155,8 @@ public List toMetrics(String advertisedBrokerAddress) { return metrics; } -} \ No newline at end of file + + public long updatedAt() { + return updatedAt; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java index 67503db34eee7..e1087ab6e53ba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java @@ -18,29 +18,21 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.models; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; +import lombok.AllArgsConstructor; import lombok.Data; /** * Defines the information required to unload or transfer a service unit(e.g. bundle). */ @Data +@AllArgsConstructor public class UnloadDecision { - Multimap unloads; + Unload unload; Label label; Reason reason; - Double loadAvg; - Double loadStd; + public enum Label { Success, Skip, @@ -55,39 +47,20 @@ public enum Reason { OutDatedData, NoLoadData, NoBrokers, + Admin, Unknown } public UnloadDecision() { - unloads = ArrayListMultimap.create(); + unload = null; label = null; reason = null; - loadAvg = null; - loadStd = null; } public void clear() { - unloads.clear(); + unload = null; label = null; reason = null; - loadAvg = null; - loadStd = null; - } - - public void skip(int numOfOverloadedBrokers, - int numOfUnderloadedBrokers, - int numOfBrokersWithEmptyLoadData, - int numOfBrokersWithFewBundles) { - label = Skip; - if (numOfOverloadedBrokers == 0 && numOfUnderloadedBrokers == 0) { - reason = Balanced; - } else if (numOfBrokersWithEmptyLoadData > 0) { - reason = NoLoadData; - } else if (numOfBrokersWithFewBundles > 0) { - reason = NoBundles; - } else { - reason = Unknown; - } } public void skip(Reason reason) { @@ -95,22 +68,9 @@ public void skip(Reason reason) { this.reason = reason; } - public void succeed( - int numOfOverloadedBrokers, - int numOfUnderloadedBrokers) { - - label = Success; - if (numOfOverloadedBrokers > numOfUnderloadedBrokers) { - reason = Overloaded; - } else { - reason = Underloaded; - } - } - - - public void fail() { - label = Failure; - reason = Unknown; + public void succeed(Reason reason) { + this.label = Success; + this.reason = reason; } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java index 28acf5fba0ea1..c8332a1d7b5e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java @@ -54,9 +54,6 @@ public boolean canUnload( String bundle, String srcBroker, Optional dstBroker) { - - - try { var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup( pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java index b4dc92d92187d..42af396abcad4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.scheduler; import java.util.Map; +import java.util.Set; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; @@ -38,8 +39,8 @@ public interface NamespaceUnloadStrategy { * @param recentlyUnloadedBrokers The recently unloaded brokers. * @return unloadDecision containing a list of the bundles that should be unloaded. */ - UnloadDecision findBundlesForUnloading(LoadManagerContext context, - Map recentlyUnloadedBundles, - Map recentlyUnloadedBrokers); + Set findBundlesForUnloading(LoadManagerContext context, + Map recentlyUnloadedBundles, + Map recentlyUnloadedBrokers); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 9f9582df2cc28..3c67479bcc7cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -18,12 +18,20 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.scheduler; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.MinMaxPriorityQueue; +import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -39,6 +47,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; @@ -80,19 +89,26 @@ public class TransferShedder implements NamespaceUnloadStrategy { private final IsolationPoliciesHelper isolationPoliciesHelper; private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; - private final UnloadDecision decision = new UnloadDecision(); + private final Set decisionCache; + private final UnloadCounter counter; @VisibleForTesting - public TransferShedder(){ + public TransferShedder(UnloadCounter counter){ this.pulsar = null; + this.decisionCache = new HashSet<>(); this.allocationPolicies = null; + this.counter = counter; this.isolationPoliciesHelper = null; this.antiAffinityGroupPolicyHelper = null; } - public TransferShedder(PulsarService pulsar, AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper) { + public TransferShedder(PulsarService pulsar, + UnloadCounter counter, + AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper){ this.pulsar = pulsar; + this.decisionCache = new HashSet<>(); this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); + this.counter = counter; this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies); this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper; } @@ -241,13 +257,12 @@ public String toString() { @Override - public UnloadDecision findBundlesForUnloading(LoadManagerContext context, + public Set findBundlesForUnloading(LoadManagerContext context, Map recentlyUnloadedBundles, Map recentlyUnloadedBrokers) { final var conf = context.brokerConfiguration(); - decision.clear(); + decisionCache.clear(); stats.clear(); - var selectedBundlesCache = decision.getUnloads(); try { final var loadStore = context.brokerLoadDataStore(); @@ -255,22 +270,17 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); var skipReason = stats.update(context.brokerLoadDataStore(), recentlyUnloadedBrokers, conf); - if (!skipReason.isEmpty()) { - decision.skip(skipReason.get()); - log.warn("Failed to update load stat. Reason:{}. Stop unloading.", decision.getReason()); - return decision; + if (skipReason.isPresent()) { + log.warn("Failed to update load stat. Reason:{}. Stop unloading.", skipReason.get()); + counter.update(Skip, skipReason.get()); + return decisionCache; } - decision.setLoadAvg(stats.avg); - decision.setLoadStd(stats.std); + counter.updateLoadData(stats.avg, stats.std); if (debugMode) { log.info("brokers' load stats:{}", stats); } - // success metrics - int numOfOverloadedBrokers = 0; - int numOfUnderloadedBrokers = 0; - // skip metrics int numOfBrokersWithEmptyLoadData = 0; int numOfBrokersWithFewBundles = 0; @@ -283,9 +293,9 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync() .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); } catch (ExecutionException | InterruptedException | TimeoutException e) { - decision.skip(Unknown); - log.warn("Failed to fetch available brokers. Reason:{}. Stop unloading.", decision.getReason(), e); - return decision; + counter.update(Skip, Unknown); + log.warn("Failed to fetch available brokers. Reason: Unknown. Stop unloading.", e); + return decisionCache; } while (true) { @@ -295,6 +305,7 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, } break; } + UnloadDecision.Reason reason; if (stats.std() <= targetStd) { if (hasMsgThroughput(context, stats.minBrokers.peekLast())) { if (debugMode) { @@ -303,10 +314,10 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, } break; } else { - numOfUnderloadedBrokers++; + reason = Underloaded; } } else { - numOfOverloadedBrokers++; + reason = Overloaded; } String maxBroker = stats.maxBrokers().pollLast(); @@ -365,13 +376,16 @@ && isTransferable(context, availableBrokers, if (remainingTopBundles > 1 && (trafficMarkedToOffload < offloadThroughput || !atLeastOneBundleSelected)) { + Unload unload; if (transfer) { - selectedBundlesCache.put(maxBroker, - new Unload(maxBroker, bundle, Optional.of(minBroker))); + unload = new Unload(maxBroker, bundle, Optional.of(minBroker)); } else { - selectedBundlesCache.put(maxBroker, - new Unload(maxBroker, bundle)); + unload = new Unload(maxBroker, bundle); } + var decision = new UnloadDecision(); + decision.setUnload(unload); + decision.succeed(reason); + decisionCache.add(decision); trafficMarkedToOffload += throughput; atLeastOneBundleSelected = true; remainingTopBundles--; @@ -403,26 +417,24 @@ && isTransferable(context, availableBrokers, } if (debugMode) { - log.info("selectedBundlesCache:{}", selectedBundlesCache); + log.info("decisionCache:{}", decisionCache); } - - if (decision.getUnloads().isEmpty()) { - decision.skip( - numOfOverloadedBrokers, - numOfUnderloadedBrokers, - numOfBrokersWithEmptyLoadData, - numOfBrokersWithFewBundles); - } else { - decision.succeed( - numOfOverloadedBrokers, - numOfUnderloadedBrokers); + if (decisionCache.isEmpty()) { + UnloadDecision.Reason reason; + if (numOfBrokersWithEmptyLoadData > 0) { + reason = NoLoadData; + } else if (numOfBrokersWithFewBundles > 0) { + reason = NoBundles; + } else { + reason = Balanced; + } + counter.update(Skip, reason); } } catch (Throwable e) { log.error("Failed to process unloading. ", e); - decision.fail(); + this.counter.update(Failure, Unknown); } - - return decision; + return decisionCache; } @@ -457,7 +469,6 @@ private boolean isTransferable(LoadManagerContext context, if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) { return false; } - return true; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java index bc3c8eb6a94fd..31310c5c9cc80 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java @@ -18,21 +18,30 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.scheduler; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; +import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; +import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Reflections; @@ -43,6 +52,8 @@ public class UnloadScheduler implements LoadManagerScheduler { private final ScheduledExecutorService loadManagerExecutor; + private final PulsarService pulsar; + private final UnloadManager unloadManager; private final LoadManagerContext context; @@ -51,32 +62,49 @@ public class UnloadScheduler implements LoadManagerScheduler { private final ServiceConfiguration conf; + private final UnloadCounter counter; + + private final AtomicReference> unloadMetrics; + + private long counterLastUpdatedAt = 0; + private volatile ScheduledFuture task; + private final Set unloadBrokers; + private final Map recentlyUnloadedBundles; private final Map recentlyUnloadedBrokers; - private volatile CompletableFuture currentRunningFuture = null; - - public UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + public UnloadScheduler(PulsarService pulsar, + ScheduledExecutorService loadManagerExecutor, UnloadManager unloadManager, LoadManagerContext context, - ServiceUnitStateChannel channel) { - this(loadManagerExecutor, unloadManager, context, - channel, createNamespaceUnloadStrategy(context.brokerConfiguration())); + ServiceUnitStateChannel channel, + AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper, + UnloadCounter counter, + AtomicReference> unloadMetrics) { + this(pulsar, loadManagerExecutor, unloadManager, context, channel, + createNamespaceUnloadStrategy(pulsar, antiAffinityGroupPolicyHelper, counter), counter, unloadMetrics); } @VisibleForTesting - protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + protected UnloadScheduler(PulsarService pulsar, + ScheduledExecutorService loadManagerExecutor, UnloadManager unloadManager, LoadManagerContext context, ServiceUnitStateChannel channel, - NamespaceUnloadStrategy strategy) { + NamespaceUnloadStrategy strategy, + UnloadCounter counter, + AtomicReference> unloadMetrics) { + this.pulsar = pulsar; this.namespaceUnloadStrategy = strategy; this.recentlyUnloadedBundles = new HashMap<>(); this.recentlyUnloadedBrokers = new HashMap<>(); + this.unloadBrokers = new HashSet<>(); this.loadManagerExecutor = loadManagerExecutor; + this.counter = counter; + this.unloadMetrics = unloadMetrics; this.unloadManager = unloadManager; this.context = context; this.conf = context.brokerConfiguration(); @@ -96,62 +124,72 @@ public synchronized void execute() { } return; } - if (currentRunningFuture != null && !currentRunningFuture.isDone()) { - if (debugMode) { - log.info("Auto namespace unload is running. Skipping."); - } - return; - } // Remove bundles who have been unloaded for longer than the grace period from the recently unloaded map. final long timeout = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(conf.getLoadBalancerSheddingGracePeriodMinutes()); recentlyUnloadedBundles.keySet().removeIf(e -> recentlyUnloadedBundles.get(e) < timeout); - this.currentRunningFuture = channel.isChannelOwnerAsync().thenCompose(isChannelOwner -> { - if (!isChannelOwner) { - if (debugMode) { - log.info("Current broker is not channel owner. Skipping."); + long asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs(); + synchronized (namespaceUnloadStrategy) { + try { + Boolean isChannelOwner = channel.isChannelOwnerAsync().get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS); + if (!isChannelOwner) { + if (debugMode) { + log.info("Current broker is not channel owner. Skipping."); + } + return; } - return CompletableFuture.completedFuture(null); - } - return context.brokerRegistry().getAvailableBrokersAsync().thenCompose(availableBrokers -> { + List availableBrokers = context.brokerRegistry().getAvailableBrokersAsync() + .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS); if (debugMode) { - log.info("Available brokers: {}", availableBrokers); + log.info("Available brokers: {}", availableBrokers); } if (availableBrokers.size() <= 1) { log.info("Only 1 broker available: no load shedding will be performed. Skipping."); - return CompletableFuture.completedFuture(null); + return; } - final UnloadDecision unloadDecision = namespaceUnloadStrategy + final Set decisions = namespaceUnloadStrategy .findBundlesForUnloading(context, recentlyUnloadedBundles, recentlyUnloadedBrokers); if (debugMode) { log.info("[{}] Unload decision result: {}", - namespaceUnloadStrategy.getClass().getSimpleName(), unloadDecision.toString()); + namespaceUnloadStrategy.getClass().getSimpleName(), decisions); } - if (unloadDecision.getUnloads().isEmpty()) { + if (decisions.isEmpty()) { if (debugMode) { log.info("[{}] Unload decision unloads is empty. Skipping.", namespaceUnloadStrategy.getClass().getSimpleName()); } - return CompletableFuture.completedFuture(null); + return; } List> futures = new ArrayList<>(); - unloadDecision.getUnloads().forEach((broker, unload) -> { - log.info("[{}] Unloading bundle: {}", namespaceUnloadStrategy.getClass().getSimpleName(), unload); - futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload), unload.serviceUnit(), - conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS) - .thenAccept(__ -> { - recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis()); - recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis()); - })); - }); - return FutureUtil.waitForAll(futures).exceptionally(ex -> { - log.error("[{}] Namespace unload has exception.", - namespaceUnloadStrategy.getClass().getSimpleName(), ex); - return null; + unloadBrokers.clear(); + decisions.forEach(decision -> { + if (decision.getLabel() == Success) { + Unload unload = decision.getUnload(); + log.info("[{}] Unloading bundle: {}", + namespaceUnloadStrategy.getClass().getSimpleName(), unload); + futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload), + unload.serviceUnit(), decision, asyncOpTimeoutMs, TimeUnit.MILLISECONDS) + .thenAccept(__ -> { + unloadBrokers.add(unload.sourceBroker()); + recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis()); + recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis()); + })); + } }); - }); - }); + FutureUtil.waitForAll(futures) + .whenComplete((__, ex) -> counter.updateUnloadBrokerCount(unloadBrokers.size())) + .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS); + } catch (Exception ex) { + log.error("[{}] Namespace unload has exception.", + namespaceUnloadStrategy.getClass().getSimpleName(), ex); + } finally { + if (counter.updatedAt() > counterLastUpdatedAt) { + unloadMetrics.set(counter.toMetrics(pulsar.getAdvertisedAddress())); + counterLastUpdatedAt = counter.updatedAt(); + } + } + } } @Override @@ -174,7 +212,10 @@ public void close() { this.recentlyUnloadedBrokers.clear(); } - private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(ServiceConfiguration conf) { + private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar, + AntiAffinityGroupPolicyHelper helper, + UnloadCounter counter) { + ServiceConfiguration conf = pulsar.getConfiguration(); try { return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), NamespaceUnloadStrategy.class, Thread.currentThread().getContextClassLoader()); @@ -183,7 +224,7 @@ private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(ServiceConf conf.getLoadBalancerLoadPlacementStrategy(), e); } log.error("create namespace unload strategy failed. using TransferShedder instead."); - return new TransferShedder(); + return new TransferShedder(pulsar, counter, helper); } private boolean isLoadBalancerSheddingEnabled() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index aa8583c6b57b6..354a27a63c02a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -64,8 +64,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentMap; -import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -74,9 +72,7 @@ import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; -import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; -import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -206,10 +202,9 @@ protected void cleanup() throws Exception { } @BeforeMethod - protected void initializeState() throws IllegalAccessException { + protected void initializeState() throws PulsarAdminException { + admin.namespaces().unload("public/default"); reset(primaryLoadManager, secondaryLoadManager); - cleanTableView(channel1); - cleanTableView(channel2); } @Test @@ -561,20 +556,20 @@ public void testGetMetrics() throws Exception { FieldUtils.writeDeclaredField(unloadCounter, "loadStd", 0.3, true); FieldUtils.writeDeclaredField(unloadCounter, "breakdownCounters", Map.of( Success, new LinkedHashMap<>() {{ - put(Overloaded, new MutableLong(1)); - put(Underloaded, new MutableLong(2)); + put(Overloaded, new AtomicLong(1)); + put(Underloaded, new AtomicLong(2)); }}, Skip, new LinkedHashMap<>() {{ - put(Balanced, new MutableLong(3)); - put(NoBundles, new MutableLong(4)); - put(CoolDown, new MutableLong(5)); - put(OutDatedData, new MutableLong(6)); - put(NoLoadData, new MutableLong(7)); - put(NoBrokers, new MutableLong(8)); - put(Unknown, new MutableLong(9)); + put(Balanced, new AtomicLong(3)); + put(NoBundles, new AtomicLong(4)); + put(CoolDown, new AtomicLong(5)); + put(OutDatedData, new AtomicLong(6)); + put(NoLoadData, new AtomicLong(7)); + put(NoBrokers, new AtomicLong(8)); + put(Unknown, new AtomicLong(9)); }}, Failure, Map.of( - Unknown, new MutableLong(10)) + Unknown, new AtomicLong(10)) ), true); unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress())); } @@ -722,15 +717,6 @@ public void initialize(PulsarService pulsar) { } - private static void cleanTableView(ServiceUnitStateChannel channel) - throws IllegalAccessException { - var tv = (TableViewImpl) - FieldUtils.readField(channel, "tableview", true); - var cache = (ConcurrentMap) - FieldUtils.readField(tv, "data", true); - cache.clear(); - } - private void setPrimaryLoadManager() throws IllegalAccessException { ExtensibleLoadManagerWrapper wrapper = (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java index 75ef913b8a851..6a2ae1cc562cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java @@ -19,6 +19,10 @@ package org.apache.pulsar.broker.loadbalance.extensions.manager; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -32,6 +36,9 @@ import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; +import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.common.util.FutureUtil; import org.testng.annotations.Test; @@ -41,10 +48,13 @@ public class UnloadManagerTest { @Test public void testEventPubFutureHasException() { - UnloadManager manager = new UnloadManager(); + UnloadCounter counter = new UnloadCounter(); + UnloadManager manager = new UnloadManager(counter); + var unloadDecision = + new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = manager.waitAsync(FutureUtil.failedFuture(new Exception("test")), - "bundle-1", 10, TimeUnit.SECONDS); + "bundle-1", unloadDecision, 10, TimeUnit.SECONDS); assertTrue(future.isCompletedExceptionally()); try { @@ -53,14 +63,18 @@ public void testEventPubFutureHasException() { } catch (Exception ex) { assertEquals(ex.getCause().getMessage(), "test"); } + assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1); } @Test public void testTimeout() throws IllegalAccessException { - UnloadManager manager = new UnloadManager(); + UnloadCounter counter = new UnloadCounter(); + UnloadManager manager = new UnloadManager(counter); + var unloadDecision = + new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = manager.waitAsync(CompletableFuture.completedFuture(null), - "bundle-1", 3, TimeUnit.SECONDS); + "bundle-1", unloadDecision, 3, TimeUnit.SECONDS); Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); assertEquals(inFlightUnloadRequestMap.size(), 1); @@ -73,14 +87,18 @@ public void testTimeout() throws IllegalAccessException { } assertEquals(inFlightUnloadRequestMap.size(), 0); + assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1); } @Test public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException { - UnloadManager manager = new UnloadManager(); + UnloadCounter counter = new UnloadCounter(); + UnloadManager manager = new UnloadManager(counter); + var unloadDecision = + new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = manager.waitAsync(CompletableFuture.completedFuture(null), - "bundle-1", 5, TimeUnit.SECONDS); + "bundle-1", unloadDecision, 5, TimeUnit.SECONDS); Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); assertEquals(inFlightUnloadRequestMap.size(), 1); @@ -109,10 +127,11 @@ public void testSuccess() throws IllegalAccessException, ExecutionException, Int new ServiceUnitStateData(ServiceUnitState.Free, "broker-1", VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 0); future.get(); + assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 1); // Success with Owned state. future = manager.waitAsync(CompletableFuture.completedFuture(null), - "bundle-1", 5, TimeUnit.SECONDS); + "bundle-1", unloadDecision, 5, TimeUnit.SECONDS); inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); assertEquals(inFlightUnloadRequestMap.size(), 1); @@ -121,14 +140,19 @@ public void testSuccess() throws IllegalAccessException, ExecutionException, Int new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 0); future.get(); + + assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 2); } @Test public void testFailedStage() throws IllegalAccessException { - UnloadManager manager = new UnloadManager(); + UnloadCounter counter = new UnloadCounter(); + UnloadManager manager = new UnloadManager(counter); + var unloadDecision = + new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = manager.waitAsync(CompletableFuture.completedFuture(null), - "bundle-1", 5, TimeUnit.SECONDS); + "bundle-1", unloadDecision, 5, TimeUnit.SECONDS); Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); assertEquals(inFlightUnloadRequestMap.size(), 1); @@ -146,14 +170,18 @@ public void testFailedStage() throws IllegalAccessException { } assertEquals(inFlightUnloadRequestMap.size(), 0); + assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1); } @Test public void testClose() throws IllegalAccessException { - UnloadManager manager = new UnloadManager(); + UnloadCounter counter = new UnloadCounter(); + UnloadManager manager = new UnloadManager(counter); + var unloadDecision = + new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = manager.waitAsync(CompletableFuture.completedFuture(null), - "bundle-1", 5, TimeUnit.SECONDS); + "bundle-1", unloadDecision,5, TimeUnit.SECONDS); Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); assertEquals(inFlightUnloadRequestMap.size(), 1); manager.close(); @@ -165,6 +193,7 @@ public void testClose() throws IllegalAccessException { } catch (Exception ex) { assertTrue(ex.getCause() instanceof IllegalStateException); } + assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1); } private Map> getInFlightUnloadRequestMap(UnloadManager manager) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 3955f1ed9af2e..a89f33bb98737 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -45,6 +45,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Random; @@ -64,6 +65,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; @@ -351,19 +353,19 @@ public void startTableView() throws LoadDataStoreException { @Test public void testEmptyBrokerLoadData() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = getContext(); ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.setReason(NoBrokers); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBrokers).get(), 1); } @Test public void testEmptyTopBundlesLoadData() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = getContext(); ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); @@ -373,21 +375,20 @@ public void testEmptyTopBundlesLoadData() { brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 20)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.setReason(NoLoadData); - expected.setLoadAvg(0.39999999999999997); - expected.setLoadStd(0.35590260840104376); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoLoadData).get(), 1); + assertEquals(counter.getLoadAvg(), 0.39999999999999997); + assertEquals(counter.getLoadStd(), 0.35590260840104376); } @Test public void testOutDatedLoadData() throws IllegalAccessException { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - assertEquals(res.getUnloads().size(), 2); + assertEquals(res.size(), 2); FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker1").get(), "updatedAt", 0, true); @@ -398,16 +399,14 @@ public void testOutDatedLoadData() throws IllegalAccessException { res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.setReason(OutDatedData); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(OutDatedData).get(), 1); } @Test public void testRecentlyUnloadedBrokers() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); Map recentlyUnloadedBrokers = new HashMap<>(); @@ -416,32 +415,27 @@ public void testRecentlyUnloadedBrokers() { recentlyUnloadedBrokers.put("broker1", oldTS); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers); - var expected = new UnloadDecision(); - var unloads = expected.getUnloads(); - unloads.put("broker5", - new Unload("broker5", bundleE1, Optional.of("broker1"))); - unloads.put("broker4", - new Unload("broker4", bundleD1, Optional.of("broker2"))); - - expected.setLabel(Success); - expected.setReason(Overloaded); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Overloaded)); assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); var now = System.currentTimeMillis(); recentlyUnloadedBrokers.put("broker1", now); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers); - expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.setReason(CoolDown); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(CoolDown).get(), 1); } @Test public void testRecentlyUnloadedBundles() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); Map recentlyUnloadedBundles = new HashMap<>(); var now = System.currentTimeMillis(); @@ -451,35 +445,34 @@ public void testRecentlyUnloadedBundles() { recentlyUnloadedBundles.put(bundleD2, now); var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of()); - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.skip(NoBundles); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } @Test public void testGetAvailableBrokersFailed() { - TransferShedder transferShedder = new TransferShedder(); + var pulsar = getMockPulsar(); + UnloadCounter counter = new UnloadCounter(); + AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class); + TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper); var ctx = setupContext(); BrokerRegistry registry = ctx.brokerRegistry(); doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.skip(Unknown); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } @Test(timeOut = 30 * 1000) public void testBundlesWithIsolationPolicies() throws IllegalAccessException { - - - TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper); + var pulsar = getMockPulsar(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper)); var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); @@ -487,33 +480,27 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException { IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPoliciesSpy); FieldUtils.writeDeclaredField(transferShedder, "isolationPoliciesHelper", isolationPoliciesHelper, true); - // Test transfer to a has isolation policies broker. setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", Set.of("broker5"), Set.of(), Set.of(), 1); var ctx = setupContext(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - var unloads = expected.getUnloads(); - unloads.put("broker4", - new Unload("broker4", bundleD1, Optional.of("broker2"))); - expected.setLabel(Success); - expected.setReason(Overloaded); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Overloaded)); assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); // Test unload a has isolation policies broker. ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - expected = new UnloadDecision(); - unloads = expected.getUnloads(); - unloads.put("broker4", - new Unload("broker4", bundleD1, Optional.empty())); - expected.setLabel(Success); - expected.setReason(Overloaded); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); + + expected = new HashSet<>(); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.empty()), + Success, Overloaded)); assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } public BrokerLookupData getLookupData() { @@ -590,7 +577,8 @@ private PulsarService getMockPulsar() { @Test public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException { var pulsar = getMockPulsar(); - TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper); + var counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper); var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); doReturn(false).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); @@ -603,29 +591,26 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any()); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.skip(NoBundles); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected2 = new UnloadDecision(); - var unloads = expected2.getUnloads(); - unloads.put("broker5", - new Unload("broker5", bundleE1, Optional.of("broker1"))); - expected2.setLabel(Success); - expected2.setReason(Overloaded); - expected2.setLoadAvg(setupLoadAvg); - expected2.setLoadStd(setupLoadStd); - assertEquals(res2, expected2); + var expected2 = new HashSet<>(); + expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + assertEquals(res, expected2); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } @Test public void testTargetStd() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = getContext(); ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); @@ -641,17 +626,16 @@ public void testTargetStd() { var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.skip(Balanced); - expected.setLoadAvg(0.2000000063578288); - expected.setLoadStd(0.08164966587949089); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(Balanced).get(), 1); + assertEquals(counter.getLoadAvg(), 0.2000000063578288); + assertEquals(counter.getLoadStd(), 0.08164966587949089); } @Test public void testSingleTopBundlesLoadData() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1)); @@ -661,38 +645,35 @@ public void testSingleTopBundlesLoadData() { topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 70)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.skip(NoBundles); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); - assertEquals(res, expected); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } @Test public void testTargetStdAfterTransfer() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55)); brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - var unloads = expected.getUnloads(); - unloads.put("broker5", - new Unload("broker5", bundleE1, Optional.of("broker1"))); - expected.setLabel(Success); - expected.setReason(Overloaded); - expected.setLoadAvg(0.26400000000000007); - expected.setLoadStd(0.27644891028904417); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), 0.26400000000000007); + assertEquals(counter.getLoadStd(), 0.27644891028904417); } @Test public void testMinBrokerWithZeroTraffic() throws IllegalAccessException { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55)); @@ -705,64 +686,57 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException { var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - var unloads = expected.getUnloads(); - unloads.put("broker5", - new Unload("broker5", bundleE1, Optional.of("broker1"))); - unloads.put("broker4", - new Unload("broker4", bundleD1, Optional.of("broker2"))); - expected.setLabel(Success); - expected.setReason(Underloaded); - expected.setLoadAvg(0.26400000000000007); - expected.setLoadStd(0.27644891028904417); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Underloaded)); assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), 0.26400000000000007); + assertEquals(counter.getLoadStd(), 0.27644891028904417); } @Test public void testMaxNumberOfTransfersPerShedderCycle() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); ctx.brokerConfiguration() .setLoadBalancerMaxNumberOfBrokerTransfersPerCycle(10); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - - var expected = new UnloadDecision(); - var unloads = expected.getUnloads(); - unloads.put("broker5", - new Unload("broker5", bundleE1, Optional.of("broker1"))); - unloads.put("broker4", - new Unload("broker4", bundleD1, Optional.of("broker2"))); - expected.setLabel(Success); - expected.setReason(Overloaded); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Overloaded)); assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } @Test public void testRemainingTopBundles() { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new UnloadDecision(); - var unloads = expected.getUnloads(); - unloads.put("broker5", - new Unload("broker5", bundleE1, Optional.of("broker1"))); - unloads.put("broker4", - new Unload("broker4", bundleD1, Optional.of("broker2"))); - expected.setLabel(Success); - expected.setReason(Overloaded); - expected.setLoadAvg(setupLoadAvg); - expected.setLoadStd(setupLoadStd); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Overloaded)); assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } @Test public void testRandomLoad() throws IllegalAccessException { - TransferShedder transferShedder = new TransferShedder(); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); for (int i = 0; i < 5; i++) { var ctx = setupContext(10); var conf = ctx.brokerConfiguration(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java index 73d4eb1f18bfb..7d9cf556360e4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java @@ -28,23 +28,29 @@ import static org.mockito.Mockito.verify; import com.google.common.collect.Lists; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.stats.Metrics; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; @Test(groups = "broker") public class UnloadSchedulerTest { @@ -70,23 +76,28 @@ public void tearDown() { @Test(timeOut = 30 * 1000) public void testExecuteSuccess() { + AtomicReference> reference = new AtomicReference<>(); + UnloadCounter counter = new UnloadCounter(); LoadManagerContext context = setupContext(); BrokerRegistry registry = context.brokerRegistry(); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); UnloadManager unloadManager = mock(UnloadManager.class); + PulsarService pulsar = mock(PulsarService.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); doReturn(CompletableFuture.completedFuture(Lists.newArrayList("broker-1", "broker-2"))) .when(registry).getAvailableBrokersAsync(); doReturn(CompletableFuture.completedFuture(null)).when(channel).publishUnloadEventAsync(any()); doReturn(CompletableFuture.completedFuture(null)).when(unloadManager) - .waitAsync(any(), any(), anyLong(), any()); + .waitAsync(any(), any(), any(), anyLong(), any()); UnloadDecision decision = new UnloadDecision(); Unload unload = new Unload("broker-1", "bundle-1"); - decision.getUnloads().put("broker-1", unload); - doReturn(decision).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); + decision.setUnload(unload); + decision.setLabel(UnloadDecision.Label.Success); + doReturn(Set.of(decision)).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); + UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context, + channel, unloadStrategy, counter, reference); scheduler.execute(); @@ -94,7 +105,7 @@ public void testExecuteSuccess() { // Test empty unload. UnloadDecision emptyUnload = new UnloadDecision(); - doReturn(emptyUnload).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); + doReturn(Set.of(emptyUnload)).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); scheduler.execute(); @@ -103,26 +114,29 @@ public void testExecuteSuccess() { @Test(timeOut = 30 * 1000) public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedException { + AtomicReference> reference = new AtomicReference<>(); + UnloadCounter counter = new UnloadCounter(); LoadManagerContext context = setupContext(); BrokerRegistry registry = context.brokerRegistry(); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); UnloadManager unloadManager = mock(UnloadManager.class); + PulsarService pulsar = mock(PulsarService.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); doAnswer(__ -> CompletableFuture.supplyAsync(() -> { try { // Delay 5 seconds to finish. - TimeUnit.SECONDS.sleep(5); + TimeUnit.SECONDS.sleep(1); } catch (InterruptedException e) { throw new RuntimeException(e); } return Lists.newArrayList("broker-1", "broker-2"); }, Executors.newFixedThreadPool(1))).when(registry).getAvailableBrokersAsync(); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); - - ExecutorService executorService = Executors.newFixedThreadPool(10); - CountDownLatch latch = new CountDownLatch(10); - for (int i = 0; i < 10; i++) { + UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context, + channel, unloadStrategy, counter, reference); + ExecutorService executorService = Executors.newFixedThreadPool(5); + CountDownLatch latch = new CountDownLatch(5); + for (int i = 0; i < 5; i++) { executorService.execute(() -> { scheduler.execute(); latch.countDown(); @@ -130,18 +144,21 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio } latch.await(); - verify(registry, times(1)).getAvailableBrokersAsync(); + verify(registry, times(5)).getAvailableBrokersAsync(); } @Test(timeOut = 30 * 1000) public void testDisableLoadBalancer() { + AtomicReference> reference = new AtomicReference<>(); + UnloadCounter counter = new UnloadCounter(); LoadManagerContext context = setupContext(); context.brokerConfiguration().setLoadBalancerEnabled(false); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); UnloadManager unloadManager = mock(UnloadManager.class); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); - + PulsarService pulsar = mock(PulsarService.class); + UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context, + channel, unloadStrategy, counter, reference); scheduler.execute(); verify(channel, times(0)).isChannelOwnerAsync(); @@ -155,12 +172,16 @@ public void testDisableLoadBalancer() { @Test(timeOut = 30 * 1000) public void testNotChannelOwner() { + AtomicReference> reference = new AtomicReference<>(); + UnloadCounter counter = new UnloadCounter(); LoadManagerContext context = setupContext(); context.brokerConfiguration().setLoadBalancerEnabled(false); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); UnloadManager unloadManager = mock(UnloadManager.class); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); + PulsarService pulsar = mock(PulsarService.class); + UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context, + channel, unloadStrategy, counter, reference); doReturn(CompletableFuture.completedFuture(false)).when(channel).isChannelOwnerAsync(); scheduler.execute(); From 13d78dcbdb59907d2541e7ee8fe96101fd441b12 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Sun, 19 Mar 2023 11:16:34 +0800 Subject: [PATCH 004/280] [improve][broker] PIP-192: Make split admin API functional (#19773) Master Issue: https://github.com/apache/pulsar/issues/16691 ### Motivation Raising a PR to implement https://github.com/apache/pulsar/issues/16691, this PR makes the split admin API functional. ### Modifications * Make the split admin API functional when enabling the `ExtensibleLoadManager`. * Support more split algorithm. --- .../extensions/ExtensibleLoadManagerImpl.java | 60 +++++++++++++++++- .../channel/ServiceUnitStateChannelImpl.java | 34 +++++++++-- .../channel/ServiceUnitStateData.java | 17 ++++-- .../loadbalance/extensions/models/Split.java | 7 +++ ...faultNamespaceBundleSplitStrategyImpl.java | 2 +- .../broker/namespace/NamespaceService.java | 9 +-- .../pulsar/broker/web/PulsarWebResource.java | 7 ++- .../pulsar/common/naming/NamespaceBundle.java | 2 +- ...pecifiedPositionsBundleSplitAlgorithm.java | 2 +- .../ExtensibleLoadManagerImplTest.java | 61 +++++++++++++++++++ .../channel/ServiceUnitStateChannelTest.java | 11 ++-- .../extensions/channel/models/SplitTest.java | 22 ++++++- .../scheduler/SplitSchedulerTest.java | 5 +- ...faultNamespaceBundleSplitStrategyTest.java | 17 +++--- 14 files changed, 216 insertions(+), 40 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 486c32153589f..fedcad4d009b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -18,10 +18,11 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; +import static java.lang.String.format; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; @@ -56,7 +57,9 @@ import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager; import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; @@ -71,11 +74,15 @@ import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreFactory; import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; @@ -406,7 +413,7 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, } Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker); UnloadDecision unloadDecision = - new UnloadDecision(unload, Success, Admin); + new UnloadDecision(unload, UnloadDecision.Label.Success, UnloadDecision.Reason.Admin); return unloadAsync(unloadDecision, conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); }); @@ -421,6 +428,53 @@ private CompletableFuture unloadAsync(UnloadDecision unloadDecision, .thenRun(() -> unloadCounter.updateUnloadBrokerCount(1)); } + public CompletableFuture splitNamespaceBundleAsync(ServiceUnitId bundle, + NamespaceBundleSplitAlgorithm splitAlgorithm, + List boundaries) { + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle.toString()); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle.toString()); + NamespaceBundle namespaceBundle = + pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange); + return pulsar.getNamespaceService().getSplitBoundary(namespaceBundle, splitAlgorithm, boundaries) + .thenCompose(splitBundlesPair -> { + if (splitBundlesPair == null) { + String msg = format("Bundle %s not found under namespace", namespaceBundle); + log.error(msg); + return FutureUtil.failedFuture(new IllegalStateException(msg)); + } + + return getOwnershipAsync(Optional.empty(), bundle) + .thenCompose(brokerOpt -> { + if (brokerOpt.isEmpty()) { + String msg = String.format("Namespace bundle: %s is not owned by any broker.", + bundle); + log.warn(msg); + throw new IllegalStateException(msg); + } + String sourceBroker = brokerOpt.get(); + SplitDecision splitDecision = new SplitDecision(); + List splitBundles = splitBundlesPair.getRight(); + Map> splitServiceUnitToDestBroker = new HashMap<>(); + splitBundles.forEach(splitBundle -> splitServiceUnitToDestBroker + .put(splitBundle.getBundleRange(), Optional.empty())); + splitDecision.setSplit( + new Split(bundle.toString(), sourceBroker, splitServiceUnitToDestBroker)); + splitDecision.setLabel(Success); + splitDecision.setReason(Admin); + return splitAsync(splitDecision, + conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + }); + }); + } + + private CompletableFuture splitAsync(SplitDecision decision, + long timeout, + TimeUnit timeoutUnit) { + Split split = decision.getSplit(); + CompletableFuture future = serviceUnitStateChannel.publishSplitEventAsync(split); + return splitManager.waitAsync(future, decision.getSplit().serviceUnit(), decision, timeout, timeoutUnit); + } + @Override public void close() throws PulsarServerException { if (!this.started) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 791d02649baf1..ca9e55923e799 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -87,6 +87,7 @@ import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; @@ -533,7 +534,8 @@ public CompletableFuture publishSplitEventAsync(Split split) { eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = split.serviceUnit(); ServiceUnitStateData next = - new ServiceUnitStateData(Splitting, null, split.sourceBroker(), getNextVersionId(serviceUnit)); + new ServiceUnitStateData(Splitting, null, split.sourceBroker(), + split.splitServiceUnitToDestBroker(), getNextVersionId(serviceUnit)); return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { eventCounters.get(eventType).getFailure().incrementAndGet(); @@ -784,22 +786,40 @@ private CompletableFuture closeServiceUnit(String serviceUnit) { } private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnitStateData data) { - // Write the child ownerships to BSC. + // Write the child ownerships to channel. long startTime = System.nanoTime(); NamespaceService namespaceService = pulsar.getNamespaceService(); NamespaceBundleFactory bundleFactory = namespaceService.getNamespaceBundleFactory(); NamespaceBundle bundle = getNamespaceBundle(serviceUnit); CompletableFuture completionFuture = new CompletableFuture<>(); + Map> bundleToDestBroker = data.splitServiceUnitToDestBroker(); + List boundaries = null; + NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm = + namespaceService.getNamespaceBundleSplitAlgorithmByName( + config.getDefaultNamespaceBundleSplitAlgorithm()); + if (bundleToDestBroker != null && bundleToDestBroker.size() == 2) { + Set boundariesSet = new HashSet<>(); + String namespace = bundle.getNamespaceObject().toString(); + bundleToDestBroker.forEach((bundleRange, destBroker) -> { + NamespaceBundle subBundle = bundleFactory.getBundle(namespace, bundleRange); + boundariesSet.add(subBundle.getKeyRange().lowerEndpoint()); + boundariesSet.add(subBundle.getKeyRange().upperEndpoint()); + }); + boundaries = new ArrayList<>(boundariesSet); + nsBundleSplitAlgorithm = NamespaceBundleSplitAlgorithm.SPECIFIED_POSITIONS_DIVIDE_ALGO; + } final AtomicInteger counter = new AtomicInteger(0); - this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, bundle, serviceUnit, data, - counter, startTime, completionFuture); + this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, nsBundleSplitAlgorithm, + bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture); return completionFuture; } @VisibleForTesting protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, NamespaceBundleFactory bundleFactory, + NamespaceBundleSplitAlgorithm algorithm, NamespaceBundle bundle, + List boundaries, String serviceUnit, ServiceUnitStateData data, AtomicInteger counter, @@ -807,7 +827,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, CompletableFuture completionFuture) { CompletableFuture> updateFuture = new CompletableFuture<>(); - pulsar.getNamespaceService().getSplitBoundary(bundle, null).thenAccept(splitBundlesPair -> { + namespaceService.getSplitBoundary(bundle, algorithm, boundaries) + .thenAccept(splitBundlesPair -> { // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper. if (splitBundlesPair == null) { String msg = format("Bundle %s not found under namespace", serviceUnit); @@ -877,7 +898,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit", counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex); pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, - bundle, serviceUnit, data, counter, startTime, completionFuture), 100, MILLISECONDS); + algorithm, bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture), + 100, MILLISECONDS); } else if (throwable instanceof IllegalArgumentException) { completionFuture.completeExceptionally(throwable); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index c26dce83a4434..52c7e27d65033 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; /** @@ -27,7 +29,8 @@ */ public record ServiceUnitStateData( - ServiceUnitState state, String dstBroker, String sourceBroker, boolean force, long timestamp, long versionId) { + ServiceUnitState state, String dstBroker, String sourceBroker, + Map> splitServiceUnitToDestBroker, boolean force, long timestamp, long versionId) { public ServiceUnitStateData { Objects.requireNonNull(state); @@ -36,16 +39,22 @@ public record ServiceUnitStateData( } } + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, + Map> splitServiceUnitToDestBroker, long versionId) { + this(state, dstBroker, sourceBroker, splitServiceUnitToDestBroker, false, + System.currentTimeMillis(), versionId); + } + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, long versionId) { - this(state, dstBroker, sourceBroker, false, System.currentTimeMillis(), versionId); + this(state, dstBroker, sourceBroker, null, false, System.currentTimeMillis(), versionId); } public ServiceUnitStateData(ServiceUnitState state, String dstBroker, long versionId) { - this(state, dstBroker, null, false, System.currentTimeMillis(), versionId); + this(state, dstBroker, null, null, false, System.currentTimeMillis(), versionId); } public ServiceUnitStateData(ServiceUnitState state, String dstBroker, boolean force, long versionId) { - this(state, dstBroker, null, force, System.currentTimeMillis(), versionId); + this(state, dstBroker, null, null, force, System.currentTimeMillis(), versionId); } public static ServiceUnitState state(ServiceUnitStateData data) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java index fcec8b0ae5541..ac9a36e6dbfd3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java @@ -30,5 +30,12 @@ public record Split( public Split { Objects.requireNonNull(serviceUnit); + if (splitServiceUnitToDestBroker != null && splitServiceUnitToDestBroker.size() != 2) { + throw new IllegalArgumentException("Split service unit should be split into 2 service units."); + } + } + + public Split(String serviceUnit, String sourceBroker) { + this(serviceUnit, sourceBroker, null); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java index e572fd4161bdb..da04a287f1ac5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java @@ -138,7 +138,7 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS maxBundleBandwidth / LoadManagerShared.MIBI); } var decision = new SplitDecision(); - decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId(), new HashMap<>())); + decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId())); decision.succeed(reason); decisionCache.add(decision); int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 899539e1db6a7..de18d50f3e144 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -822,7 +822,10 @@ public boolean isNamespaceBundleDisabled(NamespaceBundle bundle) throws Exceptio public CompletableFuture splitAndOwnBundle(NamespaceBundle bundle, boolean unload, NamespaceBundleSplitAlgorithm splitAlgorithm, List boundaries) { - + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + return ExtensibleLoadManagerImpl.get(loadManager.get()) + .splitNamespaceBundleAsync(bundle, splitAlgorithm, boundaries); + } final CompletableFuture unloadFuture = new CompletableFuture<>(); final AtomicInteger counter = new AtomicInteger(BUNDLE_SPLIT_RETRY_LIMIT); splitAndOwnBundleOnceAndRetry(bundle, unload, counter, unloadFuture, splitAlgorithm, boundaries); @@ -963,10 +966,8 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, * @return A pair, left is target namespace bundle, right is split bundles. */ public CompletableFuture>> getSplitBoundary( - NamespaceBundle bundle, List boundaries) { + NamespaceBundle bundle, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm, List boundaries) { BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config); - NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm = - getNamespaceBundleSplitAlgorithmByName(config.getDefaultNamespaceBundleSplitAlgorithm()); CompletableFuture> splitBoundary = nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption); return splitBoundary.thenCompose(splitBoundaries -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 321a127ad97bd..c5713ebddaa2f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -636,9 +636,6 @@ protected CompletableFuture validateNamespaceBundleOwnershipAsy } catch (WebApplicationException wae) { return CompletableFuture.failedFuture(wae); } - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { - return CompletableFuture.completedFuture(nsBundle); - } return validateBundleOwnershipAsync(nsBundle, authoritative, readOnly) .thenApply(__ -> nsBundle); } @@ -718,6 +715,10 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund throw new RestException(Status.PRECONDITION_FAILED, "Failed to find ownership for ServiceUnit:" + bundle.toString()); } + // If the load manager is extensible load manager, we don't need check the authoritative. + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + return CompletableFuture.completedFuture(null); + } return nsService.isServiceUnitOwnedAsync(bundle) .thenAccept(owned -> { if (!owned) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java index 0b7e09fee2cec..78a80e077e42a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java @@ -127,7 +127,7 @@ private static String getKey(NamespaceName nsname, Range keyRange) { return String.format("%s/0x%08x_0x%08x", nsname, keyRange.lowerEndpoint(), keyRange.upperEndpoint()); } - Range getKeyRange() { + public Range getKeyRange() { return this.keyRange; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java index 6c341f185ca60..1fd9a85cfa782 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java @@ -54,4 +54,4 @@ public CompletableFuture> getSplitBoundary(BundleSplitOption bundleSp return CompletableFuture.completedFuture(splitBoundaries); }); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 354a27a63c02a..d977e63330586 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -95,6 +95,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfo; @@ -372,6 +373,66 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle) assertFalse(otherLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); } + @Test(timeOut = 30 * 1000) + public void testSplitBundleAdminAPI() throws Exception { + String namespace = "public/default"; + String topic = "persistent://" + namespace + "/test-split"; + admin.topics().createPartitionedTopic(topic, 10); + BundlesData bundles = admin.namespaces().getBundles(namespace); + int numBundles = bundles.getNumBundles(); + var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList(); + + String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1); + + long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2; + + admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null); + + BundlesData bundlesData = admin.namespaces().getBundles(namespace); + assertEquals(bundlesData.getNumBundles(), numBundles + 1); + String lowBundle = String.format("0x%08x", bundleRanges.get(0)); + String midBundle = String.format("0x%08x", mid); + String highBundle = String.format("0x%08x", bundleRanges.get(1)); + assertTrue(bundlesData.getBoundaries().contains(lowBundle)); + assertTrue(bundlesData.getBoundaries().contains(midBundle)); + assertTrue(bundlesData.getBoundaries().contains(highBundle)); + + // Test split bundle with invalid bundle range. + try { + admin.namespaces().splitNamespaceBundle(namespace, "invalid", true, null); + fail(); + } catch (PulsarAdminException ex) { + assertTrue(ex.getMessage().contains("Invalid bundle range")); + } + } + + @Test(timeOut = 30 * 1000) + public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { + String namespace = "public/default"; + String topic = "persistent://" + namespace + "/test-split-with-specific-position"; + admin.topics().createPartitionedTopic(topic, 10); + BundlesData bundles = admin.namespaces().getBundles(namespace); + int numBundles = bundles.getNumBundles(); + + var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList(); + + String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1); + + long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2; + long splitPosition = mid + 100; + + admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, + "specified_positions_divide", List.of(bundleRanges.get(0), bundleRanges.get(1), splitPosition)); + + BundlesData bundlesData = admin.namespaces().getBundles(namespace); + assertEquals(bundlesData.getNumBundles(), numBundles + 1); + String lowBundle = String.format("0x%08x", bundleRanges.get(0)); + String midBundle = String.format("0x%08x", splitPosition); + String highBundle = String.format("0x%08x", bundleRanges.get(1)); + assertTrue(bundlesData.getBoundaries().contains(lowBundle)); + assertTrue(bundlesData.getBoundaries().contains(midBundle)); + assertTrue(bundlesData.getBoundaries().contains(highBundle)); + } @Test public void testMoreThenOneFilter() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index e0bd69fce64c7..c05c8ac741565 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -54,7 +54,6 @@ import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertNotNull; import java.lang.reflect.Field; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -297,7 +296,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } } try { - channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1, Map.of())) + channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1)) .get(2, TimeUnit.SECONDS); } catch (ExecutionException e) { if (e.getCause() instanceof IllegalStateException) { @@ -554,11 +553,15 @@ public void splitAndRetryTest() throws Exception { } // Call the real method reset(namespaceService); + doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) + .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); return future; }).when(namespaceService).updateNamespaceBundles(any(), any()); doReturn(namespaceService).when(pulsar1).getNamespaceService(); + doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) + .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); - Split split = new Split(bundle, ownerAddr1.get(), new HashMap<>()); + Split split = new Split(bundle, ownerAddr1.get()); channel1.publishSplitEventAsync(split); waitUntilState(channel1, bundle, Deleted); @@ -570,7 +573,7 @@ public void splitAndRetryTest() throws Exception { validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); // Verify the retry count verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount + 1)) - .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), anyLong(), any()); + .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), any(), any(), anyLong(), any()); // Assert child bundle ownerships in the channels. String childBundle1 = "public/default/0x7fffffff_0xffffffff"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java index 67c9555c88f01..1e23ea74dca5e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.channel.models; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -32,6 +33,7 @@ public class SplitTest { public void testConstructor() { Map> map = new HashMap<>(); map.put("C", Optional.of("test")); + map.put("D", Optional.of("test")); Split split = new Split("A", "B", map); assertEquals(split.serviceUnit(), "A"); @@ -44,4 +46,22 @@ public void testNullBundle() { new Split(null, "A", Map.of()); } -} \ No newline at end of file + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidSplitServiceUnitToDestBroker() { + Map> map = new HashMap<>(); + map.put("C", Optional.of("test")); + map.put("D", Optional.of("test")); + map.put("E", Optional.of("test")); + new Split("A", "B", map); + } + + @Test + public void testNullSplitServiceUnitToDestBroker() { + var split = new Split("A", "B"); + assertEquals(split.serviceUnit(), "A"); + assertEquals(split.sourceBroker(), "B"); + assertNull(split.splitServiceUnitToDestBroker()); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java index 7988aa413366f..f7726f8e7a1aa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java @@ -27,7 +27,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; -import java.util.HashMap; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -78,11 +77,11 @@ public void setUp() { doReturn(CompletableFuture.completedFuture(null)).when(channel).publishSplitEventAsync(any()); decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle1, broker)); decision1.succeed(SplitDecision.Reason.MsgRate); decision2 = new SplitDecision(); - decision2.setSplit(new Split(bundle2, broker, new HashMap<>())); + decision2.setSplit(new Split(bundle2, broker)); decision2.succeed(SplitDecision.Reason.Sessions); Set decisions = Set.of(decision1, decision2); doReturn(decisions).when(strategy).findBundlesToSplit(any(), any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java index 71606bb85a3fe..deff6fb00fafb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java @@ -28,7 +28,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -172,14 +171,14 @@ public void testMaxMsgRate() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle1, broker)); decision1.succeed(SplitDecision.Reason.MsgRate); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle2, broker)); decision1.succeed(SplitDecision.Reason.MsgRate); assertEquals(actual, Set.of(decision1)); @@ -200,14 +199,14 @@ public void testMaxTopics() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle1, broker)); decision1.succeed(SplitDecision.Reason.Topics); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle2, broker)); decision1.succeed(SplitDecision.Reason.Topics); assertEquals(actual, Set.of(decision1)); @@ -231,14 +230,14 @@ public void testMaxSessions() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle1, broker)); decision1.succeed(SplitDecision.Reason.Sessions); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle2, broker)); decision1.succeed(SplitDecision.Reason.Sessions); assertEquals(actual, Set.of(decision1)); @@ -262,14 +261,14 @@ public void testMaxBandwidthMbytes() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle1, broker)); decision1.succeed(SplitDecision.Reason.Bandwidth); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker, new HashMap<>())); + decision1.setSplit(new Split(bundle2, broker)); decision1.succeed(SplitDecision.Reason.Bandwidth); assertEquals(actual, Set.of(decision1)); From f20dc93bc7d3defcc824857a558f2f78726f13a1 Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 19 Mar 2023 12:13:10 +0800 Subject: [PATCH 005/280] [improve][misc] Add idea icon (#19855) Signed-off-by: tison --- .github/changes-filter.yaml | 6 +++--- .idea/icon.svg | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 .idea/icon.svg diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index 3ec2fc22946da..be6faa957887d 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -3,13 +3,13 @@ all: - '**' docs: - - 'site2/**' - - 'deployment/**' - - '.asf.yaml' - '*.md' - '**/*.md' + - '.asf.yaml' - '.github/changes-filter.yaml' - '.github/ISSUE_TEMPLATE/**' + - '.idea/**' + - 'deployment/**' - 'wiki/**' tests: - added|modified: '**/src/test/java/**/*.java' diff --git a/.idea/icon.svg b/.idea/icon.svg new file mode 100644 index 0000000000000..bf9b232def4a4 --- /dev/null +++ b/.idea/icon.svg @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + \ No newline at end of file From 3c8ab7bc1fe1851ca24b0f12063cf2442276f656 Mon Sep 17 00:00:00 2001 From: ran Date: Mon, 20 Mar 2023 10:16:59 +0800 Subject: [PATCH 006/280] [fix][test] Fix flaky MetadataStoreStatsTest (#19829) --- .../service/ManagedLedgerCompressionTest.java | 2 +- .../broker/stats/MetadataStoreStatsTest.java | 81 +++++++++++++------ 2 files changed, 58 insertions(+), 25 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java index 4991d0a75363d..c13c8bd9fdc94 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java @@ -50,7 +50,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } - @Test(timeOut = 1000 * 20) + @Test(timeOut = 1000 * 30) public void testRestartBrokerEnableManagedLedgerInfoCompression() throws Exception { String topic = newTopicName(); @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java index f4deddf0cec9e..8ae0242c6232a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java @@ -21,9 +21,13 @@ import com.google.common.collect.Multimap; import java.io.ByteArrayOutputStream; import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.broker.service.BrokerTestBase; @@ -107,13 +111,17 @@ public void testMetadataStoreStats() throws Exception { Assert.assertTrue(opsLatency.size() > 1, metricsDebugMessage); Assert.assertTrue(putBytes.size() > 1, metricsDebugMessage); + Set expectedMetadataStoreName = new HashSet<>(); + expectedMetadataStoreName.add(MetadataStoreConfig.METADATA_STORE); + expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE); + + AtomicInteger matchCount = new AtomicInteger(0); for (PrometheusMetricsTest.Metric m : opsLatency) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); - Assert.assertNotNull(metadataStoreName, metricsDebugMessage); - Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage); + if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { + continue; + } Assert.assertNotNull(m.tags.get("status"), metricsDebugMessage); if (m.tags.get("status").equals("success")) { @@ -138,15 +146,19 @@ public void testMetadataStoreStats() throws Exception { } } } + // Because the combination quantity between status(success, fail) and type(get, del, put) is 6. + Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size() * 6); + + matchCount = new AtomicInteger(0); for (PrometheusMetricsTest.Metric m : putBytes) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); - Assert.assertNotNull(metadataStoreName, metricsDebugMessage); - Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage); + if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { + continue; + } Assert.assertTrue(m.value >= 0, metricsDebugMessage); } + Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); } @Test @@ -193,43 +205,64 @@ public void testBatchMetadataStoreMetrics() throws Exception { Assert.assertTrue(batchExecuteTime.size() > 0, metricsDebugMessage); Assert.assertTrue(opsPerBatch.size() > 0, metricsDebugMessage); + Set expectedMetadataStoreName = new HashSet<>(); + expectedMetadataStoreName.add(MetadataStoreConfig.METADATA_STORE); + expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE); + + AtomicInteger matchCount = new AtomicInteger(0); for (PrometheusMetricsTest.Metric m : executorQueueSize) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); - Assert.assertNotNull(metadataStoreName, metricsDebugMessage); - Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage); + if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { + continue; + } Assert.assertTrue(m.value >= 0, metricsDebugMessage); } + Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); + + matchCount = new AtomicInteger(0); for (PrometheusMetricsTest.Metric m : opsWaiting) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); - Assert.assertNotNull(metadataStoreName, metricsDebugMessage); - Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage); + if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { + continue; + } Assert.assertTrue(m.value >= 0, metricsDebugMessage); } + Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); + matchCount = new AtomicInteger(0); for (PrometheusMetricsTest.Metric m : batchExecuteTime) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); - Assert.assertNotNull(metadataStoreName, metricsDebugMessage); - Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage); + if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { + continue; + } Assert.assertTrue(m.value >= 0, metricsDebugMessage); } + Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); + matchCount = new AtomicInteger(0); for (PrometheusMetricsTest.Metric m : opsPerBatch) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); - Assert.assertNotNull(metadataStoreName, metricsDebugMessage); - Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) - || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage); + if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { + continue; + } Assert.assertTrue(m.value >= 0, metricsDebugMessage); } + Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); } + + private boolean isExpectedLabel(String metadataStoreName, Set expectedLabel, + AtomicInteger expectedLabelCount) { + if (StringUtils.isEmpty(metadataStoreName) + || !expectedLabel.contains(metadataStoreName)) { + return false; + } else { + expectedLabelCount.incrementAndGet(); + return true; + } + } + } From 7f5ba2bcbca2a712d11b02d3e2ef4dc2f3052a92 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 20 Mar 2023 10:29:50 +0800 Subject: [PATCH 007/280] [fix][broker] Fix create cluster with empty url (#19762) --- .../broker/admin/AdminApiClusterTest.java | 34 +++++++++++++++++++ .../pulsar/common/util/URIPreconditions.java | 2 +- .../common/util/URIPreconditionsTest.java | 2 ++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java index c37cc234b1213..28ebe39e0aa39 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java @@ -22,6 +22,8 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertThrows; import static org.testng.Assert.fail; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.UUID; import lombok.extern.slf4j.Slf4j; @@ -107,4 +109,36 @@ public void testDeleteExistFailureDomain() throws PulsarAdminException { admin.clusters().deleteFailureDomain(CLUSTER, domainName); } + + @Test + public void testCreateCluster() throws PulsarAdminException { + List clusterDataList = new ArrayList<>(); + clusterDataList.add(ClusterData.builder() + .serviceUrl("http://pulsar.app:8080") + .serviceUrlTls("") + .brokerServiceUrl("pulsar://pulsar.app:6650") + .brokerServiceUrlTls("") + .build()); + clusterDataList.add(ClusterData.builder() + .serviceUrl("") + .serviceUrlTls("https://pulsar.app:8443") + .brokerServiceUrl("") + .brokerServiceUrlTls("pulsar+ssl://pulsar.app:6651") + .build()); + clusterDataList.add(ClusterData.builder() + .serviceUrl("") + .serviceUrlTls("") + .brokerServiceUrl("") + .brokerServiceUrlTls("") + .build()); + clusterDataList.add(ClusterData.builder() + .serviceUrl(null) + .serviceUrlTls(null) + .brokerServiceUrl(null) + .brokerServiceUrlTls(null) + .build()); + for (int i = 0; i < clusterDataList.size(); i++) { + admin.clusters().createCluster("cluster-test-" + i, clusterDataList.get(i)); + } + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java index f68ed5e41a430..ea7fae2fdf0e9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java @@ -67,7 +67,7 @@ public static void checkURIIfPresent(@Nullable String uri, public static void checkURIIfPresent(@Nullable String uri, @Nonnull Predicate predicate, @Nullable String errorMessage) throws IllegalArgumentException { - if (uri == null) { + if (uri == null || uri.length() == 0) { return; } checkURI(uri, predicate, errorMessage); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java index d5809e8fdd0d3..f5b9e454e081d 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java @@ -30,6 +30,7 @@ public void testCheckURI() { // normal checkURI("http://pulsar.apache.org", uri -> true); checkURI("http://pulsar.apache.org", uri -> Objects.equals(uri.getScheme(), "http")); + checkURI("", uri -> true); // illegal try { checkURI("pulsar.apache.org", uri -> Objects.equals(uri.getScheme(), "http")); @@ -48,6 +49,7 @@ public void testCheckURI() { @Test public void testCheckURIIfPresent() { checkURIIfPresent(null, uri -> false); + checkURIIfPresent("", uri -> false); checkURIIfPresent("http://pulsar.apache.org", uri -> true); try { checkURIIfPresent("http/pulsar.apache.org", uri -> uri.getScheme() != null, "Error"); From 2637bda91d46f443762b22ac9963935725af2c8a Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 20 Mar 2023 16:39:09 +0800 Subject: [PATCH 008/280] [fix][meta] Follow up #19817, Fix race condition between ResourceLock update and invalidation (#19844) --- .../apache/pulsar/common/util/FutureUtil.java | 35 ++++++ .../pulsar/common/util/FutureUtilTest.java | 72 ++++++++++++ .../coordination/impl/LockManagerImpl.java | 2 +- .../coordination/impl/ResourceLockImpl.java | 106 +++++++----------- 4 files changed, 147 insertions(+), 68 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java index 162ef1e52ffd6..531769a1e452b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -35,6 +36,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.concurrent.ThreadSafe; /** * This class is aimed at simplifying work with {@code CompletableFuture}. @@ -166,6 +168,39 @@ public static Throwable unwrapCompletionException(Throwable ex) { } } + @ThreadSafe + public static class Sequencer { + private CompletableFuture sequencerFuture = CompletableFuture.completedFuture(null); + private final boolean allowExceptionBreakChain; + + public Sequencer(boolean allowExceptionBreakChain) { + this.allowExceptionBreakChain = allowExceptionBreakChain; + } + + public static Sequencer create(boolean allowExceptionBreakChain) { + return new Sequencer<>(allowExceptionBreakChain); + } + public static Sequencer create() { + return new Sequencer<>(false); + } + + /** + * @throws NullPointerException NPE when param is null + */ + public synchronized CompletableFuture sequential(Supplier> newTask) { + Objects.requireNonNull(newTask); + if (sequencerFuture.isDone()) { + if (sequencerFuture.isCompletedExceptionally() && allowExceptionBreakChain) { + return sequencerFuture; + } + return sequencerFuture = newTask.get(); + } + return sequencerFuture = allowExceptionBreakChain + ? sequencerFuture.thenCompose(__ -> newTask.get()) + : sequencerFuture.exceptionally(ex -> null).thenCompose(__ -> newTask.get()); + } + } + /** * Creates a new {@link CompletableFuture} instance with timeout handling. * diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java index 856ec49da6e9d..6df4494edf886 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java @@ -21,6 +21,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -32,6 +33,7 @@ import lombok.Cleanup; import org.assertj.core.util.Lists; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -178,4 +180,74 @@ public void testWaitForAny() { assertTrue(ex.getCause() instanceof RuntimeException); } } + + @Test + public void testSequencer() { + int concurrentNum = 1000; + final ScheduledExecutorService executor = Executors.newScheduledThreadPool(concurrentNum); + final FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create(); + // normal case -- allowExceptionBreakChain=false + final List list = new ArrayList<>(); + final List> futures = new ArrayList<>(); + for (int i = 0; i < concurrentNum; i++) { + int finalI = i; + futures.add(sequencer.sequential(() -> CompletableFuture.runAsync(() -> { + list.add(finalI); + }, executor))); + } + FutureUtil.waitForAll(futures).join(); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i), (Integer) i); + } + + // exception case -- allowExceptionBreakChain=false + final List list2 = new ArrayList<>(); + final List> futures2 = new ArrayList<>(); + for (int i = 0; i < concurrentNum; i++) { + int finalI = i; + futures2.add(sequencer.sequential(() -> CompletableFuture.runAsync(() -> { + if (finalI == 2) { + throw new IllegalStateException(); + } + list2.add(finalI); + }, executor))); + } + try { + FutureUtil.waitForAll(futures2).join(); + } catch (Throwable ignore) { + + } + for (int i = 0; i < concurrentNum - 1; i++) { + if (i >= 2) { + Assert.assertEquals(list2.get(i), i + 1); + } else { + Assert.assertEquals(list2.get(i), (Integer) i); + } + } + // allowExceptionBreakChain=true + final FutureUtil.Sequencer sequencer2 = FutureUtil.Sequencer.create(true); + final List list3 = new ArrayList<>(); + final List> futures3 = new ArrayList<>(); + for (int i = 0; i < concurrentNum; i++) { + int finalI = i; + futures3.add(sequencer2.sequential(() -> CompletableFuture.runAsync(() -> { + if (finalI == 2) { + throw new IllegalStateException(); + } + list3.add(finalI); + }, executor))); + } + try { + FutureUtil.waitForAll(futures3).join(); + } catch (Throwable ignore) { + + } + for (int i = 2; i < concurrentNum; i++) { + Assert.assertTrue(futures3.get(i).isCompletedExceptionally()); + } + + for (int i = 0; i < 2; i++) { + Assert.assertEquals(list3.get(i), (Integer) i); + } + } } \ No newline at end of file diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java index ca768d38490cc..f5e7e0528bd7b 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java @@ -124,7 +124,7 @@ private void handleSessionEvent(SessionEvent se) { if (se == SessionEvent.SessionReestablished) { log.info("Metadata store session has been re-established. Revalidating all the existing locks."); for (ResourceLockImpl lock : locks.values()) { - futures.add(lock.revalidate(lock.getValue(), true, true)); + futures.add(lock.silentRevalidateOnce()); } } else if (se == SessionEvent.Reconnected) { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java index 5271a73249d80..93c994b2436b9 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java @@ -44,7 +44,7 @@ public class ResourceLockImpl implements ResourceLock { private long version; private final CompletableFuture expiredFuture; private boolean revalidateAfterReconnection = false; - private CompletableFuture pendingOperationFuture; + private final FutureUtil.Sequencer sequencer; private enum State { Init, @@ -61,7 +61,7 @@ public ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, Str this.path = path; this.version = -1; this.expiredFuture = new CompletableFuture<>(); - this.pendingOperationFuture = CompletableFuture.completedFuture(null); + this.sequencer = FutureUtil.Sequencer.create(); this.state = State.Init; } @@ -74,11 +74,7 @@ public synchronized T getValue() { public synchronized CompletableFuture updateValue(T newValue) { // If there is an operation in progress, we're going to let it complete before attempting to // update the value - if (pendingOperationFuture.isDone()) { - pendingOperationFuture = CompletableFuture.completedFuture(null); - } - - pendingOperationFuture = pendingOperationFuture.thenCompose(v -> { + return sequencer.sequential(() -> { synchronized (ResourceLockImpl.this) { if (state != State.Valid) { return CompletableFuture.failedFuture( @@ -88,8 +84,6 @@ public synchronized CompletableFuture updateValue(T newValue) { return acquire(newValue); } }); - - return pendingOperationFuture; } @Override @@ -146,7 +140,7 @@ synchronized CompletableFuture acquire(T newValue) { .thenRun(() -> result.complete(null)) .exceptionally(ex -> { if (ex.getCause() instanceof LockBusyException) { - revalidate(newValue, false, false) + revalidate(newValue) .thenAccept(__ -> result.complete(null)) .exceptionally(ex1 -> { result.completeExceptionally(ex1); @@ -197,76 +191,54 @@ private CompletableFuture acquireWithNoRevalidation(T newValue) { } synchronized void lockWasInvalidated() { - if (state != State.Valid) { - // Ignore notifications while we're releasing the lock ourselves - return; - } - - log.info("Lock on resource {} was invalidated", path); - revalidate(value, true, true) - .thenRun(() -> log.info("Successfully revalidated the lock on {}", path)); + log.info("Lock on resource {} was invalidated. state {}", path, state); + silentRevalidateOnce(); } synchronized CompletableFuture revalidateIfNeededAfterReconnection() { if (revalidateAfterReconnection) { revalidateAfterReconnection = false; log.warn("Revalidate lock at {} after reconnection", path); - return revalidate(value, true, true); + return silentRevalidateOnce(); } else { return CompletableFuture.completedFuture(null); } } - synchronized CompletableFuture revalidate(T newValue, boolean trackPendingOperation, - boolean revalidateAfterReconnection) { - - final CompletableFuture trackFuture; - - if (!trackPendingOperation) { - trackFuture = doRevalidate(newValue); - } else if (pendingOperationFuture.isDone()) { - pendingOperationFuture = doRevalidate(newValue); - trackFuture = pendingOperationFuture; - } else { - if (log.isDebugEnabled()) { - log.debug("Previous revalidating is not finished while revalidate newValue={}, value={}, version={}", - newValue, value, version); - } - trackFuture = new CompletableFuture<>(); - trackFuture.whenComplete((unused, throwable) -> { - doRevalidate(newValue).thenRun(() -> trackFuture.complete(null)) - .exceptionally(throwable1 -> { - trackFuture.completeExceptionally(throwable1); - return null; - }); - }); - pendingOperationFuture = trackFuture; - } - - trackFuture.exceptionally(ex -> { - synchronized (ResourceLockImpl.this) { - Throwable realCause = FutureUtil.unwrapCompletionException(ex); - if (!revalidateAfterReconnection || realCause instanceof BadVersionException - || realCause instanceof LockBusyException) { - log.warn("Failed to revalidate the lock at {}. Marked as expired. {}", - path, realCause.getMessage()); - state = State.Released; - expiredFuture.complete(null); - } else { - // We failed to revalidate the lock due to connectivity issue - // Continue assuming we hold the lock, until we can revalidate it, either - // on Reconnected or SessionReestablished events. - ResourceLockImpl.this.revalidateAfterReconnection = true; - log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path, - realCause.getMessage()); - } - } - return null; - }); - return trackFuture; + /** + * Revalidate the distributed lock if it is not released. + * This method is thread-safe and it will perform multiple re-validation operations in turn. + */ + synchronized CompletableFuture silentRevalidateOnce() { + return sequencer.sequential(() -> revalidate(value)) + .thenRun(() -> log.info("Successfully revalidated the lock on {}", path)) + .exceptionally(ex -> { + synchronized (ResourceLockImpl.this) { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + if (realCause instanceof BadVersionException || realCause instanceof LockBusyException) { + log.warn("Failed to revalidate the lock at {}. Marked as expired. {}", + path, realCause.getMessage()); + state = State.Released; + expiredFuture.complete(null); + } else { + // We failed to revalidate the lock due to connectivity issue + // Continue assuming we hold the lock, until we can revalidate it, either + // on Reconnected or SessionReestablished events. + revalidateAfterReconnection = true; + log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path, + realCause.getMessage()); + } + } + return null; + }); } - private synchronized CompletableFuture doRevalidate(T newValue) { + private synchronized CompletableFuture revalidate(T newValue) { + // Since the distributed lock has been expired, we don't need to revalidate it. + if (state != State.Valid && state != State.Init) { + return CompletableFuture.failedFuture( + new IllegalStateException("Lock was not in valid state: " + state)); + } if (log.isDebugEnabled()) { log.debug("doRevalidate with newValue={}, version={}", newValue, version); } From 5d2d61e298e337b93fb178da191bc41d311107c2 Mon Sep 17 00:00:00 2001 From: ran Date: Mon, 20 Mar 2023 17:41:20 +0800 Subject: [PATCH 009/280] [feat][txn] Transaction buffer snapshot writer reuse (#19641) --- .../SystemTopicTxnBufferSnapshotService.java | 100 ++++++++++++++---- ...SingleSnapshotAbortedTxnProcessorImpl.java | 21 ++-- ...napshotSegmentAbortedTxnProcessorImpl.java | 45 ++++---- .../SegmentAbortedTxnProcessorTest.java | 19 +++- .../TopicTransactionBufferRecoverTest.java | 17 +-- .../broker/transaction/TransactionTest.java | 85 ++++++++++++++- .../transaction/TransactionTestBase.java | 20 ++++ .../buffer/TransactionBufferCloseTest.java | 79 ++++++++------ 8 files changed, 293 insertions(+), 93 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java index a1b78d89a13eb..332d754cf97d2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java @@ -21,63 +21,127 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.systopic.SystemTopicClientBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.util.FutureUtil; +@Slf4j public class SystemTopicTxnBufferSnapshotService { - protected final Map> clients; + protected final ConcurrentHashMap> clients; protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory; protected final Class schemaType; protected final EventType systemTopicType; + private final ConcurrentHashMap> refCountedWriterMap; + + // The class ReferenceCountedWriter will maintain the reference count, + // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed. + public static class ReferenceCountedWriter { + + private final AtomicLong referenceCount; + private final NamespaceName namespaceName; + private final CompletableFuture> future; + private final SystemTopicTxnBufferSnapshotService snapshotService; + + public ReferenceCountedWriter(NamespaceName namespaceName, + CompletableFuture> future, + SystemTopicTxnBufferSnapshotService snapshotService) { + this.referenceCount = new AtomicLong(1); + this.namespaceName = namespaceName; + this.snapshotService = snapshotService; + this.future = future; + this.future.exceptionally(t -> { + log.error("[{}] Failed to create TB snapshot writer.", namespaceName, t); + snapshotService.refCountedWriterMap.remove(namespaceName, this); + return null; + }); + } + + public CompletableFuture> getFuture() { + return future; + } + + private synchronized boolean retain() { + return this.referenceCount.incrementAndGet() > 0; + } + + public synchronized void release() { + if (this.referenceCount.decrementAndGet() == 0) { + snapshotService.refCountedWriterMap.remove(namespaceName, this); + future.thenAccept(writer -> { + final String topicName = writer.getSystemTopicClient().getTopicName().toString(); + writer.closeAsync().exceptionally(t -> { + if (t != null) { + log.error("[{}] Failed to close TB snapshot writer.", topicName, t); + } else { + if (log.isDebugEnabled()) { + log.debug("[{}] Success to close TB snapshot writer.", topicName); + } + } + return null; + }); + }); + } + } + + } + public SystemTopicTxnBufferSnapshotService(PulsarClient client, EventType systemTopicType, Class schemaType) { this.namespaceEventsSystemTopicFactory = new NamespaceEventsSystemTopicFactory(client); this.systemTopicType = systemTopicType; this.schemaType = schemaType; this.clients = new ConcurrentHashMap<>(); - } - - public CompletableFuture> createWriter(TopicName topicName) { - return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newWriterAsync); + this.refCountedWriterMap = new ConcurrentHashMap<>(); } public CompletableFuture> createReader(TopicName topicName) { - return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newReaderAsync); + return getTransactionBufferSystemTopicClient(topicName.getNamespaceObject()).newReaderAsync(); } public void removeClient(TopicName topicName, SystemTopicClientBase transactionBufferSystemTopicClient) { if (transactionBufferSystemTopicClient.getReaders().size() == 0 && transactionBufferSystemTopicClient.getWriters().size() == 0) { - clients.remove(topicName); + clients.remove(topicName.getNamespaceObject()); } } - protected CompletableFuture> getTransactionBufferSystemTopicClient(TopicName topicName) { + public ReferenceCountedWriter getReferenceWriter(NamespaceName namespaceName) { + return refCountedWriterMap.compute(namespaceName, (k, v) -> { + if (v != null && v.retain()) { + return v; + } else { + return new ReferenceCountedWriter<>(namespaceName, + getTransactionBufferSystemTopicClient(namespaceName).newWriterAsync(), this); + } + }); + } + + private SystemTopicClient getTransactionBufferSystemTopicClient(NamespaceName namespaceName) { TopicName systemTopicName = NamespaceEventsSystemTopicFactory - .getSystemTopicName(topicName.getNamespaceObject(), systemTopicType); + .getSystemTopicName(namespaceName, systemTopicType); if (systemTopicName == null) { - return FutureUtil.failedFuture( - new PulsarClientException - .InvalidTopicNameException("Can't create SystemTopicBaseTxnBufferSnapshotIndexService, " - + "because the topicName is null!")); + throw new RuntimeException(new PulsarClientException.InvalidTopicNameException( + "Can't get the TB system topic client for namespace " + namespaceName + + " with type " + systemTopicType + ".")); } - return CompletableFuture.completedFuture(clients.computeIfAbsent(systemTopicName, + + return clients.computeIfAbsent(namespaceName, (v) -> namespaceEventsSystemTopicFactory - .createTransactionBufferSystemTopicClient(systemTopicName, - this, schemaType))); + .createTransactionBufferSystemTopicClient(systemTopicName, this, schemaType)); } public void close() throws Exception { - for (Map.Entry> entry : clients.entrySet()) { + for (Map.Entry> entry : clients.entrySet()) { entry.getValue().close(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 87161e97512b9..5d582d564eadd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -28,6 +28,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; @@ -42,7 +43,7 @@ @Slf4j public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor { private final PersistentTopic topic; - private final CompletableFuture> takeSnapshotWriter; + private final ReferenceCountedWriter takeSnapshotWriter; /** * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this * position have been deleted. @@ -51,12 +52,14 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso private volatile long lastSnapshotTimestamps; + private volatile boolean isClosed = false; + public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); - this.takeSnapshotWriter.exceptionally((ex) -> { + .getTxnBufferSnapshotService().getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject()); + this.takeSnapshotWriter.getFuture().exceptionally((ex) -> { log.error("{} Failed to create snapshot writer", topic.getName()); topic.close(); return null; @@ -132,7 +135,7 @@ public CompletableFuture recoverFromSnapshot() { @Override public CompletableFuture clearAbortedTxnSnapshot() { - return this.takeSnapshotWriter.thenCompose(writer -> { + return this.takeSnapshotWriter.getFuture().thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); return writer.deleteAsync(snapshot.getTopicName(), snapshot); @@ -141,7 +144,7 @@ public CompletableFuture clearAbortedTxnSnapshot() { @Override public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { - return takeSnapshotWriter.thenCompose(writer -> { + return takeSnapshotWriter.getFuture().thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); @@ -175,8 +178,12 @@ public long getLastSnapshotTimestamps() { } @Override - public CompletableFuture closeAsync() { - return takeSnapshotWriter.thenCompose(SystemTopicClient.Writer::closeAsync); + public synchronized CompletableFuture closeAsync() { + if (!isClosed) { + isClosed = true; + takeSnapshotWriter.release(); + } + return CompletableFuture.completedFuture(null); } private void closeReader(SystemTopicClient.Reader reader) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 7a9e0e1abedd9..751c03aff95a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -43,6 +43,7 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; @@ -442,10 +443,10 @@ public class PersistentWorker { private final PersistentTopic topic; //Persistent snapshot segment and index at the single thread. - private final CompletableFuture> - snapshotSegmentsWriterFuture; - private final CompletableFuture> - snapshotIndexWriterFuture; + private final ReferenceCountedWriter snapshotSegmentsWriter; + private final ReferenceCountedWriter snapshotIndexWriter; + + private volatile boolean closed = false; private enum OperationState { None, @@ -470,18 +471,20 @@ public enum OperationType { public PersistentWorker(PersistentTopic topic) { this.topic = topic; - this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() + this.snapshotSegmentsWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); - this.snapshotSegmentsWriterFuture.exceptionally(ex -> { + .getTxnBufferSnapshotSegmentService() + .getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject()); + this.snapshotSegmentsWriter.getFuture().exceptionally(ex -> { log.error("{} Failed to create snapshot index writer", topic.getName()); topic.close(); return null; }); - this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() + this.snapshotIndexWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); - this.snapshotIndexWriterFuture.exceptionally((ex) -> { + .getTxnBufferSnapshotIndexService() + .getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject()); + this.snapshotIndexWriter.getFuture().exceptionally((ex) -> { log.error("{} Failed to create snapshot writer", topic.getName()); topic.close(); return null; @@ -631,7 +634,7 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm transactionBufferSnapshotSegment.setPersistentPositionLedgerId( abortedMarkerPersistentPosition.getLedgerId()); - return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { + return snapshotSegmentsWriter.getFuture().thenCompose(segmentWriter -> { transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); }).thenCompose((messageId) -> { @@ -668,7 +671,7 @@ private CompletableFuture deleteSnapshotSegment(List positio List> results = new ArrayList<>(); for (PositionImpl positionNeedToDelete : positionNeedToDeletes) { long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); - CompletableFuture res = snapshotSegmentsWriterFuture + CompletableFuture res = snapshotSegmentsWriter.getFuture() .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) .thenCompose(messageId -> { if (log.isDebugEnabled()) { @@ -695,7 +698,7 @@ private CompletableFuture deleteSnapshotSegment(List positio private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); - CompletableFuture res = snapshotIndexWriterFuture + CompletableFuture res = snapshotIndexWriter.getFuture() .thenCompose((indexesWriter) -> { snapshotIndexes.setIndexList(indexes.values().stream().toList()); snapshotIndexes.setSnapshot(snapshotSegment); @@ -712,7 +715,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd private CompletableFuture clearSnapshotSegmentAndIndexes() { CompletableFuture res = persistentWorker.clearAllSnapshotSegments() - .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose((ignore) -> snapshotIndexWriter.getFuture() .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) .thenRun(() -> log.debug("Successes to clear the snapshot segment and indexes for the topic [{}]", @@ -747,7 +750,7 @@ private CompletableFuture clearAllSnapshotSegments() { Message message = reader.readNextAsync() .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); if (topic.getName().equals(message.getValue().getTopicName())) { - snapshotSegmentsWriterFuture.get().write(message.getKey(), null); + snapshotSegmentsWriter.getFuture().get().write(message.getKey(), null); } } return CompletableFuture.completedFuture(null); @@ -760,11 +763,13 @@ private CompletableFuture clearAllSnapshotSegments() { }); } - - CompletableFuture closeAsync() { - return CompletableFuture.allOf( - this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync), - this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); + synchronized CompletableFuture closeAsync() { + if (!closed) { + closed = true; + snapshotSegmentsWriter.release(); + snapshotIndexWriter.release(); + } + return CompletableFuture.completedFuture(null); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index ffc059de8e656..c157d7cf8c527 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.transaction; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + import java.lang.reflect.Field; import java.util.LinkedList; import java.util.NavigableMap; @@ -34,6 +36,7 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; @@ -125,6 +128,7 @@ public void testPutAbortedTxnIntoProcessor() throws Exception { newProcessor.trimExpiredAbortedTxns(); //4. Verify the two sealed segment will be deleted. Awaitility.await().untilAsserted(() -> verifyAbortedTxnIDAndSegmentIndex(newProcessor, 11, 4)); + processor.closeAsync().get(5, TimeUnit.SECONDS); } private void waitTaskExecuteCompletely(AbortedTxnProcessor processor) throws Exception { @@ -177,8 +181,11 @@ public void testFuturesCanCompleteWhenItIsCanceled() throws Exception { new MutablePair<>(new CompletableFuture<>(), task))); try { processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); + fail("The update index operation should fail."); } catch (Exception e) { Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException); + } finally { + processor.closeAsync().get(5, TimeUnit.SECONDS); } } @@ -200,12 +207,13 @@ public void testClearSnapshotSegments() throws Exception { SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker worker = (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) field.get(processor); Field indexWriteFutureField = SnapshotSegmentAbortedTxnProcessorImpl - .PersistentWorker.class.getDeclaredField("snapshotIndexWriterFuture"); + .PersistentWorker.class.getDeclaredField("snapshotIndexWriter"); indexWriteFutureField.setAccessible(true); - CompletableFuture> snapshotIndexWriterFuture = - (CompletableFuture>) - indexWriteFutureField.get(worker); - snapshotIndexWriterFuture.get().close(); + ReferenceCountedWriter snapshotIndexWriter = + (ReferenceCountedWriter) indexWriteFutureField.get(worker); + snapshotIndexWriter.release(); + // After release, the writer should be closed, call close method again to make sure the writer was closed. + snapshotIndexWriter.getFuture().get().close(); //3. Try to write a snapshot segment that will fail to update indexes. for (int j = 0; j < SEGMENT_SIZE; j++) { TxnID txnID = new TxnID(0, j); @@ -233,6 +241,7 @@ public void testClearSnapshotSegments() throws Exception { //7. Verify the snapshot segments and index after clearing. verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 0); verifySnapshotSegmentsIndexSize(PROCESSOR_TOPIC, 1); + processor.closeAsync().get(5, TimeUnit.SECONDS); } private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index d4ddb26e014ca..2d6622571c033 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -57,6 +57,7 @@ import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService; +import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -602,11 +603,12 @@ public void testTransactionBufferRecoverThrowException() throws Exception { mock(SystemTopicTxnBufferSnapshotService.class); SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class); SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class); + ReferenceCountedWriter refCounterWriter = mock(ReferenceCountedWriter.class); + doReturn(CompletableFuture.completedFuture(writer)).when(refCounterWriter).getFuture(); doReturn(CompletableFuture.completedFuture(reader)) .when(systemTopicTxnBufferSnapshotService).createReader(any()); - doReturn(CompletableFuture.completedFuture(writer)) - .when(systemTopicTxnBufferSnapshotService).createWriter(any()); + doReturn(refCounterWriter).when(systemTopicTxnBufferSnapshotService).getReferenceWriter(any()); TransactionBufferSnapshotServiceFactory transactionBufferSnapshotServiceFactory = mock(TransactionBufferSnapshotServiceFactory.class); doReturn(systemTopicTxnBufferSnapshotService) @@ -645,8 +647,9 @@ public void testTransactionBufferRecoverThrowException() throws Exception { originalTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); // mock create writer fail - doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))) - .when(systemTopicTxnBufferSnapshotService).createWriter(any()); + ReferenceCountedWriter failedCountedWriter = mock(ReferenceCountedWriter.class); + doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))).when(failedCountedWriter).getFuture(); + doReturn(failedCountedWriter).when(systemTopicTxnBufferSnapshotService).getReferenceWriter(any()); checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceFactoryOriginal, transactionBufferSnapshotServiceFactory, originalTopic, field, producer); } @@ -703,7 +706,8 @@ public void testTransactionBufferIndexSystemTopic() throws Exception { new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotIndexService(); SystemTopicClient.Writer indexesWriter = - transactionBufferSnapshotIndexService.createWriter(TopicName.get(SNAPSHOT_INDEX)).get(); + transactionBufferSnapshotIndexService.getReferenceWriter( + TopicName.get(SNAPSHOT_INDEX).getNamespaceObject()).getFuture().get(); SystemTopicClient.Reader indexesReader = transactionBufferSnapshotIndexService.createReader(TopicName.get(SNAPSHOT_INDEX)).get(); @@ -764,7 +768,8 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotSegmentService(); SystemTopicClient.Writer - segmentWriter = transactionBufferSnapshotSegmentService.createWriter(snapshotSegmentTopicName).get(); + segmentWriter = transactionBufferSnapshotSegmentService + .getReferenceWriter(snapshotSegmentTopicName.getNamespaceObject()).getFuture().get(); // write two snapshot to snapshot segment topic TransactionBufferSnapshotSegment snapshot = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 20aeac0ed648f..c3533e70cf8be 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -86,6 +86,7 @@ import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService; +import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -1532,8 +1533,10 @@ public void testTBRecoverChangeStateError() throws InterruptedException, Timeout = mock(SystemTopicTxnBufferSnapshotService.class); SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class); when(writer.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); - when(systemTopicTxnBufferSnapshotService.createWriter(any())) - .thenReturn(CompletableFuture.completedFuture(writer)); + ReferenceCountedWriter refCounterWriter = mock(ReferenceCountedWriter.class); + doReturn(CompletableFuture.completedFuture(writer)).when(refCounterWriter).getFuture(); + when(systemTopicTxnBufferSnapshotService.getReferenceWriter(any())) + .thenReturn(refCounterWriter); TransactionBufferSnapshotServiceFactory transactionBufferSnapshotServiceFactory = mock(TransactionBufferSnapshotServiceFactory.class); when(transactionBufferSnapshotServiceFactory.getTxnBufferSnapshotService()) @@ -1676,4 +1679,82 @@ public void testDeleteNamespace() throws Exception { admin.namespaces().deleteNamespace(namespace, true); } + + @Test(timeOut = 10_000) + public void testTBSnapshotWriter() throws Exception { + String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5); + admin.namespaces().createNamespace(namespace, 16); + String topic = namespace + "/test-create-snapshot-writer-failed"; + int partitionCount = 20; + admin.topics().createPartitionedTopic(topic, partitionCount); + + Class clazz = SystemTopicTxnBufferSnapshotService.class; + Field field = clazz.getDeclaredField("refCountedWriterMap"); + field.setAccessible(true); + // inject a failed writer future + CompletableFuture> writerFuture = new CompletableFuture<>(); + for (PulsarService pulsarService : pulsarServiceList) { + SystemTopicTxnBufferSnapshotService bufferSnapshotService = + pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotService(); + ConcurrentHashMap writerMap1 = + ((ConcurrentHashMap) field.get(bufferSnapshotService)); + ReferenceCountedWriter failedCountedWriter = + new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, bufferSnapshotService); + writerMap1.put(NamespaceName.get(namespace), failedCountedWriter); + + SystemTopicTxnBufferSnapshotService segmentSnapshotService = + pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotSegmentService(); + ConcurrentHashMap writerMap2 = + ((ConcurrentHashMap) field.get(segmentSnapshotService)); + ReferenceCountedWriter failedCountedWriter2 = + new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, segmentSnapshotService); + writerMap2.put(NamespaceName.get(namespace), failedCountedWriter2); + + SystemTopicTxnBufferSnapshotService indexSnapshotService = + pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotIndexService(); + ConcurrentHashMap writerMap3 = + ((ConcurrentHashMap) field.get(indexSnapshotService)); + ReferenceCountedWriter failedCountedWriter3 = + new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, indexSnapshotService); + writerMap3.put(NamespaceName.get(namespace), failedCountedWriter3); + } + + CompletableFuture> producerFuture = pulsarClient.newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .createAsync(); + getTopic("persistent://" + topic + "-partition-0"); + Thread.sleep(3000); + // the producer shouldn't be created, because the transaction buffer snapshot writer future didn't finish. + assertFalse(producerFuture.isDone()); + + // The topic will be closed, because the transaction buffer snapshot writer future is failed, + // the failed writer future will be removed, the producer will be reconnected and work well. + writerFuture.completeExceptionally(new PulsarClientException.TopicTerminatedException("failed writer")); + Producer producer = producerFuture.get(); + + for (int i = 0; i < partitionCount * 2; i++) { + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(1, TimeUnit.MINUTES).build().get(); + producer.newMessage(txn).value("test".getBytes()).sendAsync(); + txn.commit().get(); + } + checkSnapshotPublisherCount(namespace, 1); + producer.close(); + admin.topics().unload(topic); + checkSnapshotPublisherCount(namespace, 0); + } + + private void getTopic(String topicName) { + Awaitility.await().atMost(5, TimeUnit.SECONDS).pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> { + for (PulsarService pulsarService : pulsarServiceList) { + if (pulsarService.getBrokerService().getTopicReference(topicName).isPresent()) { + return true; + } + } + return false; + }); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index fd49354342fa0..f45eda8d21fbe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -41,12 +41,17 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.PublisherStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.tests.TestRetrySupport; +import org.awaitility.Awaitility; +import org.testng.Assert; @Slf4j public abstract class TransactionTestBase extends TestRetrySupport { @@ -223,4 +228,19 @@ protected void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin ad throws Exception { MockedPulsarServiceBaseTest.deleteNamespaceWithRetry(ns, force, admin, pulsarServiceList); } + + public void checkSnapshotPublisherCount(String namespace, int expectCount) { + TopicName snTopicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(namespace), + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + List publisherStatsList = + (List) admin.topics() + .getStats(snTopicName.getPartitionedTopicName()).getPublishers(); + Assert.assertEquals(publisherStatsList.size(), expectCount); + }); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java index cd3a14da5967d..e92cf29521e34 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java @@ -19,6 +19,9 @@ package org.apache.pulsar.broker.transaction.buffer; import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pulsar.broker.transaction.TransactionTestBase; @@ -26,20 +29,13 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient; import org.apache.pulsar.client.impl.PulsarClientImpl; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.SystemTopicNames; -import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.PublisherStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; -import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.util.List; -import java.util.concurrent.TimeUnit; /** * Transaction buffer close test. @@ -71,49 +67,62 @@ public Object[][] isPartition() { @Test(timeOut = 10_000, dataProvider = "isPartition") public void deleteTopicCloseTransactionBufferTest(boolean isPartition) throws Exception { - int expectedCount = isPartition ? 30 : 1; - TopicName topicName = createAndLoadTopic(isPartition, expectedCount); - checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount); + int partitionCount = isPartition ? 30 : 1; + List topicNames = createAndLoadTopics(isPartition, partitionCount); + String namespaceName = topicNames.get(0).getNamespace(); + checkSnapshotPublisherCount(namespaceName, 1); + + for (int i = 0; i < topicNames.size(); i++) { + deleteTopic(isPartition, topicNames.get(i)); + // When delete all topics of the namespace, the publisher count should be 0. + int expectCount = i == topicNames.size() - 1 ? 0 : 1; + checkSnapshotPublisherCount(namespaceName, expectCount); + } + } + + private void deleteTopic(boolean isPartition, TopicName topicName) throws PulsarAdminException { if (isPartition) { admin.topics().deletePartitionedTopic(topicName.getPartitionedTopicName(), true); } else { admin.topics().delete(topicName.getPartitionedTopicName(), true); } - checkSnapshotPublisherCount(topicName.getNamespace(), 0); } @Test(timeOut = 10_000, dataProvider = "isPartition") public void unloadTopicCloseTransactionBufferTest(boolean isPartition) throws Exception { - int expectedCount = isPartition ? 30 : 1; - TopicName topicName = createAndLoadTopic(isPartition, expectedCount); - checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount); - admin.topics().unload(topicName.getPartitionedTopicName()); - checkSnapshotPublisherCount(topicName.getNamespace(), 0); + int partitionCount = isPartition ? 30 : 1; + List topicNames = createAndLoadTopics(isPartition, partitionCount); + String namespaceName = topicNames.get(0).getNamespace(); + checkSnapshotPublisherCount(namespaceName, 1); + + for (int i = 0; i < topicNames.size(); i++) { + admin.topics().unload(topicNames.get(i).getPartitionedTopicName()); + // When unload all topics of the namespace, the publisher count should be 0. + int expectCount = i == topicNames.size() - 1 ? 0 : 1; + checkSnapshotPublisherCount(namespaceName, expectCount); + } } - private TopicName createAndLoadTopic(boolean isPartition, int partitionCount) + private List createAndLoadTopics(boolean isPartition, int partitionCount) throws PulsarAdminException, PulsarClientException { String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5); admin.namespaces().createNamespace(namespace, 3); - String topic = namespace + "/tb-close-test-"; - if (isPartition) { - admin.topics().createPartitionedTopic(topic, partitionCount); - } - pulsarClient.newProducer() - .topic(topic) - .sendTimeout(0, TimeUnit.SECONDS) - .create() - .close(); - return TopicName.get(topic); - } + String topic = namespace + "/tb-close-test"; + List topics = new ArrayList<>(); - private void checkSnapshotPublisherCount(String namespace, int expectCount) throws PulsarAdminException { - TopicName snTopicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(namespace), - SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); - List publisherStatsList = - (List) admin.topics() - .getStats(snTopicName.getPartitionedTopicName()).getPublishers(); - Assert.assertEquals(publisherStatsList.size(), expectCount); + for (int i = 0; i < 2; i++) { + String t = topic + "-" + i; + if (isPartition) { + admin.topics().createPartitionedTopic(t, partitionCount); + } + pulsarClient.newProducer() + .topic(t) + .sendTimeout(0, TimeUnit.SECONDS) + .create() + .close(); + topics.add(TopicName.get(t)); + } + return topics; } } From c4abe7bd3c5fe59596decb0b6ce1badd40de0eeb Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 20 Mar 2023 23:09:49 +0800 Subject: [PATCH 010/280] [fix][sec] Upgrade dependency-check-maven and remove javax.el (#19764) --- pom.xml | 20 ++++++++++++++++++- ...owasp-dependency-check-false-positives.xml | 10 ++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc27ed54274fb..257018b0b0598 100644 --- a/pom.xml +++ b/pom.xml @@ -291,7 +291,7 @@ flexible messaging model and an intuitive client API. 0.1.4 1.3 0.4 - 8.0.1 + 8.1.2 0.9.15 1.6.1 6.4.0 @@ -878,6 +878,24 @@ flexible messaging model and an intuitive client API. ${caffeine.version} + + org.bouncycastle + bcpkix-jdk15on + ${bouncycastle.version} + + + + com.cronutils + cron-utils + ${cron-utils.version} + + + org.glassfish + javax.el + + + + com.yahoo.athenz athenz-zts-java-client-core diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 21a3679c0d8f7..345be8f4d2c06 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -182,6 +182,16 @@ CVE-2021-4277 + + It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-25194 is a false positive. + CVE-2023-25194 + + + + It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-34917 is a false positive. + CVE-2022-34917 + + yaml_project is not used at all. Any CVEs reported for yaml_project are false positives. cpe:/a:yaml_project:yaml From 164add6236eccb34f16683e86904f8d8d0b90c54 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 20 Mar 2023 13:55:02 -0500 Subject: [PATCH 011/280] [improve][proxy] Remove unnecessary executor callback; use assert (#19670) ### Motivation The `DirectProxyHandler` and the `ProxyConnection` are run in the same event loop to prevent context switching. As such, we do not need to schedule an event onto an event loop that is in fact the same event loop. Further, scheduling on that event loop could have resulted in uncaught failures because the method was run without any error handling. Additionally, we can use `assert` to verify that we are in the event loop. Netty makes extensive use of this paradigm, as described in this PR #19653. The primary benefit here is that we skip some unnecessary volatile reads when running the code in production. ### Modifications * Replace `checkState` with `assert` in `ProxyConnection` class * Remove unnecessary event execution in callback when starting up the `DirectProxyHandler`. ### Verifying this change This change is covered by the assertions that are added. ### Does this pull request potentially affect one of the following parts: This is a minor improvement that should not break anything. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/33 --- .../apache/pulsar/proxy/server/ProxyConnection.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 5a53f6ec014a2..1e19d760c6d7b 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -19,7 +19,6 @@ package org.apache.pulsar.proxy.server; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; import io.netty.channel.ChannelFutureListener; @@ -380,7 +379,7 @@ private synchronized void completeConnect() throws PulsarClientException { } private void handleBrokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) { - checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop"); + assert ctx.executor().inEventLoop(); if (state == State.ProxyConnectingToBroker && ctx.channel().isOpen() && this.directProxyHandler == null) { this.directProxyHandler = directProxyHandler; state = State.ProxyConnectionToBroker; @@ -401,7 +400,7 @@ private void handleBrokerConnected(DirectProxyHandler directProxyHandler, Comman } private void connectToBroker(InetSocketAddress brokerAddress) { - checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop"); + assert ctx.executor().inEventLoop(); DirectProxyHandler directProxyHandler = new DirectProxyHandler(service, this); directProxyHandler.connect(proxyToBrokerUrl, brokerAddress, protocolVersionToAdvertise); } @@ -409,10 +408,13 @@ private void connectToBroker(InetSocketAddress brokerAddress) { public void brokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) { try { final CommandConnected finalConnected = new CommandConnected().copyFrom(connected); - ctx.executor().execute(() -> handleBrokerConnected(directProxyHandler, finalConnected)); + handleBrokerConnected(directProxyHandler, finalConnected); } catch (RejectedExecutionException e) { LOG.error("Event loop was already closed. Closing broker connection.", e); directProxyHandler.close(); + } catch (AssertionError e) { + LOG.error("Failed assertion, closing direct proxy handler.", e); + directProxyHandler.close(); } } From c6de57c4b4510c5064c6e192df8fa9d51cede01d Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 20 Mar 2023 14:07:04 -0500 Subject: [PATCH 012/280] [improve][broker] Test AuthorizationService to cover proxyRoles behavior (#19845) Relates to: #19455 #19830 ### Motivation When I added the requirement for the proxy to use a role in the `proxyRoles` set, I didn't add a test that checked the negative case. This new test was first added in #19830 with one small difference. The goal of this test is to ensure that authorization of the client role and the original role is handled correctly. ### Modifications * Add new test class named `AuthorizationServiceTest`. We use `pass.proxy` and `fail.proxy` as proxy roles to simulate cases where the proxy's role passes and fails authorization, which is always possible. * Add new mock authorization provider named `MockAuthorizationProvider`. The logic is to let any role that starts with `pass` be considered authorized. ### Verifying this change This is a new test. It simply verifies the existing behavior to prevent future regressions. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping forked test since the new tests pass locally and there are no other modifications. --- .../AuthorizationServiceTest.java | 135 ++++++++++++++++ .../MockAuthorizationProvider.java | 144 ++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java new file mode 100644 index 0000000000000..6f9dffa11b948 --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authorization; + +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertTrue; +import java.util.HashSet; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.PolicyName; +import org.apache.pulsar.common.policies.data.PolicyOperation; +import org.apache.pulsar.common.policies.data.TenantOperation; +import org.apache.pulsar.common.policies.data.TopicOperation; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class AuthorizationServiceTest { + + AuthorizationService authorizationService; + + @BeforeClass + void beforeClass() throws PulsarServerException { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthorizationEnabled(true); + // Consider both of these proxy roles to make testing more comprehensive + HashSet proxyRoles = new HashSet<>(); + proxyRoles.add("pass.proxy"); + proxyRoles.add("fail.proxy"); + conf.setProxyRoles(proxyRoles); + conf.setAuthorizationProvider(MockAuthorizationProvider.class.getName()); + authorizationService = new AuthorizationService(conf, null); + } + + /** + * See {@link MockAuthorizationProvider} for the implementation of the mock authorization provider. + */ + @DataProvider(name = "roles") + public Object[][] encryptionProvider() { + return new Object[][]{ + // Schema: role, originalRole, whether authorization should pass + + // Client conditions where original role isn't passed or is blank + {"pass.client", null, Boolean.TRUE}, + {"pass.client", " ", Boolean.TRUE}, + {"fail.client", null, Boolean.FALSE}, + {"fail.client", " ", Boolean.FALSE}, + + // Proxy conditions where original role isn't passed or is blank + {"pass.proxy", null, Boolean.FALSE}, + {"pass.proxy", " ", Boolean.FALSE}, + {"fail.proxy", null, Boolean.FALSE}, + {"fail.proxy", " ", Boolean.FALSE}, + + // Normal proxy and client conditions + {"pass.proxy", "pass.client", Boolean.TRUE}, + {"pass.proxy", "fail.client", Boolean.FALSE}, + {"fail.proxy", "pass.client", Boolean.FALSE}, + {"fail.proxy", "fail.client", Boolean.FALSE}, + + // Not proxy with original principal + {"pass.not-proxy", "pass.client", Boolean.FALSE}, // non proxy role can't pass original role + {"pass.not-proxy", "fail.client", Boolean.FALSE}, + {"fail.not-proxy", "pass.client", Boolean.FALSE}, + {"fail.not-proxy", "fail.client", Boolean.FALSE}, + + // Covers an unlikely scenario, but valid in the context of this test + {null, "pass.proxy", Boolean.FALSE}, + }; + } + + private void checkResult(boolean expected, boolean actual) { + if (expected) { + assertTrue(actual); + } else { + assertFalse(actual); + } + } + + @Test(dataProvider = "roles") + public void testAllowTenantOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception { + boolean isAuthorized = authorizationService.allowTenantOperationAsync("tenant", + TenantOperation.DELETE_NAMESPACE, originalRole, role, null).get(); + checkResult(shouldPass, isAuthorized); + } + + @Test(dataProvider = "roles") + public void testNamespaceOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception { + boolean isAuthorized = authorizationService.allowNamespaceOperationAsync(NamespaceName.get("public/default"), + NamespaceOperation.PACKAGES, originalRole, role, null).get(); + checkResult(shouldPass, isAuthorized); + } + + @Test(dataProvider = "roles") + public void testTopicOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception { + boolean isAuthorized = authorizationService.allowTopicOperationAsync(TopicName.get("topic"), + TopicOperation.PRODUCE, originalRole, role, null).get(); + checkResult(shouldPass, isAuthorized); + } + + @Test(dataProvider = "roles") + public void testNamespacePolicyOperationAsync(String role, String originalRole, boolean shouldPass) + throws Exception { + boolean isAuthorized = authorizationService.allowNamespacePolicyOperationAsync( + NamespaceName.get("public/default"), PolicyName.ALL, PolicyOperation.READ, originalRole, role, null) + .get(); + checkResult(shouldPass, isAuthorized); + } + + @Test(dataProvider = "roles") + public void testTopicPolicyOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception { + boolean isAuthorized = authorizationService.allowTopicPolicyOperationAsync(TopicName.get("topic"), + PolicyName.ALL, PolicyOperation.READ, originalRole, role, null).get(); + checkResult(shouldPass, isAuthorized); + } +} diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java new file mode 100644 index 0000000000000..4c939cbd9723a --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authorization; + +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.PolicyName; +import org.apache.pulsar.common.policies.data.PolicyOperation; +import org.apache.pulsar.common.policies.data.TenantOperation; +import org.apache.pulsar.common.policies.data.TopicOperation; + +/** + * Mock implementation of the authorization provider interface used for testing. + * A role is authorized if it starts with "pass". + */ +public class MockAuthorizationProvider implements AuthorizationProvider { + + private CompletableFuture shouldPass(String role) { + return CompletableFuture.completedFuture(role != null && role.startsWith("pass")); + } + + @Override + public CompletableFuture canProduceAsync(TopicName topicName, String role, + AuthenticationDataSource authenticationData) { + return shouldPass(role); + } + + @Override + public CompletableFuture canConsumeAsync(TopicName topicName, String role, + AuthenticationDataSource authenticationData, String subscription) { + return shouldPass(role); + } + + @Override + public CompletableFuture canLookupAsync(TopicName topicName, String role, + AuthenticationDataSource authenticationData) { + return shouldPass(role); + } + + @Override + public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, + AuthenticationDataSource authenticationData) { + return shouldPass(role); + } + + @Override + public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, String role, + AuthenticationDataSource authenticationData) { + return shouldPass(role); + } + + @Override + public CompletableFuture allowTenantOperationAsync(String tenantName, String role, + TenantOperation operation, + AuthenticationDataSource authData) { + return shouldPass(role); + } + + @Override + public CompletableFuture allowNamespaceOperationAsync(NamespaceName namespaceName, String role, + NamespaceOperation operation, + AuthenticationDataSource authData) { + return shouldPass(role); + } + + @Override + public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceName namespaceName, PolicyName policy, + PolicyOperation operation, String role, + AuthenticationDataSource authData) { + return shouldPass(role); + } + + @Override + public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, TopicOperation operation, + AuthenticationDataSource authData) { + return shouldPass(role); + } + + @Override + public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic, String role, PolicyName policy, + PolicyOperation operation, + AuthenticationDataSource authData) { + return shouldPass(role); + } + + + @Override + public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role, + AuthenticationDataSource authenticationData) { + return null; + } + + + @Override + public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, + String authDataJson) { + return null; + } + + @Override + public CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName, + Set roles, String authDataJson) { + return null; + } + + @Override + public CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName, + String role, String authDataJson) { + return null; + } + + @Override + public CompletableFuture grantPermissionAsync(TopicName topicName, Set actions, String role, + String authDataJson) { + return null; + } + + @Override + public void close() throws IOException { + + } +} From ac574d58052fd6aef56175f06fe17017b40e9dc6 Mon Sep 17 00:00:00 2001 From: laminar Date: Tue, 21 Mar 2023 07:04:59 +0800 Subject: [PATCH 013/280] [improve][fn] Support processingGuarantees "EFFECTIVELY_ONCE" in python function (#18929) Signed-off-by: laminar --- .../instance/src/main/python/contextimpl.py | 13 ++++++ .../src/main/python/python_instance.py | 43 ++++++++++++++++--- .../src/main/python/python_instance_main.py | 13 +++--- .../src/test/python/test_python_instance.py | 12 +++--- 4 files changed, 64 insertions(+), 17 deletions(-) diff --git a/pulsar-functions/instance/src/main/python/contextimpl.py b/pulsar-functions/instance/src/main/python/contextimpl.py index 14247e2d647a5..bfe7b23927ba7 100755 --- a/pulsar-functions/instance/src/main/python/contextimpl.py +++ b/pulsar-functions/instance/src/main/python/contextimpl.py @@ -89,6 +89,19 @@ def get_message_properties(self): def get_current_message_topic_name(self): return self.message.topic_name() + def get_message_sequence_id(self): + if not self.get_message_id(): + return None + ledger_id = self.get_message_id().ledger_id() + entry_id = self.get_message_id().entry_id() + offset = (ledger_id << 28) | entry_id + return offset + + def get_message_partition_index(self): + if not self.get_message_id(): + return None + return self.get_message_id().partition() + def get_partition_key(self): return self.message.partition_key() diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index c679288789536..57edbf954b376 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -106,6 +106,7 @@ def __init__(self, self.execution_thread = None self.atmost_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATMOST_ONCE') self.atleast_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATLEAST_ONCE') + self.effectively_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('EFFECTIVELY_ONCE') self.manual = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('MANUAL') self.auto_ack = self.instance_config.function_details.autoAck self.contextimpl = None @@ -143,11 +144,15 @@ def run(self): if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value("FAILOVER"): mode = pulsar._pulsar.ConsumerType.Failover + if self.instance_config.function_details.retainOrdering or \ + self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value("EFFECTIVELY_ONCE"): + mode = pulsar._pulsar.ConsumerType.Failover + position = pulsar._pulsar.InitialPosition.Latest if self.instance_config.function_details.source.subscriptionPosition == Function_pb2.SubscriptionPosition.Value("EARLIEST"): position = pulsar._pulsar.InitialPosition.Earliest - subscription_name = self.instance_config.function_details.source.subscriptionName + subscription_name = self.instance_config.function_details.source.subscriptionName if not (subscription_name and subscription_name.strip()): subscription_name = str(self.instance_config.function_details.tenant) + "/" + \ @@ -293,7 +298,7 @@ def actual_execution(self): def done_producing(self, consumer, orig_message, topic, result, sent_message): if result == pulsar.Result.Ok: - if self.auto_ack and self.atleast_once: + if self.auto_ack and self.atleast_once or self.effectively_once: consumer.acknowledge(orig_message) else: error_msg = "Failed to publish to topic [%s] with error [%s] with src message id [%s]" % (topic, result, orig_message.message_id()) @@ -302,14 +307,27 @@ def done_producing(self, consumer, orig_message, topic, result, sent_message): # If producer fails send output then send neg ack for input message back to broker consumer.negative_acknowledge(orig_message) - def process_result(self, output, msg): - if output is not None and self.instance_config.function_details.sink.topic != None and \ + if output is not None and self.instance_config.function_details.sink.topic is not None and \ len(self.instance_config.function_details.sink.topic) > 0: if self.output_serde is None: self.setup_output_serde() + if self.effectively_once: + if self.contextimpl.get_message_partition_index() is None or \ + self.contextimpl.get_message_partition_index() >= 0: + Log.error("Partitioned topic is not available in effectively_once mode.") + raise Exception("Partitioned topic is not available in effectively_once mode.") + + producer_id = self.instance_config.function_details.sink.topic + producer = self.contextimpl.publish_producers.get(producer_id) + if producer is None: + self.setup_producer(producer_name=producer_id) + self.contextimpl.publish_producers[producer_id] = self.producer + Log.info("Setup producer [%s] successfully in effectively_once mode." % self.producer.producer_name()) + if self.producer is None: self.setup_producer() + Log.info("Setup producer successfully.") # only serialize function output when output schema is not set output_object = output @@ -318,9 +336,21 @@ def process_result(self, output, msg): if output_object is not None: props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())} - self.producer.send_async(output_object, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props) + if self.effectively_once: + self.producer.send_async(output_object, + partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), + properties=props, + sequence_id=self.contextimpl.get_message_sequence_id()) + Log.debug("Send message with sequence ID [%s] using the producer [%s] in effectively_once mode." % + (self.contextimpl.get_message_sequence_id(), self.producer.producer_name())) + else: + self.producer.send_async(output_object, + partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), + properties=props) elif self.auto_ack and self.atleast_once: msg.consumer.acknowledge(msg.message) + elif self.effectively_once: + msg.consumer.acknowledge_cumulative(msg.message) def setup_output_serde(self): if self.instance_config.function_details.sink.serDeClassName != None and \ @@ -332,7 +362,7 @@ def setup_output_serde(self): serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER) self.output_serde = serde_kclass() - def setup_producer(self): + def setup_producer(self, producer_name=None): if self.instance_config.function_details.sink.topic != None and \ len(self.instance_config.function_details.sink.topic) > 0: Log.debug("Setting up producer for topic %s" % self.instance_config.function_details.sink.topic) @@ -366,6 +396,7 @@ def setup_producer(self): self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), schema=self.output_schema, + producer_name=producer_name, block_if_queue_full=True, batching_enabled=True, batching_type=batch_type, diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py index 2d6520b2e99e9..9a923c7e3a18b 100755 --- a/pulsar-functions/instance/src/main/python/python_instance_main.py +++ b/pulsar-functions/instance/src/main/python/python_instance_main.py @@ -164,11 +164,7 @@ def main(): args.function_details = args.function_details[:-1] json_format.Parse(args.function_details, function_details) - if function_details.processingGuarantees == "EFFECTIVELY_ONCE": - print("Python instance current not support EFFECTIVELY_ONCE processing guarantees.") - sys.exit(1) - - if function_details.autoAck == False and function_details.processingGuarantees == "ATMOST_ONCE" \ + if function_details.autoAck is False and function_details.processingGuarantees == "ATMOST_ONCE" \ or function_details.processingGuarantees == "ATLEAST_ONCE": print("When Guarantees == " + function_details.processingGuarantees + ", autoAck must be equal to true, " "This is a contradictory configuration, autoAck will be removed later," @@ -176,6 +172,13 @@ def main(): "This is a contradictory configuration, autoAck will be removed later," "Please refer to PIP: https://github.com/apache/pulsar/issues/15560") sys.exit(1) + if function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('EFFECTIVELY_ONCE'): + if len(function_details.source.inputSpecs.keys()) != 1 or function_details.sink.topic == "": + print("When Guarantees == EFFECTIVELY_ONCE you need to ensure that the following pre-requisites have been met:" + "1. deduplication is enabled" + "2. set ProcessingGuarantees to EFFECTIVELY_ONCE" + "3. the function has only one source topic and one sink topic (both are non-partitioned)") + sys.exit(1) if os.path.splitext(str(args.py))[1] == '.whl': if args.install_usercode_dependencies: cmd = "pip install -t %s" % os.path.dirname(os.path.abspath(str(args.py))) diff --git a/pulsar-functions/instance/src/test/python/test_python_instance.py b/pulsar-functions/instance/src/test/python/test_python_instance.py index bbd6ca1fe42f1..975ddfd90288d 100644 --- a/pulsar-functions/instance/src/test/python/test_python_instance.py +++ b/pulsar-functions/instance/src/test/python/test_python_instance.py @@ -25,7 +25,7 @@ sys.modules['prometheus_client'] = Mock() from contextimpl import ContextImpl -from python_instance import InstanceConfig +from python_instance import PythonInstance, InstanceConfig from pulsar import Message import Function_pb2 @@ -49,14 +49,14 @@ def test_context_publish(self): function_id = 'test_function_id' function_version = 'test_function_version' function_details = Function_pb2.FunctionDetails() - max_buffered_tuples = 100; + max_buffered_tuples = 100 instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples) logger = log.Log pulsar_client = Mock() producer = Mock() producer.send_async = Mock(return_value=None) pulsar_client.create_producer = Mock(return_value=producer) - user_code=__file__ + user_code = __file__ consumers = None context_impl = ContextImpl(instance_config, logger, pulsar_client, user_code, consumers, None, None, None, None) @@ -77,11 +77,11 @@ def test_context_ack_partitionedtopic(self): function_id = 'test_function_id' function_version = 'test_function_version' function_details = Function_pb2.FunctionDetails() - max_buffered_tuples = 100; + max_buffered_tuples = 100 instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples) logger = log.Log pulsar_client = Mock() - user_code=__file__ + user_code = __file__ consumer = Mock() consumer.acknowledge = Mock(return_value=None) consumers = {"mytopic" : consumer} @@ -89,4 +89,4 @@ def test_context_ack_partitionedtopic(self): context_impl.ack("test_message_id", "mytopic-partition-3") args, kwargs = consumer.acknowledge.call_args - self.assertEqual(args[0], "test_message_id") \ No newline at end of file + self.assertEqual(args[0], "test_message_id") From 03f8b806ec177cdf6a5a82193a3fc82557000989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 21 Mar 2023 03:14:33 +0100 Subject: [PATCH 014/280] [fix][io] KCA sink: handle null values with KeyValue schema (#19861) Co-authored-by: Andrey Yegorov --- .../io/kafka/connect/KafkaConnectSink.java | 4 +- .../connect/schema/KafkaConnectData.java | 9 +++ .../schema/PulsarSchemaToKafkaSchema.java | 80 ++++++++++++++++++- .../kafka/connect/KafkaConnectSinkTest.java | 54 +++++++++++++ 4 files changed, 144 insertions(+), 3 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java index efbad2ef47ae0..06f66f60380d9 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java @@ -437,8 +437,8 @@ protected SinkRecord toSinkRecord(Record sourceRecord) { if (nativeObject instanceof KeyValue) { KeyValue kv = (KeyValue) nativeObject; - key = KafkaConnectData.getKafkaConnectData(kv.getKey(), keySchema); - value = KafkaConnectData.getKafkaConnectData(kv.getValue(), valueSchema); + key = KafkaConnectData.getKafkaConnectDataFromSchema(kv.getKey(), keySchema); + value = KafkaConnectData.getKafkaConnectDataFromSchema(kv.getValue(), valueSchema); } else if (nativeObject != null) { throw new IllegalStateException("Cannot extract KeyValue data from " + nativeObject.getClass()); } else { diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java index 757241d411034..a308ef01ddcf1 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java @@ -54,6 +54,14 @@ private static List arrayToList(Object nativeObject, Schema kafkaValueSc return out; } + + public static Object getKafkaConnectDataFromSchema(Object nativeObject, Schema kafkaSchema) { + if (kafkaSchema != null && nativeObject == null) { + return null; + } + return getKafkaConnectData(nativeObject, kafkaSchema); + } + @SuppressWarnings("unchecked") public static Object getKafkaConnectData(Object nativeObject, Schema kafkaSchema) { if (kafkaSchema == null) { @@ -380,6 +388,7 @@ private static Object defaultOrThrow(Schema kafkaSchema) { if (kafkaSchema.isOptional()) { return null; } + throw new DataException("Invalid null value for required " + kafkaSchema.type() + " field"); } } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java index 2eb6573374cdb..faf28585e8aed 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java @@ -26,11 +26,14 @@ import com.google.common.util.concurrent.UncheckedExecutionException; import io.confluent.connect.avro.AvroData; import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Time; @@ -41,6 +44,76 @@ @Slf4j public class PulsarSchemaToKafkaSchema { + + private static class OptionalForcingSchema implements Schema { + + Schema sourceSchema; + + public OptionalForcingSchema(Schema sourceSchema) { + this.sourceSchema = sourceSchema; + } + + @Override + public Type type() { + return sourceSchema.type(); + } + + @Override + public boolean isOptional() { + return true; + } + + @Override + public Object defaultValue() { + return sourceSchema.defaultValue(); + } + + @Override + public String name() { + return sourceSchema.name(); + } + + @Override + public Integer version() { + return sourceSchema.version(); + } + + @Override + public String doc() { + return sourceSchema.doc(); + } + + @Override + public Map parameters() { + return sourceSchema.parameters(); + } + + @Override + public Schema keySchema() { + return sourceSchema.keySchema(); + } + + @Override + public Schema valueSchema() { + return sourceSchema.valueSchema(); + } + + @Override + public List fields() { + return sourceSchema.fields(); + } + + @Override + public Field field(String s) { + return sourceSchema.field(s); + } + + @Override + public Schema schema() { + return sourceSchema.schema(); + } + } + private static final ImmutableMap pulsarSchemaTypeToKafkaSchema; private static final ImmutableSet kafkaLogicalSchemas; private static final AvroData avroData = new AvroData(1000); @@ -80,6 +153,11 @@ private static org.apache.avro.Schema parseAvroSchema(String schemaJson) { return parser.parse(schemaJson); } + public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) { + Schema s = getKafkaConnectSchema(pulsarSchema); + return new OptionalForcingSchema(s); + } + public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) { if (pulsarSchema == null || pulsarSchema.getSchemaInfo() == null) { throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Schema is required.", null); @@ -122,7 +200,7 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()), - getKafkaConnectSchema(kvSchema.getValueSchema())) + getOptionalKafkaConnectSchema(kvSchema.getValueSchema())) .build(); } org.apache.avro.Schema avroSchema = parseAvroSchema( diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index 567562d338b98..e9d454ed2fd5a 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -51,6 +51,9 @@ import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.GenericSchema; +import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; +import org.apache.pulsar.client.api.schema.SchemaBuilder; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -60,6 +63,7 @@ import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.client.util.MessageIdUtils; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; @@ -734,6 +738,56 @@ public void schemaKeyValueSchemaTest() throws Exception { Assert.assertEquals(key, 11); } + @Test + public void schemaKeyValueSchemaNullValueTest() throws Exception { + RecordSchemaBuilder builder = SchemaBuilder + .record("test"); + builder.field("test").type(SchemaType.STRING); + GenericSchema schema = GenericAvroSchema.of(builder.build(SchemaType.AVRO)); + KeyValue kv = new KeyValue<>(11, null); + SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, schema), 11, + "INT32", null, "STRUCT"); + Assert.assertNull(sinkRecord.value()); + int key = (int) sinkRecord.key(); + Assert.assertEquals(key, 11); + } + + @Test + public void schemaKeyValueSchemaNullValueNoUnwrapTest() throws Exception { + props.put("unwrapKeyValueIfAvailable", "false"); + JSONSchema jsonSchema = JSONSchema + .of(SchemaDefinition.builder() + .withPojo(PulsarSchemaToKafkaSchemaTest.StructWithAnnotations.class) + .withAlwaysAllowNull(true) + .build()); + KeyValue kv = new KeyValue<>(11, null); + Map expected = new HashMap(); + expected.put("11", null); + SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, jsonSchema), "key", + "STRING", expected, "MAP"); + Assert.assertNull(((Map)sinkRecord.value()).get(11)); + String key =(String)sinkRecord.key(); + Assert.assertEquals(key, "key"); + } + + @Test + public void schemaKeyValueSchemaNullValueNoUnwrapTestAvro() throws Exception { + props.put("unwrapKeyValueIfAvailable", "false"); + RecordSchemaBuilder builder = SchemaBuilder + .record("test"); + builder.property("op", "test"); + builder.field("test").type(SchemaType.STRING); + GenericSchema schema = GenericAvroSchema.of(builder.build(SchemaType.AVRO)); + KeyValue kv = new KeyValue<>(11, null); + Map expected = new HashMap(); + expected.put("11", null); + SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, schema), "key", + "STRING", expected, "MAP"); + Assert.assertNull(((Map)sinkRecord.value()).get(11)); + String key =(String)sinkRecord.key(); + Assert.assertEquals(key, "key"); + } + @Test public void kafkaLogicalTypesTimestampTest() { Schema schema = new TestSchema(SchemaInfoImpl.builder() From 1e44ba179bfc189d22dff046a1887e12842b0851 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Tue, 21 Mar 2023 16:17:41 +0800 Subject: [PATCH 015/280] [fix][doc] The `messageIds` in `onNegativeAcksSend` shouldn't be null (#19852) ### Motivation The `onNegativeAcksSend` is only called by: https://github.com/apache/pulsar/blob/80c5791b87482bee3392308ecef45f455f8de885/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java#L83 It will not be null. The statement `null if acknowledge fail.` is incorrect. ### Modifications * Remove the incorrect statement and refine the doc. Signed-off-by: Zike Yang Co-authored-by: Jun Ma <60642177+momo-jun@users.noreply.github.com> --- .../java/org/apache/pulsar/client/api/ConsumerInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java index d6d91cd88500b..be2f9b0f10826 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java @@ -107,7 +107,7 @@ public interface ConsumerInterceptor extends AutoCloseable { *

Any exception thrown by this method will be ignored by the caller. * * @param consumer the consumer which contains the interceptor - * @param messageIds message to ack, null if acknowledge fail. + * @param messageIds the set of message ids to negatively ack */ void onNegativeAcksSend(Consumer consumer, Set messageIds); From 905e8ef730c96cbb43ad5c892f1dbfc3cd1521cc Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 21 Mar 2023 21:31:22 +0800 Subject: [PATCH 016/280] [improve][meta] Make session notification to be async. (#19869) --- .../apache/pulsar/common/util/FutureUtil.java | 26 +++++++++++++++++++ .../coordination/impl/LeaderElectionImpl.java | 23 ++++++++-------- .../coordination/impl/LockManagerImpl.java | 22 +++++++--------- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java index 531769a1e452b..2b082b4a7899b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java @@ -28,6 +28,7 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -36,6 +37,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; /** @@ -238,6 +240,30 @@ public static CompletableFuture addTimeoutHandling(CompletableFuture f return future; } + /** + * @throws RejectedExecutionException if this task cannot be accepted for execution + * @throws NullPointerException if one of params is null + */ + public static @Nonnull CompletableFuture composeAsync(Supplier> futureSupplier, + Executor executor) { + Objects.requireNonNull(futureSupplier); + Objects.requireNonNull(executor); + final CompletableFuture future = new CompletableFuture<>(); + try { + executor.execute(() -> futureSupplier.get().whenComplete((result, error) -> { + if (error != null) { + future.completeExceptionally(error); + return; + } + future.complete(result); + })); + } catch (RejectedExecutionException ex) { + future.completeExceptionally(ex); + } + return future; + } + + /** * Creates a low-overhead timeout exception which is performance optimized to minimize allocations * and cpu consumption. It sets the stacktrace of the exception to the given source class and diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index c1121b1309c2c..11ae62226e7cd 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -24,7 +24,6 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -32,6 +31,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.common.util.SafeRunnable; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataCacheConfig; @@ -62,6 +62,7 @@ class LeaderElectionImpl implements LeaderElection { private Optional proposedValue; private final ScheduledExecutorService executor; + private final FutureUtil.Sequencer sequencer; private enum InternalState { Init, ElectionInProgress, LeaderIsPresent, Closed @@ -85,7 +86,7 @@ private enum InternalState { this.internalState = InternalState.Init; this.stateChangesListener = stateChangesListener; this.executor = executor; - + this.sequencer = FutureUtil.Sequencer.create(); store.registerListener(this::handlePathNotification); store.registerSessionListener(this::handleSessionNotification); updateCachedValueFuture = executor.scheduleWithFixedDelay(SafeRunnable.safeRun(this::getLeaderValue), @@ -277,18 +278,18 @@ public Optional getLeaderValueIfPresent() { private void handleSessionNotification(SessionEvent event) { // Ensure we're only processing one session event at a time. - executor.execute(SafeRunnable.safeRun(() -> { + sequencer.sequential(() -> FutureUtil.composeAsync(() -> { if (event == SessionEvent.SessionReestablished) { log.info("Revalidating leadership for {}", path); - - try { - LeaderElectionState les = elect().get(); - log.info("Resynced leadership for {} - State: {}", path, les); - } catch (ExecutionException | InterruptedException e) { - log.warn("Failure when processing session event", e); - } + return elect().thenAccept(leaderState -> { + log.info("Resynced leadership for {} - State: {}", path, leaderState); + }).exceptionally(ex -> { + log.warn("Failure when processing session event", ex); + return null; + }); } - })); + return CompletableFuture.completedFuture(null); + }, executor)); } private void handlePathNotification(Notification notification) { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java index f5e7e0528bd7b..4da6b7998a0c4 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java @@ -27,11 +27,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.common.util.SafeRunnable; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataSerde; @@ -53,6 +51,7 @@ class LockManagerImpl implements LockManager { private final MetadataStoreExtended store; private final MetadataCache cache; private final MetadataSerde serde; + private final FutureUtil.Sequencer sequencer; private final ExecutorService executor; private enum State { @@ -72,6 +71,7 @@ private enum State { this.cache = store.getMetadataCache(serde); this.serde = serde; this.executor = executor; + this.sequencer = FutureUtil.Sequencer.create(); store.registerSessionListener(this::handleSessionEvent); store.registerListener(this::handleDataNotification); } @@ -118,9 +118,8 @@ public CompletableFuture> acquireLock(String path, T value) { private void handleSessionEvent(SessionEvent se) { // We want to make sure we're processing one event at a time and that we're done with one event before going // for the next one. - executor.execute(SafeRunnable.safeRun(() -> { - List> futures = new ArrayList<>(); - + sequencer.sequential(() -> FutureUtil.composeAsync(() -> { + final List> futures = new ArrayList<>(); if (se == SessionEvent.SessionReestablished) { log.info("Metadata store session has been re-established. Revalidating all the existing locks."); for (ResourceLockImpl lock : locks.values()) { @@ -133,13 +132,12 @@ private void handleSessionEvent(SessionEvent se) { futures.add(lock.revalidateIfNeededAfterReconnection()); } } - - try { - FutureUtil.waitForAll(futures).get(); - } catch (ExecutionException | InterruptedException e) { - log.warn("Failure when processing session event", e); - } - })); + return FutureUtil.waitForAll(futures) + .exceptionally(ex -> { + log.warn("Failure when processing session event", ex); + return null; + }); + }, executor)); } private void handleDataNotification(Notification n) { From a50ecda11f947e5120cfb01adbfd46f22b76f1f2 Mon Sep 17 00:00:00 2001 From: Nihar Rathod Date: Tue, 21 Mar 2023 19:30:54 +0530 Subject: [PATCH 017/280] [fix][test] PulsarStandaloneTest.testMetadataInitialization (#19842) --- .../test/resources/configurations/standalone_no_client_auth.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf index 573d6cdba9933..d9411e655ad5b 100644 --- a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf +++ b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf @@ -29,3 +29,4 @@ authenticationEnabled=true authenticationProviders=org.apache.pulsar.MockTokenAuthenticationProvider brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= +loadBalancerOverrideBrokerNicSpeedGbps=2 \ No newline at end of file From 30d2469086fea989ac8baf059df8e69c66a68d89 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Tue, 21 Mar 2023 07:33:00 -0700 Subject: [PATCH 018/280] [improve][broker] PIP-192 Excluded bundles with isolation policy or anti-affinity-group policy from topk load bundles (#19742) ### Motivation Raising a PR to implement https://github.com/apache/pulsar/issues/16691. Bundles with isolation policies or anti-affinity-group policies are not ideal targets to auto-unload as destination brokers are limited. ### Modifications This PR - Excluded bundles with isolation policy or anti-affinity-group policy from topk load bundles - Introduced a config `loadBalancerSheddingBundlesWithPoliciesEnabled ` to control this behavior. --- .../pulsar/broker/ServiceConfiguration.java | 10 + .../extensions/models/TopKBundles.java | 79 ++++++-- .../AntiAffinityGroupPolicyHelper.java | 31 +-- .../reporter/TopBundleLoadDataReporter.java | 2 +- .../extensions/scheduler/TransferShedder.java | 7 + ...tiAffinityNamespaceGroupExtensionTest.java | 30 +++ .../extensions/models/TopKBundlesTest.java | 182 ++++++++++++++++-- .../TopBundleLoadDataReporterTest.java | 39 +++- .../scheduler/TransferShedderTest.java | 42 ++-- 9 files changed, 356 insertions(+), 66 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index ff242888ae0b6..ba82667690dc5 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2600,6 +2600,16 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private long loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds = 604800; + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + dynamic = true, + doc = "Option to automatically unload namespace bundles with affinity(isolation) " + + "or anti-affinity group policies." + + "Such bundles are not ideal targets to auto-unload as destination brokers are limited." + + "(only used in load balancer extension logics)" + ) + private boolean loadBalancerSheddingBundlesWithPoliciesEnabled = false; + /**** --- Replication. --- ****/ @FieldContext( category = CATEGORY_REPLICATION, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java index c189005b9539c..d49361741bec4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java @@ -24,10 +24,14 @@ import java.util.Map; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.NoArgsConstructor; import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; /** @@ -36,7 +40,7 @@ @Getter @ToString @EqualsAndHashCode -@NoArgsConstructor +@Slf4j public class TopKBundles { // temp array for sorting @@ -44,6 +48,15 @@ public class TopKBundles { private final TopBundlesLoadData loadData = new TopBundlesLoadData(); + private final PulsarService pulsar; + + private final SimpleResourceAllocationPolicies allocationPolicies; + + public TopKBundles(PulsarService pulsar) { + this.pulsar = pulsar; + this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); + } + /** * Update the topK bundles from the input bundleStats. * @@ -52,26 +65,35 @@ public class TopKBundles { */ public void update(Map bundleStats, int topk) { arr.clear(); - for (var etr : bundleStats.entrySet()) { - if (etr.getKey().startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) { - continue; + try { + var isLoadBalancerSheddingBundlesWithPoliciesEnabled = + pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled(); + for (var etr : bundleStats.entrySet()) { + String bundle = etr.getKey(); + if (bundle.startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) { + continue; + } + if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled && hasPolicies(bundle)) { + continue; + } + arr.add(etr); } - arr.add(etr); - } - var topKBundlesLoadData = loadData.getTopBundlesLoadData(); - topKBundlesLoadData.clear(); - if (arr.isEmpty()) { - return; - } - topk = Math.min(topk, arr.size()); - partitionSort(arr, topk); + var topKBundlesLoadData = loadData.getTopBundlesLoadData(); + topKBundlesLoadData.clear(); + if (arr.isEmpty()) { + return; + } + topk = Math.min(topk, arr.size()); + partitionSort(arr, topk); - for (int i = 0; i < topk; i++) { - var etr = arr.get(i); - topKBundlesLoadData.add( - new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue())); + for (int i = 0; i < topk; i++) { + var etr = arr.get(i); + topKBundlesLoadData.add( + new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue())); + } + } finally { + arr.clear(); } - arr.clear(); } static void partitionSort(List> arr, int k) { @@ -109,4 +131,23 @@ static void partitionSort(List> arr, int } Collections.sort(arr.subList(0, end), (a, b) -> b.getValue().compareTo(a.getValue())); } + + private boolean hasPolicies(String bundle) { + NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle)); + if (allocationPolicies.areIsolationPoliciesPresent(namespace)) { + return true; + } + + try { + var antiAffinityGroupOptional = + LoadManagerShared.getNamespaceAntiAffinityGroup(pulsar, namespace.toString()); + if (antiAffinityGroupOptional.isPresent()) { + return true; + } + } catch (MetadataStoreException e) { + log.error("Failed to get localPolicies for bundle:{}.", bundle, e); + throw new RuntimeException(e); + } + return false; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java index c8332a1d7b5e6..69e3302bebd50 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java @@ -54,32 +54,37 @@ public boolean canUnload( String bundle, String srcBroker, Optional dstBroker) { + try { var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup( pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle)); - if (antiAffinityGroupOptional.isPresent()) { + if (antiAffinityGroupOptional.isEmpty()) { + return true; + } - // copy to retain the input brokers - Map candidates = new HashMap<>(brokers); + // bundle has anti-affinityGroup + if (!pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) { + return false; + } - filter(candidates, bundle); + // copy to retain the input brokers + Map candidates = new HashMap<>(brokers); - candidates.remove(srcBroker); + filter(candidates, bundle); - // unload case - if (dstBroker.isEmpty()) { - return !candidates.isEmpty(); - } + candidates.remove(srcBroker); - // transfer case - return candidates.containsKey(dstBroker.get()); + // unload case + if (dstBroker.isEmpty()) { + return !candidates.isEmpty(); } + + // transfer case + return candidates.containsKey(dstBroker.get()); } catch (MetadataStoreException e) { log.error("Failed to check unload candidates. Assumes that bundle:{} cannot unload ", bundle, e); return false; } - - return true; } public void listenFailureDomainUpdate() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java index 59e328fc2be80..cc1a39add5d4e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java @@ -48,7 +48,7 @@ public TopBundleLoadDataReporter(PulsarService pulsar, this.lookupServiceAddress = lookupServiceAddress; this.bundleLoadDataStore = bundleLoadDataStore; this.lastBundleStatsUpdatedAt = 0; - this.topKBundles = new TopKBundles(); + this.topKBundles = new TopKBundles(pulsar); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 3c67479bcc7cb..effddaf9c4159 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -456,6 +456,7 @@ private boolean isTransferable(LoadManagerContext context, if (pulsar == null || allocationPolicies == null) { return true; } + String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle); final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); NamespaceBundle namespaceBundle = @@ -491,6 +492,12 @@ private boolean canTransferWithIsolationPoliciesToBroker(LoadManagerContext cont || !allocationPolicies.areIsolationPoliciesPresent(namespaceBundle.getNamespaceObject())) { return true; } + + // bundle has isolation policies. + if (!context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) { + return false; + } + boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled(); Set candidates = isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, namespaceBundle); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java index 32822c0f5b524..16b87195b1511 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java @@ -105,6 +105,8 @@ protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, Str brokersCopy.remove(srcBroker); var dstBroker = brokersCopy.entrySet().iterator().next().getKey(); + // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = true + conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, "not-enabled-" + namespace + "/" + bundle, srcBroker, Optional.of(dstBroker))); @@ -128,6 +130,34 @@ protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, Str assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, namespaceBundle, dstBroker, Optional.empty())); + + // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = false + conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); + assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, + "not-enabled-" + namespace + "/" + bundle, + srcBroker, Optional.of(dstBroker))); + + assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, + "not-enabled-" + namespace + "/" + bundle, + srcBroker, Optional.empty())); + + assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, + namespaceBundle, + srcBroker, Optional.of(dstBroker))); + + assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, + namespaceBundle, + dstBroker, Optional.of(srcBroker))); + + assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, + namespaceBundle, + srcBroker, Optional.empty())); + + assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, + namespaceBundle, + dstBroker, Optional.empty())); + + } catch (Exception e) { throw new RuntimeException(e); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java index d759dd016955a..9b42163bd664b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java @@ -18,65 +18,221 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.models; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.resources.LocalPoliciesResources; +import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; +import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; +import org.apache.pulsar.common.policies.data.LocalPolicies; +import org.apache.pulsar.common.policies.data.NamespaceIsolationData; +import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl; +import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Test(groups = "broker") public class TopKBundlesTest { + private PulsarService pulsar; + private ServiceConfiguration configuration; + private NamespaceResources.IsolationPolicyResources isolationPolicyResources; + private PulsarResources pulsarResources; + private LocalPoliciesResources localPoliciesResources; + String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF"; + String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF"; + String bundle3 = "my-tenant/my-namespace3/0x00000000_0x0FFFFFFF"; + String bundle4 = "my-tenant/my-namespace4/0x00000000_0x0FFFFFFF"; + + @BeforeMethod + public void init() throws MetadataStoreException { + pulsar = mock(PulsarService.class); + configuration = new ServiceConfiguration(); + doReturn(configuration).when(pulsar).getConfiguration(); + configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); + pulsarResources = mock(PulsarResources.class); + var namespaceResources = mock(NamespaceResources.class); + + isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class); + doReturn(pulsarResources).when(pulsar).getPulsarResources(); + doReturn(namespaceResources).when(pulsarResources).getNamespaceResources(); + doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies(); + doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any()); + + localPoliciesResources = mock(LocalPoliciesResources.class); + doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies(); + doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any()); + + } @Test public void testTopBundlesLoadData() { Map bundleStats = new HashMap<>(); - var topKBundles = new TopKBundles(); + var topKBundles = new TopKBundles(pulsar); NamespaceBundleStats stats1 = new NamespaceBundleStats(); stats1.msgRateIn = 500; - bundleStats.put("bundle-1", stats1); + bundleStats.put(bundle1, stats1); NamespaceBundleStats stats2 = new NamespaceBundleStats(); stats2.msgRateIn = 10000; - bundleStats.put("bundle-2", stats2); + bundleStats.put(bundle2, stats2); NamespaceBundleStats stats3 = new NamespaceBundleStats(); stats3.msgRateIn = 100000; - bundleStats.put("bundle-3", stats3); + bundleStats.put(bundle3, stats3); NamespaceBundleStats stats4 = new NamespaceBundleStats(); stats4.msgRateIn = 0; - bundleStats.put("bundle-4", stats4); + bundleStats.put(bundle4, stats4); topKBundles.update(bundleStats, 3); var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1); var top2 = topKBundles.getLoadData().getTopBundlesLoadData().get(2); - assertEquals(top0.bundleName(), "bundle-3"); - assertEquals(top1.bundleName(), "bundle-2"); - assertEquals(top2.bundleName(), "bundle-1"); + assertEquals(top0.bundleName(), bundle3); + assertEquals(top1.bundleName(), bundle2); + assertEquals(top2.bundleName(), bundle1); } @Test public void testSystemNamespace() { Map bundleStats = new HashMap<>(); - var topKBundles = new TopKBundles(); + var topKBundles = new TopKBundles(pulsar); NamespaceBundleStats stats1 = new NamespaceBundleStats(); stats1.msgRateIn = 500; - bundleStats.put("pulsar/system/bundle-1", stats1); + bundleStats.put("pulsar/system/0x00000000_0x0FFFFFFF", stats1); NamespaceBundleStats stats2 = new NamespaceBundleStats(); stats2.msgRateIn = 10000; - bundleStats.put("pulsar/system/bundle-2", stats2); + bundleStats.put(bundle1, stats2); topKBundles.update(bundleStats, 2); - assertTrue(topKBundles.getLoadData().getTopBundlesLoadData().isEmpty()); + + assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1); + var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); + assertEquals(top0.bundleName(), bundle1); + } + + + private void setAntiAffinityGroup() throws MetadataStoreException { + LocalPolicies localPolicies = new LocalPolicies(null, null, "namespaceAntiAffinityGroup"); + NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle2)); + doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(eq(namespace)); + } + + private void setIsolationPolicy() throws MetadataStoreException { + Map parameters = new HashMap<>(); + parameters.put("min_limit", "3"); + parameters.put("usage_threshold", "90"); + var policyData = Map.of("policy", (NamespaceIsolationDataImpl) + NamespaceIsolationData.builder() + .namespaces(Collections.singletonList("my-tenant/my-namespace1.*")) + .primary(Collections.singletonList("prod1-broker[1-3].messaging.use.example.com")) + .secondary(Collections.singletonList("prod1-broker.*.use.example.com")) + .autoFailoverPolicy(AutoFailoverPolicyData.builder() + .policyType(AutoFailoverPolicyType.min_available) + .parameters(parameters) + .build() + ).build()); + + NamespaceIsolationPolicies policies = new NamespaceIsolationPolicies(policyData); + doReturn(Optional.of(policies)).when(isolationPolicyResources).getIsolationDataPolicies(any()); + } + + @Test + public void testIsolationPolicy() throws MetadataStoreException { + + setIsolationPolicy(); + + Map bundleStats = new HashMap<>(); + var topKBundles = new TopKBundles(pulsar); + NamespaceBundleStats stats1 = new NamespaceBundleStats(); + stats1.msgRateIn = 500; + bundleStats.put(bundle1, stats1); + + NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats2.msgRateIn = 10000; + bundleStats.put(bundle2, stats2); + + topKBundles.update(bundleStats, 2); + assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1); + var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); + assertEquals(top0.bundleName(), bundle2); + } + + + @Test + public void testAntiAffinityGroupPolicy() throws MetadataStoreException { + + setAntiAffinityGroup(); + + Map bundleStats = new HashMap<>(); + var topKBundles = new TopKBundles(pulsar); + NamespaceBundleStats stats1 = new NamespaceBundleStats(); + stats1.msgRateIn = 500; + bundleStats.put(bundle1, stats1); + + NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats2.msgRateIn = 10000; + bundleStats.put(bundle2, stats2); + + topKBundles.update(bundleStats, 2); + assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1); + var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); + assertEquals(top0.bundleName(), bundle1); + + } + + @Test + public void testLoadBalancerSheddingBundlesWithPoliciesEnabledConfig() throws MetadataStoreException { + + setIsolationPolicy(); + setAntiAffinityGroup(); + + configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); + + Map bundleStats = new HashMap<>(); + var topKBundles = new TopKBundles(pulsar); + NamespaceBundleStats stats1 = new NamespaceBundleStats(); + stats1.msgRateIn = 500; + bundleStats.put(bundle1, stats1); + + NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats2.msgRateIn = 10000; + bundleStats.put(bundle2, stats2); + + topKBundles.update(bundleStats, 2); + + assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 2); + var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); + var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1); + + assertEquals(top0.bundleName(), bundle2); + assertEquals(top1.bundleName(), bundle1); + + configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); + + topKBundles.update(bundleStats, 2); + + assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 0); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java index ce2d3d8c3ea93..b5c415c405fbd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java @@ -27,6 +27,7 @@ import static org.testng.Assert.assertNull; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; @@ -34,8 +35,12 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.broker.resources.LocalPoliciesResources; +import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.PulsarStats; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -48,26 +53,44 @@ public class TopBundleLoadDataReporterTest { PulsarStats pulsarStats; Map bundleStats; ServiceConfiguration config; + private NamespaceResources.IsolationPolicyResources isolationPolicyResources; + private PulsarResources pulsarResources; + private LocalPoliciesResources localPoliciesResources; + String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF"; + String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF"; @BeforeMethod - void setup() { + void setup() throws MetadataStoreException { config = new ServiceConfiguration(); pulsar = mock(PulsarService.class); store = mock(LoadDataStore.class); brokerService = mock(BrokerService.class); pulsarStats = mock(PulsarStats.class); + pulsarResources = mock(PulsarResources.class); + isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class); + var namespaceResources = mock(NamespaceResources.class); + localPoliciesResources = mock(LocalPoliciesResources.class); + doReturn(brokerService).when(pulsar).getBrokerService(); doReturn(config).when(pulsar).getConfiguration(); doReturn(pulsarStats).when(brokerService).getPulsarStats(); doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any()); + doReturn(pulsarResources).when(pulsar).getPulsarResources(); + doReturn(namespaceResources).when(pulsarResources).getNamespaceResources(); + doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies(); + doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any()); + + doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies(); + doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any()); + bundleStats = new HashMap<>(); NamespaceBundleStats stats1 = new NamespaceBundleStats(); stats1.msgRateIn = 500; - bundleStats.put("bundle-1", stats1); + bundleStats.put(bundle1, stats1); NamespaceBundleStats stats2 = new NamespaceBundleStats(); stats2.msgRateIn = 10000; - bundleStats.put("bundle-2", stats2); + bundleStats.put(bundle2, stats2); doReturn(bundleStats).when(brokerService).getBundleStats(); } @@ -81,25 +104,25 @@ public void testGenerateLoadData() throws IllegalAccessException { doReturn(1l).when(pulsarStats).getUpdatedAt(); config.setLoadBalancerBundleLoadReportPercentage(100); var target = new TopBundleLoadDataReporter(pulsar, "", store); - var expected = new TopKBundles(); + var expected = new TopKBundles(pulsar); expected.update(bundleStats, 2); assertEquals(target.generateLoadData(), expected.getLoadData()); config.setLoadBalancerBundleLoadReportPercentage(50); FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); - expected = new TopKBundles(); + expected = new TopKBundles(pulsar); expected.update(bundleStats, 1); assertEquals(target.generateLoadData(), expected.getLoadData()); config.setLoadBalancerBundleLoadReportPercentage(1); FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); - expected = new TopKBundles(); + expected = new TopKBundles(pulsar); expected.update(bundleStats, 1); assertEquals(target.generateLoadData(), expected.getLoadData()); doReturn(new HashMap()).when(brokerService).getBundleStats(); FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); - expected = new TopKBundles(); + expected = new TopKBundles(pulsar); assertEquals(target.generateLoadData(), expected.getLoadData()); } @@ -116,7 +139,7 @@ public void testReportForce() { public void testReport(){ var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store); doReturn(1l).when(pulsarStats).getUpdatedAt(); - var expected = new TopKBundles(); + var expected = new TopKBundles(pulsar); expected.update(bundleStats, 1); target.reportAsync(false); verify(store, times(1)).pushAsync("broker-1", expected.getLoadData()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index a89f33bb98737..5ca9345d6def5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -97,8 +97,8 @@ public class TransferShedderTest { double setupLoadStd = 0.3982762860126121; PulsarService pulsar; - AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; ServiceConfiguration conf; + AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; LocalPoliciesResources localPoliciesResources; String bundleD1 = "my-tenant/my-namespaceD/0x00000000_0x0FFFFFFF"; String bundleD2 = "my-tenant/my-namespaceD/0x0FFFFFFF_0xFFFFFFFF"; @@ -109,8 +109,7 @@ public class TransferShedderTest { public void init() throws MetadataStoreException { pulsar = mock(PulsarService.class); conf = new ServiceConfiguration(); - doReturn(conf).when(pulsar).getConfiguration(); - + conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); var pulsarResources = mock(PulsarResources.class); var namespaceResources = mock(NamespaceResources.class); var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class); @@ -118,6 +117,7 @@ public void init() throws MetadataStoreException { var namespaceService = mock(NamespaceService.class); localPoliciesResources = mock(LocalPoliciesResources.class); antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class); + doReturn(conf).when(pulsar).getConfiguration(); doReturn(namespaceService).when(pulsar).getNamespaceService(); doReturn(pulsarResources).when(pulsar).getPulsarResources(); doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies(); @@ -138,9 +138,9 @@ public void init() throws MetadataStoreException { }).when(factory).getBundle(anyString(), anyString()); doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any()); } + public LoadManagerContext setupContext(){ var ctx = getContext(); - ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2)); @@ -160,7 +160,6 @@ public LoadManagerContext setupContext(){ public LoadManagerContext setupContext(int clusterSize) { var ctx = getContext(); - ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); @@ -170,7 +169,7 @@ public LoadManagerContext setupContext(int clusterSize) { int brokerLoad = rand.nextInt(100); brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad)); int bundleLoad = rand.nextInt(brokerLoad + 1); - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("bundle" + i, + topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, bundleLoad, brokerLoad - bundleLoad)); } return ctx; @@ -199,7 +198,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int namespaceBundleStats1.msgThroughputOut = load1; var namespaceBundleStats2 = new NamespaceBundleStats(); namespaceBundleStats2.msgThroughputOut = load2; - var topKBundles = new TopKBundles(); + var topKBundles = new TopKBundles(pulsar); topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1, bundlePrefix + "/0x0FFFFFFF_0xFFFFFFFF", namespaceBundleStats2), 2); return topKBundles.getLoadData(); @@ -208,7 +207,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) { var namespaceBundleStats1 = new NamespaceBundleStats(); namespaceBundleStats1.msgThroughputOut = load1; - var topKBundles = new TopKBundles(); + var topKBundles = new TopKBundles(pulsar); topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1), 2); return topKBundles.getLoadData(); } @@ -220,7 +219,7 @@ public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace, namespaceBundleStats1.msgThroughputOut = load1 * 1e6; var namespaceBundleStats2 = new NamespaceBundleStats(); namespaceBundleStats2.msgThroughputOut = load2 * 1e6; - var topKBundles = new TopKBundles(); + var topKBundles = new TopKBundles(pulsar); topKBundles.update(Map.of(namespace + "/0x00000000_0x7FFFFFF", namespaceBundleStats1, namespace + "/0x7FFFFFF_0xFFFFFFF", namespaceBundleStats2), 2); return topKBundles.getLoadData(); @@ -229,6 +228,8 @@ public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace, public LoadManagerContext getContext(){ var ctx = mock(LoadManagerContext.class); var conf = new ServiceConfiguration(); + conf.setLoadBalancerDebugModeEnabled(true); + conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); var brokerLoadDataStore = new LoadDataStore() { Map map = new HashMap<>(); @Override @@ -461,8 +462,6 @@ public void testGetAvailableBrokersFailed() { BrokerRegistry registry = ctx.brokerRegistry(); doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - - assertTrue(res.isEmpty()); assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1); assertEquals(counter.getLoadAvg(), setupLoadAvg); assertEquals(counter.getLoadStd(), setupLoadStd); @@ -501,6 +500,25 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException { assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); assertEquals(counter.getLoadStd(), setupLoadStd); + + + // test setLoadBalancerSheddingBundlesWithPoliciesEnabled=false; + doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); + ctx.brokerConfiguration().setLoadBalancerTransferEnabled(true); + ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); + res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + + // Test unload a has isolation policies broker. + ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false); + res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 2); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } public BrokerLookupData getLookupData() { @@ -602,7 +620,7 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me var expected2 = new HashSet<>(); expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), Success, Overloaded)); - assertEquals(res, expected2); + assertEquals(res2, expected2); assertEquals(counter.getLoadAvg(), setupLoadAvg); assertEquals(counter.getLoadStd(), setupLoadStd); } From f294be37ce12e311a6bd87493cf0cba11fc6c2f8 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 22 Mar 2023 15:30:00 +0800 Subject: [PATCH 019/280] [fix] [ml] make the result of delete cursor is success if cursor is deleted (#19825) When deleting the zk node of the cursor, if the exception `MetadataStoreException.NotFoundException` occurs, the deletion is considered successful. --- .../mledger/impl/MetaStoreImpl.java | 16 ++++++++---- .../mledger/impl/ManagedLedgerTest.java | 26 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index bcb73553324dd..3bdaac1e8d80e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -47,6 +47,7 @@ import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.Notification; @@ -292,7 +293,7 @@ public void asyncUpdateCursorInfo(String ledgerName, String cursorName, ManagedC @Override public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCallback callback) { String path = PREFIX + ledgerName + "/" + cursorName; - log.info("[{}] Remove consumer={}", ledgerName, cursorName); + log.info("[{}] Remove cursor={}", ledgerName, cursorName); store.delete(path, Optional.empty()) .thenAcceptAsync(v -> { @@ -301,11 +302,16 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal } callback.operationComplete(null, null); }, executor.chooseThread(ledgerName)) - .exceptionally(ex -> { - executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback - .operationFailed(getException(ex)))); + .exceptionallyAsync(ex -> { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof MetadataStoreException.NotFoundException){ + log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName); + callback.operationComplete(null, null); + return null; + } + SafeRunnable.safeRun(() -> callback.operationFailed(getException(ex))); return null; - }); + }, executor.chooseThread(ledgerName)); } @Override diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index a4d8b75d00c96..dd30cde72e769 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -128,6 +128,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.Stat; import org.apache.pulsar.metadata.api.extended.SessionEvent; @@ -3969,4 +3970,29 @@ public void testGetEstimatedBacklogSize() throws Exception { Assert.assertEquals(ledger.getEstimatedBacklogSize(((PositionImpl) positions.get(9)).getNext()), 0); ledger.close(); } + + @Test + public void testDeleteCursorTwice() throws Exception { + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("ml"); + String cursorName = "cursor_1"; + ml.openCursor(cursorName); + syncRemoveCursor(ml, cursorName); + syncRemoveCursor(ml, cursorName); + } + + private void syncRemoveCursor(ManagedLedgerImpl ml, String cursorName){ + CompletableFuture future = new CompletableFuture<>(); + ml.getStore().asyncRemoveCursor(ml.name, cursorName, new MetaStoreCallback() { + @Override + public void operationComplete(Void result, Stat stat) { + future.complete(null); + } + + @Override + public void operationFailed(MetaStoreException e) { + future.completeExceptionally(FutureUtil.unwrapCompletionException(e)); + } + }); + future.join(); + } } From a9037334a399af905fae94d2aefa5db339cbd5b1 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 22 Mar 2023 16:49:13 +0800 Subject: [PATCH 020/280] [fix] [admin] Make response code to 400 instead of 500 when delete topic fails due to enabled geo-replication (#19879) Motivation: As expected, If geo-replication is enabled, a topic cannot be deleted. However deleting that topic returns a 500, and no further info. Modifications: Make response code to 400 instead of 500 when delete topic fails due to enabled geo-replication --- .../broker/admin/v2/PersistentTopics.java | 4 ++- .../pulsar/broker/service/ReplicatorTest.java | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 7eeaec403d842..477cc8b57129d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -1106,7 +1106,9 @@ public void deleteTopic( ex = new RestException(Response.Status.PRECONDITION_FAILED, t.getMessage()); } - if (isManagedLedgerNotFoundException(t)) { + if (t instanceof IllegalStateException){ + ex = new RestException(422/* Unprocessable entity*/, t.getMessage()); + } else if (isManagedLedgerNotFoundException(t)) { ex = new RestException(Response.Status.NOT_FOUND, getTopicNotFoundErrorMessage(topicName.toString())); } else if (!isRedirectException(ex)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index ab4f6a5c7f8ba..4086c54a0ba4b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -70,6 +70,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -869,6 +870,33 @@ public void testReplicatorProducerClosing() throws Exception { assertNull(producer); } + @Test + public void testDeleteTopicFailure() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID()); + admin1.topics().createNonPartitionedTopic(topicName); + try { + admin1.topics().delete(topicName); + fail("Delete topic should fail if enabled replicator"); + } catch (Exception ex) { + assertTrue(ex instanceof PulsarAdminException); + assertEquals(((PulsarAdminException) ex).getStatusCode(), 422/* Unprocessable entity*/); + } + } + + @Test + public void testDeletePartitionedTopicFailure() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID()); + admin1.topics().createPartitionedTopic(topicName, 2); + admin1.topics().createSubscription(topicName, "sub1", MessageId.earliest); + try { + admin1.topics().deletePartitionedTopic(topicName); + fail("Delete topic should fail if enabled replicator"); + } catch (Exception ex) { + assertTrue(ex instanceof PulsarAdminException); + assertEquals(((PulsarAdminException) ex).getStatusCode(), 422/* Unprocessable entity*/); + } + } + @Test(priority = 4, timeOut = 30000) public void testReplicatorProducerName() throws Exception { log.info("--- Starting ReplicatorTest::testReplicatorProducerName ---"); From a1fdbd294cab5c48dd2b092085465eb4075cc6a4 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 23 Mar 2023 20:09:57 +0800 Subject: [PATCH 021/280] [fix][broker] Fix NPE when update topic policy. (#19875) --- .../apache/pulsar/common/policies/data/TopicPolicies.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java index caa9cd3fd1daa..4a76170d116a3 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java @@ -20,6 +20,7 @@ import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -188,4 +189,8 @@ public boolean isSchemaValidationEnforced() { public Set getReplicationClustersSet() { return replicationClusters != null ? Sets.newTreeSet(this.replicationClusters) : null; } + + public Map getSubscriptionPolicies() { + return subscriptionPolicies == null ? Collections.emptyMap() : subscriptionPolicies; + } } From c172a775fa0fc5762a9703669ed8ebcd2efbe042 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 24 Mar 2023 15:05:02 +0800 Subject: [PATCH 022/280] [fix][broker] Fix can't send ErrorCommand when message is null value (#19899) --- .../pulsar/client/api/ClientErrorsTest.java | 29 +++++++++++++++++-- .../pulsar/common/protocol/Commands.java | 4 +-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java index e8b9baa992c46..61c7a98602b69 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java @@ -22,16 +22,15 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import io.netty.channel.ChannelHandlerContext; - +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; - import lombok.Cleanup; +import org.apache.bookkeeper.common.util.JsonUtil; import org.apache.pulsar.client.impl.ConsumerBase; import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.ProducerBase; @@ -831,4 +830,28 @@ public void testConsumerReconnect() throws Exception { mockBrokerService.resetHandleConnect(); mockBrokerService.resetHandleSubscribe(); } + + @Test + public void testCommandErrorMessageIsNull() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(mockBrokerService.getBrokerAddress()).build(); + + mockBrokerService.setHandleProducer((ctx, producer) -> { + try { + ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.AuthorizationError, null)); + } catch (Exception e) { + fail("Send error command failed", e); + } + }); + + try { + client.newProducer().topic("persistent://prop/use/ns/t1").create(); + fail(); + } catch (Exception e) { + assertTrue(e instanceof PulsarClientException.AuthorizationException); + Map map = JsonUtil.fromJson(e.getMessage(), Map.class); + assertEquals(map.get("errorMsg"), ""); + } + mockBrokerService.resetHandleProducer(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 8a5684cf676b0..85c4d021fdf22 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -368,7 +368,7 @@ public static BaseCommand newErrorCommand(long requestId, ServerError serverErro cmd.setError() .setRequestId(requestId) .setError(serverError) - .setMessage(message); + .setMessage(message != null ? message : ""); return cmd; } @@ -401,7 +401,7 @@ public static BaseCommand newSendErrorCommand(long producerId, long sequenceId, .setProducerId(producerId) .setSequenceId(sequenceId) .setError(error) - .setMessage(errorMsg); + .setMessage(errorMsg != null ? errorMsg : ""); return cmd; } From f1f8dab972b098be69ad35ab3d307f19284c4e48 Mon Sep 17 00:00:00 2001 From: Abhilash Mandaliya Date: Fri, 24 Mar 2023 16:51:22 +0530 Subject: [PATCH 023/280] [improve][io][broker] Updated org.reflections-reflections library version (#19898) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 257018b0b0598..3f33069b8c252 100644 --- a/pom.xml +++ b/pom.xml @@ -149,7 +149,7 @@ flexible messaging model and an intuitive client API. 1.0.6 1.0.2.3 2.13.4.20221013 - 0.9.11 + 0.10.2 1.6.2 8.37 0.40.2 From 1af0ff35109af066662324515e59d2b8bff8c899 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 24 Mar 2023 15:03:47 +0000 Subject: [PATCH 024/280] [improve][ci] Handle retry of maven downloads for more failure cases (#19903) --- .github/workflows/ci-go-functions.yaml | 2 +- .github/workflows/ci-maven-cache-update.yaml | 2 +- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- .github/workflows/pulsar-ci-flaky.yaml | 2 +- .github/workflows/pulsar-ci.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index f4d1ae0b99887..a34cb1a90908e 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -32,7 +32,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: preconditions: diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index 87570586fde6e..a5a88e2e0b8a6 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -42,7 +42,7 @@ on: - cron: '30 */12 * * *' env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: update-maven-dependencies-cache: diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 194d88c582d42..dba1c92ca28dd 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: run-owasp-dependency-check: diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index acfa66ff43c74..ea19bc3307819 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -36,7 +36,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 721a1d2eafc72..f7dbc755264d6 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -36,7 +36,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR From 8081ee26d8d3727c720800a3453a798893763fee Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 25 Mar 2023 15:39:37 +0800 Subject: [PATCH 025/280] [improve][broker][PIP-195] Make bucket merge operation asynchronous (#19873) --- .../bucket/BucketDelayedDeliveryTracker.java | 139 +++++++++++------- .../delayed/bucket/ImmutableBucket.java | 17 ++- .../broker/delayed/bucket/MutableBucket.java | 6 + ...ayedMessageIndexBucketSnapshotFormat.proto | 1 + .../BookkeeperBucketSnapshotStorageTest.java | 3 + .../BucketDelayedDeliveryTrackerTest.java | 57 +++++-- 6 files changed, 154 insertions(+), 69 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 31fdaa6fb7667..1cf8bd2b20d17 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -90,6 +90,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final Table snapshotSegmentLastIndexTable; + private static final Long INVALID_BUCKET_ID = -1L; + public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -246,12 +248,12 @@ private void afterCreateImmutableBucket(Pair immu immutableBucket); immutableBucket.getSnapshotCreateFuture().ifPresent(createFuture -> { - CompletableFuture future = createFuture.whenComplete((__, ex) -> { + CompletableFuture future = createFuture.handle((bucketId, ex) -> { if (ex == null) { immutableBucket.setSnapshotSegments(null); log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(), immutableBucket.bucketKey()); - return; + return bucketId; } //TODO Record create snapshot failed @@ -277,6 +279,7 @@ private void afterCreateImmutableBucket(Pair immu snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getTimestamp()); } + return INVALID_BUCKET_ID; }); immutableBucket.setSnapshotCreateFuture(future); }); @@ -308,12 +311,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver lastMutableBucket.resetLastMutableBucketRange(); if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { - try { - asyncMergeBucketSnapshot().get(2 * AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS); - } catch (Exception e) { - // Ignore exception to merge bucket on the next schedule. - log.error("[{}] An exception occurs when merge bucket snapshot.", dispatcher.getName(), e); - } + asyncMergeBucketSnapshot(); } } @@ -341,18 +339,26 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver private synchronized CompletableFuture asyncMergeBucketSnapshot() { List values = immutableBuckets.asMapOfRanges().values().stream().toList(); long minNumberMessages = Long.MAX_VALUE; + long minScheduleTimestamp = Long.MAX_VALUE; int minIndex = -1; for (int i = 0; i + 1 < values.size(); i++) { ImmutableBucket bucketL = values.get(i); ImmutableBucket bucketR = values.get(i + 1); - long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages; - if (numberMessages < minNumberMessages) { - minNumberMessages = (int) numberMessages; - if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId() - && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId() - && bucketL.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone() - && bucketR.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone()) { - minIndex = i; + // We should skip the bucket which last segment already been load to memory, avoid record replicated index. + if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId() + && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId() + // Skip the bucket that is merging + && !bucketL.merging && !bucketR.merging){ + long scheduleTimestamp = + Math.min(bucketL.firstScheduleTimestamps.get(bucketL.currentSegmentEntryId + 1), + bucketR.firstScheduleTimestamps.get(bucketR.currentSegmentEntryId + 1)); + long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages; + if (scheduleTimestamp <= minScheduleTimestamp) { + minScheduleTimestamp = scheduleTimestamp; + if (numberMessages < minNumberMessages) { + minNumberMessages = numberMessages; + minIndex = i; + } } } } @@ -369,7 +375,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey()); } + + immutableBucketA.merging = true; + immutableBucketB.merging = true; return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> { + synchronized (this) { + immutableBucketA.merging = false; + immutableBucketB.merging = false; + } if (ex != null) { log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex); @@ -382,46 +395,58 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA, ImmutableBucket bucketB) { - CompletableFuture> futureA = - bucketA.getRemainSnapshotSegment(); - CompletableFuture> futureB = - bucketB.getRemainSnapshotSegment(); - return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap) - .thenAccept(combinedDelayedIndexQueue -> { - Pair immutableBucketDelayedIndexPair = - lastMutableBucket.createImmutableBucketAndAsyncPersistent( - timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment, - sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId, - bucketB.endLedgerId); - - // Merge bit map to new bucket - Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); - Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); - Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); - delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { - delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { - if (bitMapA == null) { - return bitMapB; - } + CompletableFuture createAFuture = bucketA.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE); + CompletableFuture createBFuture = bucketB.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE); - bitMapA.or(bitMapB); - return bitMapA; - }); - }); - immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); - - afterCreateImmutableBucket(immutableBucketDelayedIndexPair); + return CompletableFuture.allOf(createAFuture, createBFuture).thenCompose(bucketId -> { + if (INVALID_BUCKET_ID.equals(createAFuture.join()) || INVALID_BUCKET_ID.equals(createBFuture.join())) { + return FutureUtil.failedFuture(new RuntimeException("Can't merge buckets due to bucket create failed")); + } - immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() - .orElse(NULL_LONG_PROMISE).thenCompose(___ -> { - CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); - CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); - return CompletableFuture.allOf(removeAFuture, removeBFuture); + CompletableFuture> futureA = + bucketA.getRemainSnapshotSegment(); + CompletableFuture> futureB = + bucketB.getRemainSnapshotSegment(); + return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap) + .thenAccept(combinedDelayedIndexQueue -> { + synchronized (BucketDelayedDeliveryTracker.this) { + Pair immutableBucketDelayedIndexPair = + lastMutableBucket.createImmutableBucketAndAsyncPersistent( + timeStepPerBucketSnapshotSegmentInMillis, + maxIndexesPerBucketSnapshotSegment, + sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId, + bucketB.endLedgerId); + + // Merge bit map to new bucket + Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); + Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); + Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); + delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { + delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { + if (bitMapA == null) { + return bitMapB; + } + + bitMapA.or(bitMapB); + return bitMapA; + }); + }); + immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); + + afterCreateImmutableBucket(immutableBucketDelayedIndexPair); + + immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() + .orElse(NULL_LONG_PROMISE).thenCompose(___ -> { + CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); + CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); + return CompletableFuture.allOf(removeAFuture, removeBFuture); + }); + + immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); + immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); + } }); - - immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); - immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); - }); + }); } @Override @@ -477,6 +502,12 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId); if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) { + if (bucket.merging) { + log.info("[{}] Skip load to wait for bucket snapshot merge finish, bucketKey:{}", + dispatcher.getName(), bucket.bucketKey()); + break; + } + final int preSegmentEntryId = bucket.currentSegmentEntryId; if (log.isDebugEnabled()) { log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}", @@ -525,7 +556,6 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId); } }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS); - snapshotSegmentLastIndexTable.remove(ledgerId, entryId); } catch (Exception e) { // Ignore exception to reload this segment on the next schedule. log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}", @@ -533,6 +563,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa break; } } + snapshotSegmentLastIndexTable.remove(ledgerId, entryId); positions.add(new PositionImpl(ledgerId, entryId)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index c9223efa09243..ab1c285011d20 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds; import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE; import com.google.protobuf.ByteString; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -44,7 +45,12 @@ class ImmutableBucket extends Bucket { @Setter - private volatile List snapshotSegments; + private List snapshotSegments; + + boolean merging = false; + + @Setter + List firstScheduleTimestamps = new ArrayList<>(); ImmutableBucket(String dispatcherName, ManagedCursor cursor, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { @@ -92,6 +98,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b this.setLastSegmentEntryId(metadataList.size()); this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); + List firstScheduleTimestamps = metadataList.stream().map( + SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); + this.setFirstScheduleTimestamps(firstScheduleTimestamps); return nextSnapshotEntryIndex + 1; }); @@ -157,8 +166,10 @@ CompletableFuture> return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, lastSegmentEntryId).whenComplete((__, ex) -> { if (ex != null) { - log.warn("[{}] Failed to get remain bucket snapshot segment, bucketKey: {}.", - dispatcherName, bucketKey(), ex); + log.warn( + "[{}] Failed to get remain bucket snapshot segment, bucketKey: {}," + + " nextSegmentEntryId: {}, lastSegmentEntryId: {}", + dispatcherName, bucketKey(), nextSegmentEntryId, lastSegmentEntryId, ex); } }); }, BucketSnapshotPersistenceException.class, MaxRetryTimes); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index e743f39e6920d..1577bf8fa51d2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -75,11 +75,15 @@ Pair createImmutableBucketAndAsyncPersistent( SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); + List firstScheduleTimestamps = new ArrayList<>(); long currentTimestampUpperLimit = 0; + long currentFirstTimestamp = 0L; while (!delayedIndexQueue.isEmpty()) { DelayedIndex delayedIndex = delayedIndexQueue.peek(); long timestamp = delayedIndex.getTimestamp(); if (currentTimestampUpperLimit == 0) { + currentFirstTimestamp = timestamp; + firstScheduleTimestamps.add(currentFirstTimestamp); currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1; } @@ -104,6 +108,7 @@ Pair createImmutableBucketAndAsyncPersistent( || (maxIndexesPerBucketSnapshotSegment != -1 && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); + segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp); currentTimestampUpperLimit = 0; Iterator> iterator = bitMap.entrySet().iterator(); @@ -134,6 +139,7 @@ Pair createImmutableBucketAndAsyncPersistent( bucket.setCurrentSegmentEntryId(1); bucket.setNumberBucketDelayedMessages(numMessages); bucket.setLastSegmentEntryId(lastSegmentEntryId); + bucket.setFirstScheduleTimestamps(firstScheduleTimestamps); // Skip first segment, because it has already been loaded List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size()); diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto index 8414a583fe5b0..6996b860c5249 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto @@ -31,6 +31,7 @@ message DelayedIndex { message SnapshotSegmentMetadata { map delayed_index_bit_map = 1; required uint64 max_schedule_timestamp = 2; + required uint64 min_schedule_timestamp = 3; } message SnapshotSegment { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index 72052d22b859d..a628b58e10d32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -73,6 +73,7 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException public void testGetSnapshot() throws ExecutionException, InterruptedException { DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); @@ -122,6 +123,7 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() .setMaxScheduleTimestamp(timeMillis) + .setMinScheduleTimestamp(timeMillis) .putAllDelayedIndexBitMap(map).build(); DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = @@ -172,6 +174,7 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException { DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 74101f00b960c..0d53c278fd2f6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -52,6 +52,7 @@ import org.roaringbitmap.buffer.ImmutableRoaringBitmap; import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong; import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -251,18 +252,28 @@ public void testRoaringBitmapSerialize() { } @Test(dataProvider = "delayedTracker") - public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { + public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) { for (int i = 1; i <= 110; i++) { tracker.addMessage(i, i, i * 10); + Awaitility.await().untilAsserted(() -> { + Assert.assertTrue( + tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging)); + }); } assertEquals(110, tracker.getNumberOfDelayedMessages()); int size = tracker.getImmutableBuckets().asMapOfRanges().size(); - assertEquals(10, size); + Awaitility.await().untilAsserted(() -> { + assertEquals(10, size); + }); tracker.addMessage(111, 1011, 111 * 10); + Awaitility.await().untilAsserted(() -> { + Assert.assertTrue( + tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging)); + }); MutableLong delayedMessagesInSnapshot = new MutableLong(); tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> { @@ -271,26 +282,28 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { tracker.close(); - tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, - true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10); + BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 10); - assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue()); + assertEquals(tracker2.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue()); for (int i = 1; i <= 110; i++) { - tracker.addMessage(i, i, i * 10); + tracker2.addMessage(i, i, i * 10); } clockTime.set(110 * 10); - NavigableSet scheduledMessages = tracker.getScheduledMessages(110); + NavigableSet scheduledMessages = tracker2.getScheduledMessages(110); for (int i = 1; i <= 110; i++) { PositionImpl position = scheduledMessages.pollFirst(); assertEquals(position, PositionImpl.get(i, i)); } + + tracker2.close(); } @Test(dataProvider = "delayedTracker") - public void testWithBkException(BucketDelayedDeliveryTracker tracker) { + public void testWithBkException(final BucketDelayedDeliveryTracker tracker) { MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage; mockBucketSnapshotStorage.injectCreateException( new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); @@ -308,11 +321,25 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) { for (int i = 1; i <= 110; i++) { tracker.addMessage(i, i, i * 10); + Awaitility.await().untilAsserted(() -> { + Assert.assertTrue( + tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging)); + }); } assertEquals(110, tracker.getNumberOfDelayedMessages()); + int size = tracker.getImmutableBuckets().asMapOfRanges().size(); + + Awaitility.await().untilAsserted(() -> { + assertEquals(10, size); + }); + tracker.addMessage(111, 1011, 111 * 10); + Awaitility.await().untilAsserted(() -> { + Assert.assertTrue( + tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging)); + }); MutableLong delayedMessagesInSnapshot = new MutableLong(); tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> { @@ -321,11 +348,11 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) { tracker.close(); - tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10); Long delayedMessagesInSnapshotValue = delayedMessagesInSnapshot.getValue(); - assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue); + assertEquals(tracker2.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue); clockTime.set(110 * 10); @@ -338,14 +365,16 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) { mockBucketSnapshotStorage.injectGetSegmentException( new BucketSnapshotPersistenceException("Bookie operation timeout4, op: Get entry")); - assertEquals(tracker.getScheduledMessages(100).size(), 0); + assertEquals(tracker2.getScheduledMessages(100).size(), 0); - assertEquals(tracker.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue); + assertEquals(tracker2.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue); assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty()); assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty()); assertTrue(mockBucketSnapshotStorage.getSegmentExceptionQueue.isEmpty()); assertTrue(mockBucketSnapshotStorage.deleteExceptionQueue.isEmpty()); + + tracker2.close(); } @Test(dataProvider = "delayedTracker") @@ -390,6 +419,8 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) { tracker.getImmutableBuckets().asMapOfRanges().forEach((k, bucket) -> { assertEquals(bucket.getLastSegmentEntryId(), 4); }); + + tracker.close(); } @Test(dataProvider = "delayedTracker") @@ -408,5 +439,7 @@ public void testClear(BucketDelayedDeliveryTracker tracker) { assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0); assertEquals(tracker.getLastMutableBucket().size(), 0); assertEquals(tracker.getSharedBucketPriorityQueue().size(), 0); + + tracker.close(); } } From 329e80b847bd6584cbb031a96d5416d4df488432 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sun, 26 Mar 2023 00:30:40 +0800 Subject: [PATCH 026/280] [improve][broker] Clear delayed message when unsubscribe & Make clear operation asynchronous (#19901) --- .../delayed/DelayedDeliveryTracker.java | 5 +- .../InMemoryDelayedDeliveryTracker.java | 4 +- .../pulsar/broker/delayed/bucket/Bucket.java | 21 +++++-- .../bucket/BucketDelayedDeliveryTracker.java | 63 +++++++++++++------ .../delayed/bucket/ImmutableBucket.java | 57 +++-------------- .../broker/delayed/bucket/MutableBucket.java | 7 ++- .../pulsar/broker/service/Dispatcher.java | 4 +- ...PersistentDispatcherMultipleConsumers.java | 23 ++++++- .../persistent/PersistentSubscription.java | 11 +++- .../service/persistent/PersistentTopic.java | 46 +++++++++++++- .../BucketDelayedDeliveryTrackerTest.java | 7 ++- .../persistent/BucketDelayedDeliveryTest.java | 59 +++++++++++++++++ 12 files changed, 217 insertions(+), 90 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java index 2f248a441cdee..3cc2da8db1e4d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java @@ -20,6 +20,7 @@ import com.google.common.annotations.Beta; import java.util.NavigableSet; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.impl.PositionImpl; /** @@ -81,8 +82,10 @@ public interface DelayedDeliveryTracker extends AutoCloseable { /** * Clear all delayed messages from the tracker. + * + * @return CompletableFuture */ - void clear(); + CompletableFuture clear(); /** * Close the subscription tracker and release all resources. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index f55d5fd11694b..8de6ee58e2ce5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -23,6 +23,7 @@ import java.time.Clock; import java.util.NavigableSet; import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -147,8 +148,9 @@ public NavigableSet getScheduledMessages(int maxMessages) { } @Override - public void clear() { + public CompletableFuture clear() { this.priorityQueue.clear(); + return CompletableFuture.completedFuture(null); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 7cfccff7ba328..5b7023be5034e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -33,6 +33,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.RoaringBitmap; @Slf4j @@ -47,6 +48,9 @@ abstract class Bucket { protected final String dispatcherName; protected final ManagedCursor cursor; + + protected final FutureUtil.Sequencer sequencer; + protected final BucketSnapshotStorage bucketSnapshotStorage; long startLedgerId; @@ -67,9 +71,10 @@ abstract class Bucket { private volatile CompletableFuture snapshotCreateFuture; - Bucket(String dispatcherName, ManagedCursor cursor, + Bucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - this(dispatcherName, cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); + this(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, + null, null); } boolean containsMessage(long ledgerId, long entryId) { @@ -154,12 +159,16 @@ CompletableFuture asyncSaveBucketSnapshot( private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { Objects.requireNonNull(bucketId); - return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), - ManagedLedgerException.BadVersionException.class, MaxRetryTimes); + return sequencer.sequential(() -> { + return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), + ManagedLedgerException.BadVersionException.class, MaxRetryTimes); + }); } protected CompletableFuture removeBucketCursorProperty(String bucketKey) { - return executeWithRetry(() -> cursor.removeCursorProperty(bucketKey), - ManagedLedgerException.BadVersionException.class, MaxRetryTimes); + return sequencer.sequential(() -> { + return executeWithRetry(() -> cursor.removeCursorProperty(bucketKey), + ManagedLedgerException.BadVersionException.class, MaxRetryTimes); + }); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 1cf8bd2b20d17..39973928abf5d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -31,6 +31,7 @@ import io.netty.util.Timeout; import io.netty.util.Timer; import java.time.Clock; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -117,19 +118,23 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); this.immutableBuckets = TreeRangeMap.create(); this.snapshotSegmentLastIndexTable = HashBasedTable.create(); - this.lastMutableBucket = new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), bucketSnapshotStorage); + this.lastMutableBucket = + new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(), + bucketSnapshotStorage); this.numberDelayedMessages = recoverBucketSnapshot(); } private synchronized long recoverBucketSnapshot() throws RuntimeException { ManagedCursor cursor = this.lastMutableBucket.getCursor(); + FutureUtil.Sequencer sequencer = this.lastMutableBucket.getSequencer(); Map, ImmutableBucket> toBeDeletedBucketMap = new HashMap<>(); cursor.getCursorProperties().keySet().forEach(key -> { if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) { String[] keys = key.split(DELIMITER); checkArgument(keys.length == 3); ImmutableBucket immutableBucket = - new ImmutableBucket(dispatcher.getName(), cursor, this.lastMutableBucket.bucketSnapshotStorage, + new ImmutableBucket(dispatcher.getName(), cursor, sequencer, + this.lastMutableBucket.bucketSnapshotStorage, Long.parseLong(keys[1]), Long.parseLong(keys[2])); putAndCleanOverlapRange(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), immutableBucket, toBeDeletedBucketMap); @@ -274,7 +279,7 @@ private void afterCreateImmutableBucket(Pair immu }); immutableBucket.setCurrentSegmentEntryId(immutableBucket.lastSegmentEntryId); - immutableBuckets.remove( + immutableBuckets.asMapOfRanges().remove( Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId)); snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getTimestamp()); @@ -442,8 +447,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableB return CompletableFuture.allOf(removeAFuture, removeBFuture); }); - immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); - immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); + Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges(); + immutableBucketMap.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); + immutableBucketMap.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); } }); }); @@ -525,14 +531,15 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa } if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) { - immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); + immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); bucket.asyncDeleteBucketSnapshot(); continue; } bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> { if (CollectionUtils.isEmpty(indexList)) { - immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); + immutableBuckets.asMapOfRanges() + .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); bucket.asyncDeleteBucketSnapshot(); return; } @@ -549,13 +556,13 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa // Back bucket state bucket.setCurrentSegmentEntryId(preSegmentEntryId); - log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}", - dispatcher.getName(), bucket.bucketKey(), ex); + log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); } else { log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId); + dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); } - }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS); + }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS); } catch (Exception e) { // Ignore exception to reload this segment on the next schedule. log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}", @@ -585,30 +592,46 @@ public boolean shouldPauseAllDeliveries() { } @Override - public synchronized void clear() { - cleanImmutableBuckets(true); - sharedBucketPriorityQueue.clear(); - lastMutableBucket.clear(); - snapshotSegmentLastIndexTable.clear(); - numberDelayedMessages = 0; + public synchronized CompletableFuture clear() { + return cleanImmutableBuckets(true).thenRun(() -> { + synchronized (this) { + sharedBucketPriorityQueue.clear(); + lastMutableBucket.clear(); + snapshotSegmentLastIndexTable.clear(); + numberDelayedMessages = 0; + } + }); } @Override public synchronized void close() { super.close(); lastMutableBucket.close(); - cleanImmutableBuckets(false); sharedBucketPriorityQueue.close(); + try { + cleanImmutableBuckets(false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); + } catch (Exception e) { + log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e); + } } - private void cleanImmutableBuckets(boolean delete) { + private CompletableFuture cleanImmutableBuckets(boolean delete) { if (immutableBuckets != null) { + List> futures = new ArrayList<>(); Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); while (iterator.hasNext()) { ImmutableBucket bucket = iterator.next(); - bucket.clear(delete); + if (delete) { + futures.add(bucket.clear()); + } else { + bucket.getSnapshotCreateFuture().ifPresent(future -> futures.add(future.thenApply(x -> null))); + } + numberDelayedMessages -= bucket.getNumberBucketDelayedMessages(); iterator.remove(); } + return FutureUtil.waitForAll(futures); + } else { + return CompletableFuture.completedFuture(null); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index ab1c285011d20..969d326e28187 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.delayed.bucket; import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; -import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds; import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE; import com.google.protobuf.ByteString; import java.util.ArrayList; @@ -28,7 +27,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -38,6 +36,7 @@ import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; @@ -52,9 +51,9 @@ class ImmutableBucket extends Bucket { @Setter List firstScheduleTimestamps = new ArrayList<>(); - ImmutableBucket(String dispatcherName, ManagedCursor cursor, + ImmutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - super(dispatcherName, cursor, storage, startLedgerId, endLedgerId); + super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId); } public Optional> getSnapshotSegments() { @@ -117,8 +116,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b () -> bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId).whenComplete((___, ex) -> { if (ex != null) { - log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {}, bucketId: {}", - dispatcherName, bucketKey(), bucketId, ex); + log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {}," + + " bucketId: {}, segmentEntryId: {}", dispatcherName, bucketKey(), + bucketId, nextSegmentEntryId, ex); } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) .thenApply(bucketSnapshotSegments -> { @@ -191,48 +191,9 @@ CompletableFuture asyncDeleteBucketSnapshot() { }); } - void clear(boolean delete) { + CompletableFuture clear() { delayedIndexBitMap.clear(); - if (delete) { - final String bucketKey = bucketKey(); - try { - getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null).thenCompose(__ -> { - if (getSnapshotCreateFuture().isPresent() && getBucketId().isEmpty()) { - log.error("[{}] Can't found bucketId, don't execute delete operate, bucketKey: {}", - dispatcherName, bucketKey); - return CompletableFuture.completedFuture(null); - } - long bucketId = getAndUpdateBucketId(); - return removeBucketCursorProperty(bucketKey()).thenAccept(___ -> { - executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId), - BucketSnapshotPersistenceException.class, MaxRetryTimes) - .whenComplete((____, ex) -> { - if (ex != null) { - log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey, ex); - } else { - log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey); - } - }); - }); - }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (Exception e) { - log.error("Failed to clear bucket snapshot, bucketKey: {}", bucketKey, e); - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException(e); - } - } else { - getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> { - try { - snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (Exception e) { - log.warn("[{}] Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", dispatcherName, - getBucketId(), bucketKey()); - } - }); - } + return getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null) + .thenCompose(__ -> asyncDeleteBucketSnapshot()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index 1577bf8fa51d2..f8a4ecc7a4ddb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -34,6 +34,7 @@ import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.roaringbitmap.RoaringBitmap; @@ -42,9 +43,9 @@ class MutableBucket extends Bucket implements AutoCloseable { private final TripleLongPriorityQueue priorityQueue; - MutableBucket(String dispatcherName, ManagedCursor cursor, + MutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer, BucketSnapshotStorage bucketSnapshotStorage) { - super(dispatcherName, cursor, bucketSnapshotStorage, -1L, -1L); + super(dispatcherName, cursor, sequencer, bucketSnapshotStorage, -1L, -1L); this.priorityQueue = new TripleLongPriorityQueue(); } @@ -134,7 +135,7 @@ Pair createImmutableBucketAndAsyncPersistent( final int lastSegmentEntryId = segmentMetadataList.size(); - ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, bucketSnapshotStorage, + ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, sequencer, bucketSnapshotStorage, startLedgerId, endLedgerId); bucket.setCurrentSegmentEntryId(1); bucket.setNumberBucketDelayedMessages(numMessages); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java index 48877ce53f801..9b0c4e885e64d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java @@ -110,8 +110,8 @@ default long getNumberOfDelayedMessages() { return 0; } - default void clearDelayedMessages() { - //No-op + default CompletableFuture clearDelayedMessages() { + return CompletableFuture.completedFuture(null); } default void cursorIsReset() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 059820b1b66c3..e5c9e85bac3f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -48,6 +48,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; @@ -1089,8 +1090,26 @@ public synchronized long getNumberOfDelayedMessages() { } @Override - public void clearDelayedMessages() { - this.delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::clear); + public CompletableFuture clearDelayedMessages() { + if (!topic.isDelayedDeliveryEnabled()) { + return CompletableFuture.completedFuture(null); + } + + if (delayedDeliveryTracker.isEmpty() && topic.getBrokerService() + .getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory) { + synchronized (this) { + if (delayedDeliveryTracker.isEmpty()) { + delayedDeliveryTracker = Optional + .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this)); + } + } + } + + if (delayedDeliveryTracker.isPresent()) { + return this.delayedDeliveryTracker.get().clear(); + } else { + return CompletableFuture.completedFuture(null); + } } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index e07d7bee500a7..bdb3c9fc391ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -649,9 +649,16 @@ public void clearBacklogComplete(Object ctx) { cursor.getNumberOfEntriesInBacklog(false)); } if (dispatcher != null) { - dispatcher.clearDelayedMessages(); + dispatcher.clearDelayedMessages().whenComplete((__, ex) -> { + if (ex != null) { + future.completeExceptionally(ex); + } else { + future.complete(null); + } + }); + } else { + future.complete(null); } - future.complete(null); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index b3b6526eea54d..82a4f5312357a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -79,6 +79,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -1078,13 +1079,13 @@ public CompletableFuture unsubscribe(String subscriptionName) { new AsyncCallbacks.DeleteLedgerCallback() { @Override public void deleteLedgerComplete(Object ctx) { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture); } @Override public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { if (exception instanceof MetadataNotFoundException) { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture); return; } @@ -1094,12 +1095,51 @@ public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { } }, null); } else { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture); } return unsubscribeFuture; } + private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, + CompletableFuture unsubscribeFuture) { + if (!isDelayedDeliveryEnabled() + || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + return; + } + + PersistentSubscription persistentSubscription = subscriptions.get(subscriptionName); + if (persistentSubscription == null) { + log.warn("[{}][{}] Can't find subscription, skip clear delayed message", topic, subscriptionName); + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + return; + } + + Dispatcher dispatcher = persistentSubscription.getDispatcher(); + final Dispatcher temporaryDispatcher; + if (dispatcher == null) { + log.info("[{}][{}] Dispatcher is null, try to create temporary dispatcher to clear delayed message", topic, + subscriptionName); + dispatcher = temporaryDispatcher = + new PersistentDispatcherMultipleConsumers(this, persistentSubscription.cursor, + persistentSubscription); + } else { + temporaryDispatcher = null; + } + + dispatcher.clearDelayedMessages().whenComplete((__, ex) -> { + if (ex != null) { + unsubscribeFuture.completeExceptionally(ex); + } else { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + } + if (temporaryDispatcher != null) { + temporaryDispatcher.close(); + } + }); + } + private void asyncDeleteCursor(String subscriptionName, CompletableFuture unsubscribeFuture) { ledger.asyncDeleteCursor(Codec.encode(subscriptionName), new DeleteCursorCallback() { @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 0d53c278fd2f6..fbb866d48ed69 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -39,7 +39,9 @@ import java.util.NavigableSet; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -424,7 +426,8 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) { } @Test(dataProvider = "delayedTracker") - public void testClear(BucketDelayedDeliveryTracker tracker) { + public void testClear(BucketDelayedDeliveryTracker tracker) + throws ExecutionException, InterruptedException, TimeoutException { for (int i = 1; i <= 1001; i++) { tracker.addMessage(i, i, i * 10); } @@ -433,7 +436,7 @@ public void testClear(BucketDelayedDeliveryTracker tracker) { assertTrue(tracker.getImmutableBuckets().asMapOfRanges().size() > 0); assertEquals(tracker.getLastMutableBucket().size(), 1); - tracker.clear(); + tracker.clear().get(1, TimeUnit.MINUTES); assertEquals(tracker.getNumberOfDelayedMessages(), 0); assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index 292889e8c159a..09b7cbbf1b99d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -18,9 +18,15 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.testng.Assert.assertTrue; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; @@ -103,4 +109,57 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher2.getNumberOfDelayedMessages(), 1000)); Assert.assertEquals(bucketKeys, bucketKeys2); } + + + @Test + public void testUnsubscribe() throws Exception { + String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testUnsubscribes"); + + @Cleanup + Consumer c1 = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + for (int i = 0; i < 1000; i++) { + producer.newMessage() + .value("msg") + .deliverAfter(1, TimeUnit.HOURS) + .send(); + } + + Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher(); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000)); + + Map cursorProperties = + ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties(); + List bucketIds = cursorProperties.entrySet().stream() + .filter(x -> x.getKey().startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket")).map( + x -> Long.valueOf(x.getValue())).toList(); + + assertTrue(bucketIds.size() > 0); + + c1.close(); + + restartBroker(); + + admin.topics().deleteSubscription(topic, "sub"); + + for (Long bucketId : bucketIds) { + try { + LedgerHandle ledgerHandle = + pulsarTestContext.getBookKeeperClient() + .openLedger(bucketId, BookKeeper.DigestType.CRC32C, new byte[]{}); + Assert.fail("Should fail"); + } catch (BKException.BKNoSuchLedgerExistsException e) { + // ignore it + } + } + } } From 2b4a3c14458f564e7e0178f71979a41e7b7a42b7 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Sun, 26 Mar 2023 22:36:34 +0800 Subject: [PATCH 027/280] [fix][broker] Fix RetentionPolicies constructor (#19777) Signed-off-by: Zixuan Liu --- .../apache/bookkeeper/mledger/ManagedLedgerConfig.java | 8 ++++---- .../pulsar/broker/service/ConsumedLedgersTrimTest.java | 2 +- .../pulsar/common/policies/data/RetentionPolicies.java | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 6e88a8e650d58..0c93a5b642cf6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -62,7 +62,7 @@ public class ManagedLedgerConfig { private int ledgerRolloverTimeout = 4 * 3600; private double throttleMarkDelete = 0; private long retentionTimeMs = 0; - private int retentionSizeInMB = 0; + private long retentionSizeInMB = 0; private boolean autoSkipNonRecoverableData; private boolean lazyCursorRecovery = false; private long metadataOperationsTimeoutSeconds = 60; @@ -396,7 +396,7 @@ public ManagedLedgerConfig setThrottleMarkDelete(double throttleMarkDelete) { /** * Set the retention time for the ManagedLedger. *

- * Retention time and retention size ({@link #setRetentionSizeInMB(int)}) are together used to retain the + * Retention time and retention size ({@link #setRetentionSizeInMB(long)}) are together used to retain the * ledger data when there are no cursors or when all the cursors have marked the data for deletion. * Data will be deleted in this case when both retention time and retention size settings don't prevent deleting * the data marked for deletion. @@ -438,7 +438,7 @@ public long getRetentionTimeMillis() { * @param retentionSizeInMB * quota for message retention */ - public ManagedLedgerConfig setRetentionSizeInMB(int retentionSizeInMB) { + public ManagedLedgerConfig setRetentionSizeInMB(long retentionSizeInMB) { this.retentionSizeInMB = retentionSizeInMB; return this; } @@ -447,7 +447,7 @@ public ManagedLedgerConfig setRetentionSizeInMB(int retentionSizeInMB) { * @return quota for message retention * */ - public int getRetentionSizeInMB() { + public long getRetentionSizeInMB() { return retentionSizeInMB; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java index 099a9028c4645..80db4c30f454d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java @@ -86,7 +86,7 @@ public void TestConsumedLedgersTrim() throws Exception { PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); ManagedLedgerConfig managedLedgerConfig = persistentTopic.getManagedLedger().getConfig(); - managedLedgerConfig.setRetentionSizeInMB(1); + managedLedgerConfig.setRetentionSizeInMB(1L); managedLedgerConfig.setRetentionTime(1, TimeUnit.SECONDS); managedLedgerConfig.setMaxEntriesPerLedger(2); managedLedgerConfig.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java index 4206220c5ee58..8d5b25da43153 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java @@ -29,13 +29,13 @@ */ public class RetentionPolicies { private int retentionTimeInMinutes; - private int retentionSizeInMB; + private long retentionSizeInMB; public RetentionPolicies() { this(0, 0); } - public RetentionPolicies(int retentionTimeInMinutes, int retentionSizeInMB) { + public RetentionPolicies(int retentionTimeInMinutes, long retentionSizeInMB) { this.retentionSizeInMB = retentionSizeInMB; this.retentionTimeInMinutes = retentionTimeInMinutes; } @@ -44,7 +44,7 @@ public int getRetentionTimeInMinutes() { return retentionTimeInMinutes; } - public int getRetentionSizeInMB() { + public long getRetentionSizeInMB() { return retentionSizeInMB; } From ef18bab1badbe6ce537254b3ff8fd288da1e7d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tu=E1=BA=A5n=20V=C6=B0=C6=A1ng?= Date: Mon, 27 Mar 2023 11:59:50 +0700 Subject: [PATCH 028/280] [improve] Allow download link with basic auth (#19750) Co-authored-by: tison --- .../impl/auth/AuthenticationDataBasic.java | 20 ++++++----- .../functions/utils/FunctionCommon.java | 14 ++++++-- .../functions/utils/FunctionCommonTest.java | 35 ++++++++++++------- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java index 2fc89e128ec58..82de6dc198d77 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java @@ -27,16 +27,20 @@ public class AuthenticationDataBasic implements AuthenticationDataProvider { private static final String HTTP_HEADER_NAME = "Authorization"; - private String httpAuthToken; - private String commandAuthToken; - private Map headers = new HashMap<>(); + private final String commandAuthToken; + private final Map headers; public AuthenticationDataBasic(String userId, String password) { - httpAuthToken = "Basic " + Base64.getEncoder().encodeToString((userId + ":" + password).getBytes()); - commandAuthToken = userId + ":" + password; - headers.put(HTTP_HEADER_NAME, httpAuthToken); - headers.put(PULSAR_AUTH_METHOD_NAME, AuthenticationBasic.AUTH_METHOD_NAME); - this.headers = Collections.unmodifiableMap(this.headers); + this(userId + ":" + password); + } + + public AuthenticationDataBasic(String userInfo) { + String httpAuthToken = "Basic " + Base64.getEncoder().encodeToString(userInfo.getBytes()); + this.commandAuthToken = userInfo; + this.headers = Collections.unmodifiableMap(new HashMap(){{ + put(HTTP_HEADER_NAME, httpAuthToken); + put(PULSAR_AUTH_METHOD_NAME, AuthenticationBasic.AUTH_METHOD_NAME); + }}); } @Override diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index d3ce9d93a2d36..28cce0fe62209 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -35,9 +35,11 @@ import java.net.ServerSocket; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.Collection; +import java.util.Map; import java.util.UUID; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -48,6 +50,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.nar.NarClassLoader; @@ -245,8 +248,15 @@ public static Class getSinkType(Class sinkClass) { } public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throws IOException { - URL website = new URL(destPkgUrl); - try (InputStream in = website.openStream()) { + final URL url = new URL(destPkgUrl); + final URLConnection connection = url.openConnection(); + if (StringUtils.isNotEmpty(url.getUserInfo())) { + final AuthenticationDataBasic authBasic = new AuthenticationDataBasic(url.getUserInfo()); + for (Map.Entry header : authBasic.getHttpHeaders()) { + connection.setRequestProperty(header.getKey(), header.getValue()); + } + } + try (InputStream in = connection.getInputStream()) { log.info("Downloading function package from {} to {} ...", destPkgUrl, targetFile.getAbsoluteFile()); Files.copy(in, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java index aae2292752005..113824fc7c1a1 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java @@ -18,6 +18,11 @@ */ package org.apache.pulsar.functions.utils; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import java.io.File; import java.util.Collection; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.util.FutureUtil; @@ -26,17 +31,11 @@ import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.WindowContext; import org.apache.pulsar.functions.api.WindowFunction; +import org.assertj.core.util.Files; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.io.File; -import java.util.UUID; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; - /** * Unit test of {@link Exceptions}. */ @@ -78,12 +77,22 @@ public void testValidateHttpFileUrl() throws Exception { @Test public void testDownloadFile() throws Exception { - String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; - String testDir = FunctionCommonTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - FunctionCommon.downloadFromHttpUrl(jarHttpUrl, pkgFile); - Assert.assertTrue(pkgFile.exists()); - pkgFile.delete(); + final String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; + final File file = Files.newTemporaryFile(); + file.deleteOnExit(); + assertThat(file.length()).isZero(); + FunctionCommon.downloadFromHttpUrl(jarHttpUrl, file); + assertThat(file.length()).isGreaterThan(0); + } + + @Test + public void testDownloadFileWithBasicAuth() throws Exception { + final String jarHttpUrl = "https://foo:bar@httpbin.org/basic-auth/foo/bar"; + final File file = Files.newTemporaryFile(); + file.deleteOnExit(); + assertThat(file.length()).isZero(); + FunctionCommon.downloadFromHttpUrl(jarHttpUrl, file); + assertThat(file.length()).isGreaterThan(0); } @Test From aac99c86ea39ad673bce32240056fe308f546bef Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 27 Mar 2023 14:24:18 +0800 Subject: [PATCH 029/280] [fix][client] Fix NPE when acknowledging multiple messages (#19874) --- .../client/api/ConsumerAckListTest.java | 39 +++++++++++++++++++ .../client/impl/MultiTopicsConsumerImpl.java | 13 ++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java index d01e5b764a7a1..baf0000be6e4f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java @@ -111,4 +111,43 @@ private void sendMessagesAsyncAndWait(Producer producer, int messages) t latch.await(); } + @Test(timeOut = 30000) + public void testAckMessageInAnotherTopic() throws Exception { + final String[] topics = { + "persistent://my-property/my-ns/test-ack-message-in-other-topic1" + UUID.randomUUID(), + "persistent://my-property/my-ns/test-ack-message-in-other-topic2" + UUID.randomUUID(), + "persistent://my-property/my-ns/test-ack-message-in-other-topic3" + UUID.randomUUID() + }; + @Cleanup final Consumer allTopicsConsumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topics) + .subscriptionName("sub1") + .subscribe(); + Consumer partialTopicsConsumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topics[0], topics[1]) + .subscriptionName("sub2") + .subscribe(); + for (int i = 0; i < topics.length; i++) { + final Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topics[i]) + .create(); + producer.send("msg-" + i); + producer.close(); + } + final List messageIdList = new ArrayList<>(); + for (int i = 0; i < topics.length; i++) { + messageIdList.add(allTopicsConsumer.receive().getMessageId()); + } + try { + partialTopicsConsumer.acknowledge(messageIdList); + Assert.fail(); + } catch (PulsarClientException.NotConnectedException ignored) { + } + partialTopicsConsumer.close(); + partialTopicsConsumer = pulsarClient.newConsumer(Schema.STRING).topic(topics[0]) + .subscriptionName("sub2").subscribe(); + pulsarClient.newProducer(Schema.STRING).topic(topics[0]).create().send("done"); + final Message msg = partialTopicsConsumer.receive(); + Assert.assertEquals(msg.getValue(), "msg-0"); + partialTopicsConsumer.close(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 341a91e97348e..f993304b0780a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -32,6 +32,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -494,8 +495,16 @@ protected CompletableFuture doAcknowledge(List messageIdList, topicToMessageIdMap.get(topicMessageId.getOwnerTopic()) .add(MessageIdImpl.convertToMessageIdImpl(topicMessageId)); } - topicToMessageIdMap.forEach((topicPartitionName, messageIds) -> { - ConsumerImpl consumer = consumers.get(topicPartitionName); + final Map, List> consumerToMessageIds = new IdentityHashMap<>(); + for (Map.Entry> entry : topicToMessageIdMap.entrySet()) { + ConsumerImpl consumer = consumers.get(entry.getKey()); + if (consumer == null) { + return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException()); + } + // Trigger the acknowledgment later to avoid sending partial acknowledgments + consumerToMessageIds.put(consumer, entry.getValue()); + } + consumerToMessageIds.forEach((consumer, messageIds) -> { resultFutures.add(consumer.doAcknowledgeWithTxn(messageIds, ackType, properties, txn) .thenAccept((res) -> messageIdList.forEach(unAckedMessageTracker::remove))); }); From 19c84977f00033ef9601f43f1b703068a6207536 Mon Sep 17 00:00:00 2001 From: LinChen Date: Mon, 27 Mar 2023 15:18:01 +0800 Subject: [PATCH 030/280] [fix][broker] Fix the thread safety issue of BrokerData#getTimeAverageData access (#19889) Co-authored-by: lordcheng10 --- .../broker/loadbalance/impl/ModularLoadManagerImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index f135840d60e59..4756b885ff217 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -589,7 +589,9 @@ private void updateBundleData() { } // Using the newest data, update the aggregated time-average data for the current broker. - brokerData.getTimeAverageData().reset(statsMap.keySet(), bundleData, defaultStats); + TimeAverageBrokerData timeAverageData = new TimeAverageBrokerData(); + timeAverageData.reset(statsMap.keySet(), bundleData, defaultStats); + brokerData.setTimeAverageData(timeAverageData); final ConcurrentOpenHashMap> namespaceToBundleRange = brokerToNamespaceToBundleRange .computeIfAbsent(broker, k -> From cda2827acbe62ba6c5a4dd98dbee7c9a82548339 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 27 Mar 2023 12:32:54 +0100 Subject: [PATCH 031/280] [fix][fn] Revert change to deprecation since it broke the master branch (#19904) --- pulsar-function-go/pb/Function.pb.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-function-go/pb/Function.pb.go b/pulsar-function-go/pb/Function.pb.go index bad1afe6658da..3e4d49ab99022 100644 --- a/pulsar-function-go/pb/Function.pb.go +++ b/pulsar-function-go/pb/Function.pb.go @@ -579,7 +579,6 @@ type FunctionDetails struct { Runtime FunctionDetails_Runtime `protobuf:"varint,8,opt,name=runtime,proto3,enum=proto.FunctionDetails_Runtime" json:"runtime,omitempty"` // Deprecated since, see https://github.com/apache/pulsar/issues/15560 // - // Deprecated: Do not use. AutoAck bool `protobuf:"varint,9,opt,name=autoAck,proto3" json:"autoAck,omitempty"` Parallelism int32 `protobuf:"varint,10,opt,name=parallelism,proto3" json:"parallelism,omitempty"` Source *SourceSpec `protobuf:"bytes,11,opt,name=source,proto3" json:"source,omitempty"` From 32ad90606062c1eda660037e6277253b35d4a1e6 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 27 Mar 2023 22:16:25 +0800 Subject: [PATCH 032/280] [fix][admin] Delete tenant local policy only if exist (#19925) Signed-off-by: nodece --- .../resources/LocalPoliciesResources.java | 2 +- .../broker/admin/AdminApiTenantTest.java | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java index 8ca0c121ef1b9..c6b658c3bd025 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java @@ -85,7 +85,7 @@ public CompletableFuture deleteLocalPoliciesAsync(NamespaceName ns) { public CompletableFuture deleteLocalPoliciesTenantAsync(String tenant) { final String localPoliciesPath = joinPath(LOCAL_POLICIES_ROOT, tenant); CompletableFuture future = new CompletableFuture(); - deleteAsync(localPoliciesPath).whenComplete((ignore, ex) -> { + deleteIfExistsAsync(localPoliciesPath).whenComplete((ignore, ex) -> { if (ex != null && ex.getCause().getCause() instanceof KeeperException) { future.complete(null); } else if (ex != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java new file mode 100644 index 0000000000000..f883417614229 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +@Slf4j +public class AdminApiTenantTest extends MockedPulsarServiceBaseTest { + private final String CLUSTER = "test"; + + @BeforeClass + @Override + public void setup() throws Exception { + super.internalSetup(); + admin.clusters() + .createCluster(CLUSTER, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + } + + @BeforeClass(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testListTenant() throws PulsarAdminException { + admin.tenants().getTenants(); + } + + @Test + public void testCreateAndDeleteTenant() throws PulsarAdminException { + String tenant = "test-tenant-"+ UUID.randomUUID(); + admin.tenants().createTenant(tenant, TenantInfo.builder().allowedClusters(Collections.singleton(CLUSTER)).build()); + List tenants = admin.tenants().getTenants(); + assertTrue(tenants.contains(tenant)); + admin.tenants().deleteTenant(tenant); + tenants = admin.tenants().getTenants(); + assertFalse(tenants.contains(tenant)); + } + + @Test + public void testDeleteNonExistTenant() { + String tenant = "test-non-exist-tenant-" + UUID.randomUUID(); + assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.tenants().deleteTenant(tenant)); + } +} From d14e43ed9bd079b76301f10dc1c039cafd43e1cf Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 28 Mar 2023 11:22:22 +0800 Subject: [PATCH 033/280] [improve][client] Exclude log4j-slf4j-impl from compile dep in pulsar-client-all (#19937) Signed-off-by: tison --- pulsar-client-all/pom.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index b06c71616b6a5..00da6e4895097 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -54,11 +54,25 @@ ${project.parent.version} true + + org.apache.logging.log4j + log4j-api + test + + + org.apache.logging.log4j + log4j-core + test + + + org.apache.logging.log4j + log4j-slf4j-impl + test + - org.apache.maven.plugins maven-dependency-plugin From 7a99e74220bdff7a0dff3dbbf3befb2316f3a6c8 Mon Sep 17 00:00:00 2001 From: SeasonPan <244014926@qq.com> Date: Tue, 28 Mar 2023 17:58:58 +0800 Subject: [PATCH 034/280] [fix][doc] fix typo in ConsumerConfigurationData and ServiceConfiguration. (#19936) --- .../java/org/apache/pulsar/broker/ServiceConfiguration.java | 2 +- .../pulsar/client/impl/conf/ConsumerConfigurationData.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index ba82667690dc5..e2bcf03353b59 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1976,7 +1976,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, category = CATEGORY_STORAGE_ML, dynamic = true, doc = "The number of partitioned topics that is allowed to be automatically created" - + "if allowAutoTopicCreationType is partitioned." + + " if allowAutoTopicCreationType is partitioned." ) private int defaultNumPartitions = 1; @FieldContext( diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java index 373d4e66c0ecf..8760926792cd7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java @@ -99,7 +99,7 @@ public class ConsumerConfigurationData implements Serializable, Cloneable { @ApiModelProperty( name = "negativeAckRedeliveryBackoff", value = "Interface for custom message is negativeAcked policy. You can specify `RedeliveryBackoff` for a" - + "consumer." + + " consumer." ) @JsonIgnore private RedeliveryBackoff negativeAckRedeliveryBackoff; @@ -235,7 +235,7 @@ public int getMaxPendingChuckedMessage() { @ApiModelProperty( name = "autoAckOldestChunkedMessageOnQueueFull", - value = "Whether to automatically acknowledge pending chunked messages when the threashold of" + value = "Whether to automatically acknowledge pending chunked messages when the threshold of" + " `maxPendingChunkedMessage` is reached. If set to `false`, these messages will be redelivered" + " by their broker." ) From 83f6ea74e7a47927402d8f1de4200c4ad02e6147 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 28 Mar 2023 19:07:43 +0100 Subject: [PATCH 035/280] [fix][build] Dump Jacoco coverage data to file with JMX interface in TestNG listener (#19947) --- buildtools/pom.xml | 2 +- .../pulsar/tests/JacocoDumpListener.java | 103 ++++++++++++++++++ pom.xml | 3 +- 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java diff --git a/buildtools/pom.xml b/buildtools/pom.xml index de52ac0930a33..b22abf286f693 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -176,7 +176,7 @@ listener - org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier + org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java new file mode 100644 index 0000000000000..2c49d5118ae52 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import java.lang.management.ManagementFactory; +import java.util.concurrent.TimeUnit; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanServer; +import javax.management.MBeanServerInvocationHandler; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import org.testng.IExecutionListener; +import org.testng.ISuite; +import org.testng.ISuiteListener; + +/** + * A TestNG listener that dumps Jacoco coverage data to file using the Jacoco JMX interface. + * + * This ensures that coverage data is dumped even if the shutdown sequence of the Test JVM gets stuck. Coverage + * data will be dumped every 2 minutes by default and once all test suites have been run. + * Each test class runs in its own suite when run with maven-surefire-plugin. + */ +public class JacocoDumpListener implements ISuiteListener, IExecutionListener { + private final MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer(); + private final ObjectName jacocoObjectName; + private final JacocoProxy jacocoProxy; + private final boolean enabled; + + private long lastDumpTime; + + private static final long DUMP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(120); + + public JacocoDumpListener() { + try { + jacocoObjectName = new ObjectName("org.jacoco:type=Runtime"); + } catch (MalformedObjectNameException e) { + // this won't happen since the ObjectName is static and valid + throw new RuntimeException(e); + } + enabled = checkEnabled(); + if (enabled) { + jacocoProxy = MBeanServerInvocationHandler.newProxyInstance(platformMBeanServer, jacocoObjectName, + JacocoProxy.class, false); + } else { + jacocoProxy = null; + } + lastDumpTime = System.currentTimeMillis(); + } + + private boolean checkEnabled() { + try { + platformMBeanServer.getObjectInstance(jacocoObjectName); + } catch (InstanceNotFoundException e) { + // jacoco jmx is not enabled + return false; + } + return true; + } + + public void onFinish(ISuite suite) { + // dump jacoco coverage data to file using the Jacoco JMX interface if more than DUMP_INTERVAL_MILLIS has passed + // since the last dump + if (enabled && System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS) { + // dump jacoco coverage data to file using the Jacoco JMX interface + triggerJacocoDump(); + } + } + @Override + public void onExecutionFinish() { + if (enabled) { + // dump jacoco coverage data to file using the Jacoco JMX interface when all tests have finished + triggerJacocoDump(); + } + } + + private void triggerJacocoDump() { + System.out.println("Dumping Jacoco coverage data to file..."); + long start = System.currentTimeMillis(); + jacocoProxy.dump(true); + lastDumpTime = System.currentTimeMillis(); + System.out.println("Completed in " + (lastDumpTime - start) + "ms."); + } + + public interface JacocoProxy { + void dump(boolean reset); + } +} diff --git a/pom.xml b/pom.xml index 3f33069b8c252..ac640f1a6a05f 100644 --- a/pom.xml +++ b/pom.xml @@ -1531,7 +1531,7 @@ flexible messaging model and an intuitive client API. listener - org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener + org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener @@ -1995,6 +1995,7 @@ flexible messaging model and an intuitive client API. ${project.build.directory}/jacoco_${maven.build.timestamp}_${surefire.forkNumber}.exec true + true org.apache.pulsar.* org.apache.bookkeeper.mledger.* From 9fc0b5e197aa36ff64087cd13d156bb65a097fad Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 29 Mar 2023 09:24:12 +0800 Subject: [PATCH 036/280] [fix] [admin] fix incorrect state replication.connected on API partitioned-topic stat (#19942) ### Motivation Pulsar will merge the variable `PartitionedTopicStatsImpl.replication[x].connected` by the way below when we call `pulsar-admin topics partitioned-stats` ``` java this.connected = this.connected & other.connected ``` But the variable `connected` of `PartitionedTopicStatsImpl.replication` is initialized `false`, so the expression `this.connected & other.connected` will always be `false`. Then we will always get the value `false` if we call `pulsar-admin topics partitioned-stats`. ### Modifications make the variable `` of `PartitionedTopicStatsImpl` is initialized `true` --- .../pulsar/broker/service/ReplicatorTest.java | 25 +++++++++++++++++++ .../data/stats/ReplicatorStatsImpl.java | 4 ++- .../policies/data/stats/TopicStatsImpl.java | 2 ++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 4086c54a0ba4b..901451c022bfe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -97,8 +97,10 @@ import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BacklogQuota.RetentionPolicy; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.PartitionedTopicStats; import org.apache.pulsar.common.policies.data.ReplicatorStats; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.schema.Schemas; @@ -870,6 +872,29 @@ public void testReplicatorProducerClosing() throws Exception { assertNull(producer); } + @Test(priority = 5, timeOut = 30000) + public void testReplicatorConnected() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID()); + final TopicName dest = TopicName.get(topicName); + admin1.topics().createPartitionedTopic(topicName, 1); + + @Cleanup + MessageProducer producer1 = new MessageProducer(url1, dest); + + Awaitility.await().until(() -> { + TopicStats topicStats = admin1.topics().getStats(topicName + "-partition-0"); + return topicStats.getReplication().values().stream() + .map(ReplicatorStats::isConnected).reduce((a, b) -> a & b).get(); + }); + + PartitionedTopicStats + partitionedTopicStats = admin1.topics().getPartitionedStats(topicName, true); + + for (ReplicatorStats replicatorStats : partitionedTopicStats.getReplication().values()){ + assertTrue(replicatorStats.isConnected()); + } + } + @Test public void testDeleteTopicFailure() throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID()); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java index 71196e4978682..6933f5cc7ed76 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java @@ -72,7 +72,9 @@ public ReplicatorStatsImpl add(ReplicatorStatsImpl stats) { this.msgThroughputOut += stats.msgThroughputOut; this.msgRateExpired += stats.msgRateExpired; this.replicationBacklog += stats.replicationBacklog; - this.connected &= stats.connected; + if (this.connected) { + this.connected &= stats.connected; + } this.replicationDelayInSeconds = Math.max(this.replicationDelayInSeconds, stats.replicationDelayInSeconds); return this; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java index 238170952f08e..12d30124f7dcd 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java @@ -287,6 +287,7 @@ public TopicStatsImpl add(TopicStats ts) { if (this.replication.size() != stats.replication.size()) { for (String repl : stats.replication.keySet()) { ReplicatorStatsImpl replStats = new ReplicatorStatsImpl(); + replStats.setConnected(true); this.replication.put(repl, replStats.add(stats.replication.get(repl))); } } else { @@ -295,6 +296,7 @@ public TopicStatsImpl add(TopicStats ts) { this.replication.get(repl).add(stats.replication.get(repl)); } else { ReplicatorStatsImpl replStats = new ReplicatorStatsImpl(); + replStats.setConnected(true); this.replication.put(repl, replStats.add(stats.replication.get(repl))); } } From eedf7026ae60f39bcf74ce67728b47d966fe237f Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 29 Mar 2023 10:23:10 +0800 Subject: [PATCH 037/280] [improve][broker] PIP-192: Support delete namespace bundle admin API (#19851) PIP: https://github.com/apache/pulsar/issues/16691 ### Motivation Raising a PR to implement https://github.com/apache/pulsar/issues/16691. We need to support delete namespace bundle admin API. ### Modifications * Support delete namespace bundle admin API. * Add units test. --- .../broker/admin/impl/NamespacesBase.java | 12 ++--- .../extensions/ExtensibleLoadManagerImpl.java | 11 +++- .../extensions/data/BrokerLookupData.java | 6 +++ .../broker/namespace/NamespaceService.java | 47 ++++++++++++++--- .../ExtensibleLoadManagerImplTest.java | 50 +++++++++++++++++++ 5 files changed, 111 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index cffc94b189225..19f8d7c437ded 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -297,11 +297,11 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime .thenCompose(ignore -> pulsar().getNamespaceService() .getNamespaceBundleFactory().getBundlesAsync(namespaceName)) .thenCompose(bundles -> FutureUtil.waitForAll(bundles.getBundles().stream() - .map(bundle -> pulsar().getNamespaceService().getOwnerAsync(bundle) - .thenCompose(owner -> { + .map(bundle -> pulsar().getNamespaceService().checkOwnershipPresentAsync(bundle) + .thenCompose(present -> { // check if the bundle is owned by any broker, // if not then we do not need to delete the bundle - if (owner.isPresent()) { + if (present) { PulsarAdmin admin; try { admin = pulsar().getAdminClient(); @@ -1411,7 +1411,7 @@ protected void internalClearNamespaceBacklog(AsyncResponse asyncResponse, boolea .getBundles(namespaceName); for (NamespaceBundle nsBundle : bundles.getBundles()) { // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear - if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) { + if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) { futures.add(pulsar().getAdminClient().namespaces() .clearNamespaceBundleBacklogAsync(namespaceName.toString(), nsBundle.getBundleRange())); } @@ -1476,7 +1476,7 @@ protected void internalClearNamespaceBacklogForSubscription(AsyncResponse asyncR .getBundles(namespaceName); for (NamespaceBundle nsBundle : bundles.getBundles()) { // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear - if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) { + if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) { futures.add(pulsar().getAdminClient().namespaces().clearNamespaceBundleBacklogForSubscriptionAsync( namespaceName.toString(), nsBundle.getBundleRange(), subscription)); } @@ -1543,7 +1543,7 @@ protected void internalUnsubscribeNamespace(AsyncResponse asyncResponse, String .getBundles(namespaceName); for (NamespaceBundle nsBundle : bundles.getBundles()) { // check if the bundle is owned by any broker, if not then there are no subscriptions - if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) { + if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) { futures.add(pulsar().getAdminClient().namespaces().unsubscribeNamespaceBundleAsync( namespaceName.toString(), nsBundle.getBundleRange(), subscription)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index fedcad4d009b2..7aefef596e7cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -383,7 +383,7 @@ public CompletableFuture checkOwnershipAsync(Optional to .thenApply(broker -> brokerRegistry.getBrokerId().equals(broker.orElse(null))); } - private CompletableFuture> getOwnershipAsync(Optional topic, + public CompletableFuture> getOwnershipAsync(Optional topic, ServiceUnitId bundleUnit) { final String bundle = bundleUnit.toString(); CompletableFuture> owner; @@ -395,6 +395,15 @@ private CompletableFuture> getOwnershipAsync(Optional> getOwnershipWithLookupDataAsync(ServiceUnitId bundleUnit) { + return getOwnershipAsync(Optional.empty(), bundleUnit).thenCompose(broker -> { + if (broker.isEmpty()) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return getBrokerRegistry().lookupAsync(broker.get()); + }); + } + public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, Optional destinationBroker) { return getOwnershipAsync(Optional.empty(), bundle) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java index 504ae13003e04..4c9e503129e66 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Optional; import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; @@ -70,4 +71,9 @@ public LookupResult toLookupResult() { return new LookupResult(webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, LookupResult.Type.BrokerUrl, false); } + + public NamespaceEphemeralData toNamespaceEphemeralData() { + return new NamespaceEphemeralData(pulsarServiceUrl, pulsarServiceUrlTls, webServiceUrl, webServiceUrlTls, + false, advertisedListeners); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index de18d50f3e144..d092ef04018c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1165,8 +1165,14 @@ public CompletableFuture checkTopicOwnership(TopicName topicName) { } public CompletableFuture removeOwnedServiceUnitAsync(NamespaceBundle nsBundle) { - return ownershipCache.removeOwnership(nsBundle) - .thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject())); + CompletableFuture future; + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); + future = extensibleLoadManager.unloadNamespaceBundleAsync(nsBundle, Optional.empty()); + } else { + future = ownershipCache.removeOwnership(nsBundle); + } + return future.thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject())); } protected void onNamespaceBundleOwned(NamespaceBundle bundle) { @@ -1445,15 +1451,40 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) { }); } - public Optional getOwner(NamespaceBundle bundle) throws Exception { - // if there is no znode for the service unit, it is not owned by any broker - return getOwnerAsync(bundle).get(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } - public CompletableFuture> getOwnerAsync(NamespaceBundle bundle) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); + return extensibleLoadManager.getOwnershipWithLookupDataAsync(bundle) + .thenCompose(lookupData -> { + if (lookupData.isPresent()) { + return CompletableFuture.completedFuture( + Optional.of(lookupData.get().toNamespaceEphemeralData())); + } else { + return CompletableFuture.completedFuture(Optional.empty()); + } + }); + } return ownershipCache.getOwnerAsync(bundle); } + public boolean checkOwnershipPresent(NamespaceBundle bundle) throws Exception { + return checkOwnershipPresentAsync(bundle).get(pulsar.getConfiguration() + .getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } + + public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bundle) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (bundle.getNamespaceObject().equals(SYSTEM_NAMESPACE)) { + return FutureUtil.failedFuture(new UnsupportedOperationException( + "Ownership check for system namespace is not supported")); + } + ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); + return extensibleLoadManager.getOwnershipAsync(Optional.empty(), bundle) + .thenApply(Optional::isPresent); + } + return getOwnerAsync(bundle).thenApply(Optional::isPresent); + } + public void unloadSLANamespace() throws Exception { PulsarAdmin adminClient = null; NamespaceName namespaceName = getSLAMonitorNamespace(host, config); @@ -1461,7 +1492,7 @@ public void unloadSLANamespace() throws Exception { LOG.info("Checking owner for SLA namespace {}", namespaceName); NamespaceBundle nsFullBundle = getFullBundle(namespaceName); - if (!getOwner(nsFullBundle).isPresent()) { + if (!checkOwnershipPresent(nsFullBundle)) { // No one owns the namespace so no point trying to unload it // Next lookup will assign the bundle to this broker. return; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index d977e63330586..f8a7a9b629f4e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -134,6 +134,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { @BeforeClass @Override public void setup() throws Exception { + conf.setForceDeleteNamespaceAllowed(true); conf.setAllowAutoTopicCreation(true); conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); @@ -142,6 +143,7 @@ public void setup() throws Exception { pulsar1 = pulsar; ServiceConfiguration defaultConf = getDefaultConf(); defaultConf.setAllowAutoTopicCreation(true); + defaultConf.setForceDeleteNamespaceAllowed(true); defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); defaultConf.setLoadBalancerSheddingEnabled(false); @@ -433,6 +435,54 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { assertTrue(bundlesData.getBoundaries().contains(midBundle)); assertTrue(bundlesData.getBoundaries().contains(highBundle)); } + @Test(timeOut = 30 * 1000) + public void testDeleteNamespaceBundle() throws Exception { + TopicName topicName = TopicName.get("test-delete-namespace-bundle"); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + + String broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + + checkOwnershipState(broker, bundle); + + admin.namespaces().deleteNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange()); + assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + } + + @Test(timeOut = 30 * 1000) + public void testDeleteNamespace() throws Exception { + String namespace = "public/test-delete-namespace"; + TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-topic"); + admin.namespaces().createNamespace(namespace); + admin.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(this.conf.getClusterName())); + assertTrue(admin.namespaces().getNamespaces("public").contains(namespace)); + admin.topics().createPartitionedTopic(topicName.toString(), 2); + admin.lookups().lookupTopic(topicName.toString()); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + try { + admin.namespaces().deleteNamespaceBundle(namespace, bundle.getBundleRange()); + fail(); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Cannot delete non empty bundle")); + } + admin.namespaces().deleteNamespaceBundle(namespace, bundle.getBundleRange(), true); + admin.lookups().lookupTopic(topicName.toString()); + + admin.namespaces().deleteNamespace(namespace, true); + assertFalse(admin.namespaces().getNamespaces("public").contains(namespace)); + } + + @Test(timeOut = 30 * 1000) + public void testCheckOwnershipPresentWithSystemNamespace() throws Exception { + NamespaceBundle namespaceBundle = + getBundleAsync(pulsar1, TopicName.get(NamespaceName.SYSTEM_NAMESPACE + "/test")).get(); + try { + pulsar1.getNamespaceService().checkOwnershipPresent(namespaceBundle); + } catch (Exception ex) { + log.info("Got exception", ex); + assertTrue(ex.getCause() instanceof UnsupportedOperationException); + } + } @Test public void testMoreThenOneFilter() throws Exception { From d8774671f5d52be4ef940ea894ba4f9b1f5717f9 Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Wed, 29 Mar 2023 10:36:23 +0800 Subject: [PATCH 038/280] [fix][test] Shutdown executor on PerformanceProducer closed (#19926) Co-authored-by: lushiji --- .../pulsar/testclient/PerformanceProducer.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 9aedc8c4bd327..0c56f1d5c736d 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -356,6 +356,7 @@ public static void main(String[] args) throws Exception { long start = System.nanoTime(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { + executorShutdownNow(); printAggregatedThroughput(start, arguments); printAggregatedStats(); })); @@ -481,6 +482,18 @@ public static void main(String[] args) throws Exception { } } + private static void executorShutdownNow() { + executor.shutdownNow(); + try { + if (!executor.awaitTermination(10, TimeUnit.SECONDS)) { + log.warn("Failed to terminate executor within timeout. The following are stack" + + " traces of still running threads."); + } + } catch (InterruptedException e) { + log.warn("Shutdown of thread pool was interrupted"); + } + } + static IMessageFormatter getMessageFormatter(String formatterClass) { try { ClassLoader classLoader = PerformanceProducer.class.getClassLoader(); From 5611faf61e70ce563bb43eb499436517b54d0bb5 Mon Sep 17 00:00:00 2001 From: crossoverJie Date: Wed, 29 Mar 2023 13:06:02 +0800 Subject: [PATCH 039/280] [fix][client] Fix client erroneous code comments (#19915) Co-authored-by: tison --- .../pulsar/client/api/ClientBuilder.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java index 8f2813ed14fff..8b959690a0363 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java @@ -227,23 +227,25 @@ ClientBuilder authentication(String authPluginClassName, Map aut /** * Set lookup timeout (default: matches operation timeout) * - * Lookup operations have a different load pattern to other operations. They can be handled by any broker, are not - * proportional to throughput, and are harmless to retry. Given this, it makes sense to allow them to retry longer - * than normal operation, especially if they experience a timeout. + *

+ * Lookup operations have a different load pattern to other operations. + * They can be handled by any broker, are not proportional to throughput, + * and are harmless to retry. Given this, it makes sense to allow them to + * retry longer than normal operation, especially if they experience a timeout. * - * By default this is set to match operation timeout. This is to maintain legacy behaviour. However, in practice - * it should be set to 5-10x the operation timeout. + *

+ * By default, this is set to match operation timeout. This is to maintain legacy behaviour. + * However, in practice it should be set to 5-10x the operation timeout. * - * @param lookupTimeout - * lookup timeout - * @param unit - * time unit for {@code lookupTimeout} + * @param lookupTimeout lookup timeout + * @param unit time unit for {@code lookupTimeout} * @return the client builder instance */ ClientBuilder lookupTimeout(int lookupTimeout, TimeUnit unit); /** - * Set the number of threads to be used for handling connections to brokers (default: 1 thread). + * Set the number of threads to be used for handling connections to brokers + * (default: Runtime.getRuntime().availableProcessors()). * * @param numIoThreads the number of IO threads * @return the client builder instance @@ -251,11 +253,12 @@ ClientBuilder authentication(String authPluginClassName, Map aut ClientBuilder ioThreads(int numIoThreads); /** - * Set the number of threads to be used for message listeners (default: 1 thread). + * Set the number of threads to be used for message listeners + * (default: Runtime.getRuntime().availableProcessors()). * *

The listener thread pool is shared across all the consumers and readers that are - * using a "listener" model to get messages. For a given consumer, the listener will be - * always invoked from the same thread, to ensure ordering. + * using a "listener" model to get messages. For a given consumer, the listener will + * always be invoked from the same thread, to ensure ordering. * * @param numListenerThreads the number of listener threads * @return the client builder instance From bbf52736f7542423b9699387eb9521c35e187816 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 29 Mar 2023 13:13:18 +0800 Subject: [PATCH 040/280] [improve][broker][PIP-195] Merge multiple buckets at once (#19927) --- .../mledger/impl/ManagedCursorImpl.java | 5 +- .../bucket/BucketDelayedDeliveryTracker.java | 199 ++++++++++-------- .../CombinedSegmentDelayedIndexQueue.java | 100 +++++---- .../BucketDelayedDeliveryTrackerTest.java | 8 +- .../delayed/bucket/DelayedIndexQueueTest.java | 21 +- 5 files changed, 180 insertions(+), 153 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 5851395b08566..05cad09b01833 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -363,8 +363,7 @@ private CompletableFuture computeCursorProperties( name, copy, lastCursorLedgerStat, new MetaStoreCallback<>() { @Override public void operationComplete(Void result, Stat stat) { - log.info("[{}] Updated ledger cursor: {} properties {}", ledger.getName(), - name, cursorProperties); + log.info("[{}] Updated ledger cursor: {}", ledger.getName(), name); ManagedCursorImpl.this.cursorProperties = Collections.unmodifiableMap(newProperties); updateCursorLedgerStat(copy, stat); updateCursorPropertiesResult.complete(result); @@ -373,7 +372,7 @@ public void operationComplete(Void result, Stat stat) { @Override public void operationFailed(MetaStoreException e) { log.error("[{}] Error while updating ledger cursor: {} properties {}", ledger.getName(), - name, cursorProperties, e); + name, newProperties, e); updateCursorPropertiesResult.completeExceptionally(e); } }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 39973928abf5d..ef7be187cec3d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -32,6 +32,7 @@ import io.netty.util.Timer; import java.time.Clock; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -43,6 +44,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import javax.annotation.concurrent.ThreadSafe; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -93,6 +95,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private static final Long INVALID_BUCKET_ID = -1L; + private static final int MAX_MERGE_NUM = 4; + public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -341,115 +345,136 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver return true; } - private synchronized CompletableFuture asyncMergeBucketSnapshot() { - List values = immutableBuckets.asMapOfRanges().values().stream().toList(); + private synchronized List selectMergedBuckets(final List values, int mergeNum) { + checkArgument(mergeNum < values.size()); long minNumberMessages = Long.MAX_VALUE; long minScheduleTimestamp = Long.MAX_VALUE; int minIndex = -1; - for (int i = 0; i + 1 < values.size(); i++) { - ImmutableBucket bucketL = values.get(i); - ImmutableBucket bucketR = values.get(i + 1); - // We should skip the bucket which last segment already been load to memory, avoid record replicated index. - if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId() - && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId() - // Skip the bucket that is merging - && !bucketL.merging && !bucketR.merging){ - long scheduleTimestamp = - Math.min(bucketL.firstScheduleTimestamps.get(bucketL.currentSegmentEntryId + 1), - bucketR.firstScheduleTimestamps.get(bucketR.currentSegmentEntryId + 1)); - long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages; - if (scheduleTimestamp <= minScheduleTimestamp) { - minScheduleTimestamp = scheduleTimestamp; - if (numberMessages < minNumberMessages) { - minNumberMessages = numberMessages; + for (int i = 0; i + (mergeNum - 1) < values.size(); i++) { + List immutableBuckets = values.subList(i, i + mergeNum); + if (immutableBuckets.stream().allMatch(bucket -> { + // We should skip the bucket which last segment already been load to memory, + // avoid record replicated index. + return bucket.lastSegmentEntryId > bucket.currentSegmentEntryId && !bucket.merging; + })) { + long numberMessages = immutableBuckets.stream() + .mapToLong(bucket -> bucket.numberBucketDelayedMessages) + .sum(); + if (numberMessages <= minNumberMessages) { + minNumberMessages = numberMessages; + long scheduleTimestamp = immutableBuckets.stream() + .mapToLong(bucket -> bucket.firstScheduleTimestamps.get(bucket.currentSegmentEntryId + 1)) + .min().getAsLong(); + if (scheduleTimestamp < minScheduleTimestamp) { + minScheduleTimestamp = scheduleTimestamp; minIndex = i; } } } } - if (minIndex == -1) { - log.warn("[{}] Can't find able merged bucket", dispatcher.getName()); - return CompletableFuture.completedFuture(null); + if (minIndex >= 0) { + return values.subList(minIndex, minIndex + MAX_MERGE_NUM); + } else if (mergeNum > 2){ + return selectMergedBuckets(values, mergeNum - 1); + } else { + return Collections.emptyList(); } + } - ImmutableBucket immutableBucketA = values.get(minIndex); - ImmutableBucket immutableBucketB = values.get(minIndex + 1); + private synchronized CompletableFuture asyncMergeBucketSnapshot() { + List immutableBucketList = immutableBuckets.asMapOfRanges().values().stream().toList(); + List toBeMergeImmutableBuckets = selectMergedBuckets(immutableBucketList, MAX_MERGE_NUM); + if (toBeMergeImmutableBuckets.isEmpty()) { + log.warn("[{}] Can't find able merged buckets", dispatcher.getName()); + return CompletableFuture.completedFuture(null); + } + + final String bucketsStr = toBeMergeImmutableBuckets.stream().map(Bucket::bucketKey).collect( + Collectors.joining(",")).replaceAll(DELAYED_BUCKET_KEY_PREFIX + "_", ""); if (log.isDebugEnabled()) { - log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(), - immutableBucketA.bucketKey(), immutableBucketB.bucketKey()); + log.info("[{}] Merging bucket snapshot, bucketKeys: {}", dispatcher.getName(), bucketsStr); } - immutableBucketA.merging = true; - immutableBucketB.merging = true; - return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> { + for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) { + immutableBucket.merging = true; + } + return asyncMergeBucketSnapshot(toBeMergeImmutableBuckets).whenComplete((__, ex) -> { synchronized (this) { - immutableBucketA.merging = false; - immutableBucketB.merging = false; + for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) { + immutableBucket.merging = false; + } } if (ex != null) { - log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}", - dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex); + log.error("[{}] Failed to merge bucket snapshot, bucketKeys: {}", + dispatcher.getName(), bucketsStr, ex); } else { - log.info("[{}] Merge bucket snapshot finish, bucketAKey: {}, bucketBKey: {}", - dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey()); + log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}", + dispatcher.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size()); } }); } - private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA, - ImmutableBucket bucketB) { - CompletableFuture createAFuture = bucketA.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE); - CompletableFuture createBFuture = bucketB.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE); + private synchronized CompletableFuture asyncMergeBucketSnapshot(List buckets) { + List> createFutures = + buckets.stream().map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE)) + .toList(); - return CompletableFuture.allOf(createAFuture, createBFuture).thenCompose(bucketId -> { - if (INVALID_BUCKET_ID.equals(createAFuture.join()) || INVALID_BUCKET_ID.equals(createBFuture.join())) { + return FutureUtil.waitForAll(createFutures).thenCompose(bucketId -> { + if (createFutures.stream().anyMatch(future -> INVALID_BUCKET_ID.equals(future.join()))) { return FutureUtil.failedFuture(new RuntimeException("Can't merge buckets due to bucket create failed")); } - CompletableFuture> futureA = - bucketA.getRemainSnapshotSegment(); - CompletableFuture> futureB = - bucketB.getRemainSnapshotSegment(); - return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap) + List>> getRemainFutures = + buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList(); + + return FutureUtil.waitForAll(getRemainFutures) + .thenApply(__ -> { + return CombinedSegmentDelayedIndexQueue.wrap( + getRemainFutures.stream().map(CompletableFuture::join).toList()); + }) .thenAccept(combinedDelayedIndexQueue -> { synchronized (BucketDelayedDeliveryTracker.this) { Pair immutableBucketDelayedIndexPair = lastMutableBucket.createImmutableBucketAndAsyncPersistent( timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment, - sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId, - bucketB.endLedgerId); + sharedBucketPriorityQueue, combinedDelayedIndexQueue, + buckets.get(0).startLedgerId, + buckets.get(buckets.size() - 1).endLedgerId); // Merge bit map to new bucket - Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); - Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); - Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); - delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { - delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { - if (bitMapA == null) { - return bitMapB; - } - - bitMapA.or(bitMapB); - return bitMapA; + Map delayedIndexBitMap = + new HashMap<>(buckets.get(0).getDelayedIndexBitMap()); + for (int i = 1; i < buckets.size(); i++) { + buckets.get(i).delayedIndexBitMap.forEach((ledgerId, bitMapB) -> { + delayedIndexBitMap.compute(ledgerId, (k, bitMap) -> { + if (bitMap == null) { + return bitMapB; + } + + bitMap.or(bitMapB); + return bitMap; + }); }); - }); + } immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); afterCreateImmutableBucket(immutableBucketDelayedIndexPair); immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() .orElse(NULL_LONG_PROMISE).thenCompose(___ -> { - CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); - CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); - return CompletableFuture.allOf(removeAFuture, removeBFuture); + List> removeFutures = + buckets.stream().map(ImmutableBucket::asyncDeleteBucketSnapshot) + .toList(); + return FutureUtil.waitForAll(removeFutures); }); - Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges(); - immutableBucketMap.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); - immutableBucketMap.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); + for (ImmutableBucket bucket : buckets) { + immutableBuckets.asMapOfRanges() + .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); + } } }); }); @@ -593,14 +618,12 @@ public boolean shouldPauseAllDeliveries() { @Override public synchronized CompletableFuture clear() { - return cleanImmutableBuckets(true).thenRun(() -> { - synchronized (this) { - sharedBucketPriorityQueue.clear(); - lastMutableBucket.clear(); - snapshotSegmentLastIndexTable.clear(); - numberDelayedMessages = 0; - } - }); + CompletableFuture future = cleanImmutableBuckets(); + sharedBucketPriorityQueue.clear(); + lastMutableBucket.clear(); + snapshotSegmentLastIndexTable.clear(); + numberDelayedMessages = 0; + return future; } @Override @@ -609,30 +632,24 @@ public synchronized void close() { lastMutableBucket.close(); sharedBucketPriorityQueue.close(); try { - cleanImmutableBuckets(false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); + List> completableFutures = immutableBuckets.asMapOfRanges().values().stream() + .map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE)).toList(); + FutureUtil.waitForAll(completableFutures).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); } catch (Exception e) { log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e); } } - private CompletableFuture cleanImmutableBuckets(boolean delete) { - if (immutableBuckets != null) { - List> futures = new ArrayList<>(); - Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); - while (iterator.hasNext()) { - ImmutableBucket bucket = iterator.next(); - if (delete) { - futures.add(bucket.clear()); - } else { - bucket.getSnapshotCreateFuture().ifPresent(future -> futures.add(future.thenApply(x -> null))); - } - numberDelayedMessages -= bucket.getNumberBucketDelayedMessages(); - iterator.remove(); - } - return FutureUtil.waitForAll(futures); - } else { - return CompletableFuture.completedFuture(null); + private CompletableFuture cleanImmutableBuckets() { + List> futures = new ArrayList<>(); + Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); + while (iterator.hasNext()) { + ImmutableBucket bucket = iterator.next(); + futures.add(bucket.clear()); + numberDelayedMessages -= bucket.getNumberBucketDelayedMessages(); + iterator.remove(); } + return FutureUtil.waitForAll(futures); } private boolean removeIndexBit(long ledgerId, long entryId) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java index 3f89cc9fdfb15..5655a26878296 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java @@ -18,37 +18,48 @@ */ package org.apache.pulsar.broker.delayed.bucket; +import java.util.Comparator; import java.util.List; +import java.util.Objects; +import java.util.PriorityQueue; import javax.annotation.concurrent.NotThreadSafe; +import lombok.AllArgsConstructor; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; @NotThreadSafe class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue { - private final List segmentListA; - private final List segmentListB; + @AllArgsConstructor + static class Node { + List segmentList; - private int segmentListACursor = 0; - private int segmentListBCursor = 0; - private int segmentACursor = 0; - private int segmentBCursor = 0; + int segmentListCursor; - private CombinedSegmentDelayedIndexQueue(List segmentListA, - List segmentListB) { - this.segmentListA = segmentListA; - this.segmentListB = segmentListB; + int segmentCursor; } - public static CombinedSegmentDelayedIndexQueue wrap( - List segmentListA, - List segmentListB) { - return new CombinedSegmentDelayedIndexQueue(segmentListA, segmentListB); + private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare( + node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor), + node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor)); + + private final PriorityQueue kpq; + + private CombinedSegmentDelayedIndexQueue(List> segmentLists) { + this.kpq = new PriorityQueue<>(segmentLists.size(), COMPARATOR_NODE); + for (List segmentList : segmentLists) { + Node node = new Node(segmentList, 0, 0); + kpq.offer(node); + } + } + + public static CombinedSegmentDelayedIndexQueue wrap(List> segmentLists) { + return new CombinedSegmentDelayedIndexQueue(segmentLists); } @Override public boolean isEmpty() { - return segmentListACursor >= segmentListA.size() && segmentListBCursor >= segmentListB.size(); + return kpq.isEmpty(); } @Override @@ -62,48 +73,35 @@ public DelayedIndex pop() { } private DelayedIndex getValue(boolean needAdvanceCursor) { - // skip empty segment - while (segmentListACursor < segmentListA.size() - && segmentListA.get(segmentListACursor).getIndexesCount() == 0) { - segmentListACursor++; - } - while (segmentListBCursor < segmentListB.size() - && segmentListB.get(segmentListBCursor).getIndexesCount() == 0) { - segmentListBCursor++; - } + Node node = kpq.peek(); + Objects.requireNonNull(node); - DelayedIndex delayedIndexA = null; - DelayedIndex delayedIndexB = null; - if (segmentListACursor >= segmentListA.size()) { - delayedIndexB = segmentListB.get(segmentListBCursor).getIndexes(segmentBCursor); - } else if (segmentListBCursor >= segmentListB.size()) { - delayedIndexA = segmentListA.get(segmentListACursor).getIndexes(segmentACursor); - } else { - delayedIndexA = segmentListA.get(segmentListACursor).getIndexes(segmentACursor); - delayedIndexB = segmentListB.get(segmentListBCursor).getIndexes(segmentBCursor); + SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor); + DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor); + if (!needAdvanceCursor) { + return delayedIndex; } - DelayedIndex resultValue; - if (delayedIndexB == null || (delayedIndexA != null && COMPARATOR.compare(delayedIndexA, delayedIndexB) < 0)) { - resultValue = delayedIndexA; - if (needAdvanceCursor) { - if (++segmentACursor >= segmentListA.get(segmentListACursor).getIndexesCount()) { - segmentListA.set(segmentListACursor, null); - ++segmentListACursor; - segmentACursor = 0; - } - } - } else { - resultValue = delayedIndexB; - if (needAdvanceCursor) { - if (++segmentBCursor >= segmentListB.get(segmentListBCursor).getIndexesCount()) { - segmentListB.set(segmentListBCursor, null); - ++segmentListBCursor; - segmentBCursor = 0; + kpq.poll(); + + if (node.segmentCursor + 1 < snapshotSegment.getIndexesCount()) { + node.segmentCursor++; + kpq.offer(node); + } else { + // help GC + node.segmentList.set(node.segmentListCursor, null); + while (node.segmentListCursor + 1 < node.segmentList.size()) { + node.segmentListCursor++; + node.segmentCursor = 0; + + // skip empty segment + if (node.segmentList.get(node.segmentListCursor).getIndexesCount() > 0) { + kpq.offer(node); + break; } } } - return resultValue; + return delayedIndex; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index fbb866d48ed69..717bada7705dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -267,9 +267,7 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) { int size = tracker.getImmutableBuckets().asMapOfRanges().size(); - Awaitility.await().untilAsserted(() -> { - assertEquals(10, size); - }); + assertTrue(size <= 10); tracker.addMessage(111, 1011, 111 * 10); Awaitility.await().untilAsserted(() -> { @@ -333,9 +331,7 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) { int size = tracker.getImmutableBuckets().asMapOfRanges().size(); - Awaitility.await().untilAsserted(() -> { - assertEquals(10, size); - }); + assertTrue(size <= 10); tracker.addMessage(111, 1011, 111 * 10); Awaitility.await().untilAsserted(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java index 865ccb6934ab7..8f87f0d49a20c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java @@ -101,8 +101,25 @@ public void testCombinedSegmentDelayedIndexQueue() { segmentListB.add(snapshotSegmentB); segmentListB.add(SnapshotSegment.newBuilder().build()); + List listC = new ArrayList<>(); + for (int i = 10; i < 30; i+=2) { + DelayedIndex delayedIndex = + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) + .build(); + + DelayedIndex delayedIndex2 = + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) + .build(); + listC.add(delayedIndex); + listC.add(delayedIndex2); + } + + SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build(); + List segmentListC = new ArrayList<>(); + segmentListC.add(snapshotSegmentC); + CombinedSegmentDelayedIndexQueue delayedIndexQueue = - CombinedSegmentDelayedIndexQueue.wrap(segmentListA, segmentListB); + CombinedSegmentDelayedIndexQueue.wrap(List.of(segmentListA, segmentListB, segmentListC)); int count = 0; while (!delayedIndexQueue.isEmpty()) { @@ -114,7 +131,7 @@ public void testCombinedSegmentDelayedIndexQueue() { Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } - Assert.assertEquals(38, count); + Assert.assertEquals(58, count); } @Test From 7cb48fd9d41f8606d86a0503b0130aec6b9d00d5 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Tue, 28 Mar 2023 23:54:05 -0700 Subject: [PATCH 041/280] [improve][io] KCA: option to collapse partitioned topics (#19923) --- pulsar-io/kafka-connect-adaptor/pom.xml | 7 ++ .../io/kafka/connect/KafkaConnectSink.java | 20 ++++- .../connect/PulsarKafkaConnectSinkConfig.java | 5 ++ .../kafka/connect/KafkaConnectSinkTest.java | 88 +++++++++++++++++++ 4 files changed, 118 insertions(+), 2 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 5e192be95ecf4..12ebda087eb3d 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -39,6 +39,13 @@ provided + + ${project.groupId} + pulsar-common + ${project.version} + compile + + com.fasterxml.jackson.core jackson-databind diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java index 06f66f60380d9..475724cc4e545 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java @@ -54,6 +54,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.KeyValueSchema; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; @@ -91,6 +92,9 @@ public class KafkaConnectSink implements Sink { protected String topicName; private boolean sanitizeTopicName = false; + // Thi is a workaround for https://github.com/apache/pulsar/issues/19922 + private boolean collapsePartitionedTopics = false; + private final Cache sanitizedTopicCache = CacheBuilder.newBuilder().maximumSize(1000) .expireAfterAccess(30, TimeUnit.MINUTES).build(); @@ -159,6 +163,7 @@ public void open(Map config, SinkContext ctx) throws Exception { topicName = kafkaSinkConfig.getTopic(); unwrapKeyValueIfAvailable = kafkaSinkConfig.isUnwrapKeyValueIfAvailable(); sanitizeTopicName = kafkaSinkConfig.isSanitizeTopicName(); + collapsePartitionedTopics = kafkaSinkConfig.isCollapsePartitionedTopics(); useIndexAsOffset = kafkaSinkConfig.isUseIndexAsOffset(); maxBatchBitsForOffset = kafkaSinkConfig.getMaxBatchBitsForOffset(); @@ -417,8 +422,19 @@ static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId me @SuppressWarnings("rawtypes") protected SinkRecord toSinkRecord(Record sourceRecord) { - final int partition = sourceRecord.getPartitionIndex().orElse(0); - final String topic = sanitizeNameIfNeeded(sourceRecord.getTopicName().orElse(topicName), sanitizeTopicName); + final int partition; + final String topic; + + if (collapsePartitionedTopics + && sourceRecord.getTopicName().isPresent() + && TopicName.get(sourceRecord.getTopicName().get()).isPartitioned()) { + TopicName tn = TopicName.get(sourceRecord.getTopicName().get()); + partition = tn.getPartitionIndex(); + topic = sanitizeNameIfNeeded(tn.getPartitionedTopicName(), sanitizeTopicName); + } else { + partition = sourceRecord.getPartitionIndex().orElse(0); + topic = sanitizeNameIfNeeded(sourceRecord.getTopicName().orElse(topicName), sanitizeTopicName); + } final Object key; final Object value; final Schema keySchema; diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java index 19dd784578915..2525081a41ebb 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java @@ -94,6 +94,11 @@ public class PulsarKafkaConnectSinkConfig implements Serializable { + "In some cases it may result in topic name collisions (topic_a and topic.a will become the same)") private boolean sanitizeTopicName = false; + @FieldDoc( + defaultValue = "false", + help = "Supply kafka record with topic name without -partition- suffix for partitioned topics.") + private boolean collapsePartitionedTopics = false; + public static PulsarKafkaConnectSinkConfig load(String yamlFile) throws IOException { ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); return mapper.readValue(new File(yamlFile), PulsarKafkaConnectSinkConfig.class); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index e9d454ed2fd5a..6ccfa3b71a2f7 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -1571,6 +1571,94 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception { assertEquals(ref.getBatchIdx(), batchIdx); } + @Test + public void collapsePartitionedTopicEnabledTest() throws Exception { + testCollapsePartitionedTopic(true, + "persistent://a/b/fake-topic-partition-0", + "persistent://a/b/fake-topic", + 0); + + testCollapsePartitionedTopic(true, + "persistent://a/b/fake-topic-partition-1", + "persistent://a/b/fake-topic", + 1); + + testCollapsePartitionedTopic(true, + "persistent://a/b/fake-topic", + "persistent://a/b/fake-topic", + 0); + + testCollapsePartitionedTopic(true, + "fake-topic-partition-5", + "persistent://public/default/fake-topic", + 5); + } + + @Test + public void collapsePartitionedTopicDisabledTest() throws Exception { + testCollapsePartitionedTopic(false, + "persistent://a/b/fake-topic-partition-0", + "persistent://a/b/fake-topic-partition-0", + 0); + + testCollapsePartitionedTopic(false, + "persistent://a/b/fake-topic-partition-1", + "persistent://a/b/fake-topic-partition-1", + 0); + + testCollapsePartitionedTopic(false, + "persistent://a/b/fake-topic", + "persistent://a/b/fake-topic", + 0); + + testCollapsePartitionedTopic(false, + "fake-topic-partition-5", + "fake-topic-partition-5", + 0); + } + + private void testCollapsePartitionedTopic(boolean isEnabled, + String pulsarTopic, + String expectedKafkaTopic, + int expectedPartition) throws Exception { + props.put("kafkaConnectorSinkClass", SchemaedFileStreamSinkConnector.class.getCanonicalName()); + props.put("collapsePartitionedTopics", Boolean.toString(isEnabled)); + + KafkaConnectSink sink = new KafkaConnectSink(); + sink.open(props, context); + + AvroSchema pulsarAvroSchema + = AvroSchema.of(PulsarSchemaToKafkaSchemaTest.StructWithAnnotations.class); + + final GenericData.Record obj = new GenericData.Record(pulsarAvroSchema.getAvroSchema()); + obj.put("field1", (byte) 10); + obj.put("field2", "test"); + obj.put("field3", (short) 100); + + final GenericRecord rec = getGenericRecord(obj, pulsarAvroSchema); + Message msg = mock(MessageImpl.class); + when(msg.getValue()).thenReturn(rec); + when(msg.getKey()).thenReturn("key"); + when(msg.hasKey()).thenReturn(true); + when(msg.getMessageId()).thenReturn(new MessageIdImpl(1, 0, 0)); + + final AtomicInteger status = new AtomicInteger(0); + Record record = PulsarRecord.builder() + .topicName(pulsarTopic) + .message(msg) + .schema(pulsarAvroSchema) + .ackFunction(status::incrementAndGet) + .failFunction(status::decrementAndGet) + .build(); + + SinkRecord sinkRecord = sink.toSinkRecord(record); + + Assert.assertEquals(sinkRecord.topic(), expectedKafkaTopic); + Assert.assertEquals(sinkRecord.kafkaPartition(), expectedPartition); + + sink.close(); + } + @SneakyThrows private java.util.Date getDateFromString(String dateInString) { SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss"); From ee5ac8607ebd7f90843789d4eead48de8600c1f0 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 29 Mar 2023 02:30:24 -0700 Subject: [PATCH 042/280] [improve][broker] PIP-192 Improved Auto Unload Logic (#19813) Master Issue: Master Issue: https://github.com/apache/pulsar/issues/16691, https://github.com/apache/pulsar/issues/18099 ### Motivation Raising a PR to implement Master Issue: https://github.com/apache/pulsar/issues/16691, https://github.com/apache/pulsar/issues/18099 We want to reduce unload frequencies from flaky traffic. ### Modifications This PR - Introduced a config `loadBalancerSheddingConditionHitCountThreshold` to further restrict shedding conditions based on the hit count. - Normalized offload traffic - Lowered the default `loadBalanceSheddingDelayInSeconds` value from 600 to 180, as 10 mins are too long. 3 mins can be long enough to catch the new load after unloads. - Changed the config `loadBalancerBundleLoadReportPercentage` to `loadBalancerMaxNumberOfBundlesInBundleLoadReport` to make the topk bundle count absolute instead of relative. - Renamed `loadBalancerNamespaceBundleSplitConditionThreshold` to `loadBalancerNamespaceBundleSplitConditionHitCountThreshold` to be consistent with `*ConditionHitCountThreshold`. - Renamed `loadBalancerMaxNumberOfBrokerTransfersPerCycle ` to `loadBalancerMaxNumberOfBrokerSheddingPerCycle`. - Added LoadDataStore cleanup logic in BSC monitor. - Added `msgThroughputEMA` in BrokerLoadData to smooth the broker throughput info. - Updated Topk bundles sorted in a ascending order (instead of descending) - Update some info logs to only show in the debug mode. - Added load data tombstone upon Own, Releasing, Splitting - Added the bundle ownership(isOwned) check upon split and unload. - Added swap unload logic --- conf/broker.conf | 85 +++ .../pulsar/broker/ServiceConfiguration.java | 23 +- .../extensions/ExtensibleLoadManagerImpl.java | 67 ++- .../channel/ServiceUnitStateChannel.java | 25 + .../channel/ServiceUnitStateChannelImpl.java | 85 ++- .../extensions/data/BrokerLoadData.java | 34 +- .../extensions/models/TopKBundles.java | 2 +- .../extensions/models/UnloadCounter.java | 4 +- .../extensions/models/UnloadDecision.java | 2 +- .../reporter/BrokerLoadDataReporter.java | 90 ++- .../reporter/TopBundleLoadDataReporter.java | 63 ++- .../scheduler/NamespaceUnloadStrategy.java | 8 + .../extensions/scheduler/SplitScheduler.java | 16 +- .../extensions/scheduler/TransferShedder.java | 469 +++++++++++----- .../extensions/scheduler/UnloadScheduler.java | 19 +- ...faultNamespaceBundleSplitStrategyImpl.java | 144 +++-- .../LeastResourceUsageWithWeight.java | 8 +- .../ExtensibleLoadManagerImplTest.java | 6 +- .../channel/ServiceUnitStateChannelTest.java | 66 +++ .../extensions/data/BrokerLoadDataTest.java | 10 +- .../extensions/models/TopKBundlesTest.java | 14 +- .../reporter/BrokerLoadDataReporterTest.java | 93 +++- .../TopBundleLoadDataReporterTest.java | 90 ++- .../scheduler/TransferShedderTest.java | 515 +++++++++++++++--- .../scheduler/UnloadSchedulerTest.java | 12 +- ...faultNamespaceBundleSplitStrategyTest.java | 49 +- .../loadbalancer/NamespaceBundleStats.java | 2 + .../data/loadbalancer/ResourceUsage.java | 2 + 28 files changed, 1648 insertions(+), 355 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index d52adb254563d..1bbb12313f9a6 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1373,6 +1373,91 @@ loadBalancerBundleUnloadMinThroughputThreshold=10 # Time to wait for the unloading of a namespace bundle namespaceBundleUnloadingTimeoutMs=60000 +### --- Load balancer extension --- ### + +# Option to enable the debug mode for the load balancer logics. +# The debug mode prints more logs to provide more information such as load balance states and decisions. +# (only used in load balancer extension logics) +loadBalancerDebugModeEnabled=false + +# The target standard deviation of the resource usage across brokers +# (100% resource usage is 1.0 load). +# The shedder logic tries to distribute bundle load across brokers to meet this target std. +# The smaller value will incur load balancing more frequently. +# (only used in load balancer extension TransferSheddeer) +loadBalancerBrokerLoadTargetStd=0.25 + +# Threshold to the consecutive count of fulfilled shedding(unload) conditions. +# If the unload scheduler consecutively finds bundles that meet unload conditions +# many times bigger than this threshold, the scheduler will shed the bundles. +# The bigger value will incur less bundle unloading/transfers. +# (only used in load balancer extension TransferSheddeer) +loadBalancerSheddingConditionHitCountThreshold=3 + +# Option to enable the bundle transfer mode when distributing bundle loads. +# On: transfer bundles from overloaded brokers to underloaded +# -- pre-assigns the destination broker upon unloading). +# Off: unload bundles from overloaded brokers +# -- post-assigns the destination broker upon lookups). +# (only used in load balancer extension TransferSheddeer) +loadBalancerTransferEnabled=true + +# Maximum number of brokers to unload bundle load for each unloading cycle. +# The bigger value will incur more unloading/transfers for each unloading cycle. +# (only used in load balancer extension TransferSheddeer) +loadBalancerMaxNumberOfBrokerSheddingPerCycle=3 + +# Delay (in seconds) to the next unloading cycle after unloading. +# The logic tries to give enough time for brokers to recompute load after unloading. +# The bigger value will delay the next unloading cycle longer. +# (only used in load balancer extension TransferSheddeer) +loadBalanceSheddingDelayInSeconds=180 + +# Broker load data time to live (TTL in seconds). +# The logic tries to avoid (possibly unavailable) brokers with out-dated load data, +# and those brokers will be ignored in the load computation. +# When tuning this value, please consider loadBalancerReportUpdateMaxIntervalMinutes. +#The current default is loadBalancerReportUpdateMaxIntervalMinutes * 2. +# (only used in load balancer extension TransferSheddeer) +loadBalancerBrokerLoadDataTTLInSeconds=1800 + +# Max number of bundles in bundle load report from each broker. +# The load balancer distributes bundles across brokers, +# based on topK bundle load data and other broker load data. +# The bigger value will increase the overhead of reporting many bundles in load data. +# (only used in load balancer extension logics) +loadBalancerMaxNumberOfBundlesInBundleLoadReport=10 + +# Service units'(bundles) split interval. Broker periodically checks whether +# some service units(e.g. bundles) should split if they become hot-spots. +# (only used in load balancer extension logics) +loadBalancerSplitIntervalMinutes=1 + +# Max number of bundles to split to per cycle. +# (only used in load balancer extension logics) +loadBalancerMaxNumberOfBundlesToSplitPerCycle=10 + +# Threshold to the consecutive count of fulfilled split conditions. +# If the split scheduler consecutively finds bundles that meet split conditions +# many times bigger than this threshold, the scheduler will trigger splits on the bundles +# (if the number of bundles is less than loadBalancerNamespaceMaximumBundles). +# (only used in load balancer extension logics) +loadBalancerNamespaceBundleSplitConditionHitCountThreshold=3 + +# After this delay, the service-unit state channel tombstones any service units (e.g., bundles) +# in semi-terminal states. For example, after splits, parent bundles will be `deleted`, +# and then after this delay, the parent bundles' state will be `tombstoned` +# in the service-unit state channel. +# Pulsar does not immediately remove such semi-terminal states +# to avoid unnecessary system confusion, +# as the bundles in the `tombstoned` state might temporarily look available to reassign. +# Rarely, one could lower this delay in order to aggressively clean +# the service-unit state channel when there are a large number of bundles. +# minimum value = 30 secs +# (only used in load balancer extension logics) +loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds=604800 + + ### --- Replication --- ### # Enable replication metrics diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index e2bcf03353b59..03e0fec0a5346 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2506,6 +2506,17 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private double loadBalancerBrokerLoadTargetStd = 0.25; + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + dynamic = true, + doc = "Threshold to the consecutive count of fulfilled shedding(unload) conditions. " + + "If the unload scheduler consecutively finds bundles that meet unload conditions " + + "many times bigger than this threshold, the scheduler will shed the bundles. " + + "The bigger value will incur less bundle unloading/transfers. " + + "(only used in load balancer extension TransferSheddeer)" + ) + private int loadBalancerSheddingConditionHitCountThreshold = 3; + @FieldContext( category = CATEGORY_LOAD_BALANCER, dynamic = true, @@ -2521,11 +2532,11 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_LOAD_BALANCER, dynamic = true, - doc = "Maximum number of brokers to transfer bundle load for each unloading cycle. " + doc = "Maximum number of brokers to unload bundle load for each unloading cycle. " + "The bigger value will incur more unloading/transfers for each unloading cycle. " + "(only used in load balancer extension TransferSheddeer)" ) - private int loadBalancerMaxNumberOfBrokerTransfersPerCycle = 3; + private int loadBalancerMaxNumberOfBrokerSheddingPerCycle = 3; @FieldContext( category = CATEGORY_LOAD_BALANCER, @@ -2535,7 +2546,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "The bigger value will delay the next unloading cycle longer. " + "(only used in load balancer extension TransferSheddeer)" ) - private long loadBalanceUnloadDelayInSeconds = 600; + private long loadBalanceSheddingDelayInSeconds = 180; @FieldContext( category = CATEGORY_LOAD_BALANCER, @@ -2552,13 +2563,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "Percentage of bundles to compute topK bundle load data from each broker. " + doc = "Max number of bundles in bundle load report from each broker. " + "The load balancer distributes bundles across brokers, " + "based on topK bundle load data and other broker load data." + "The bigger value will increase the overhead of reporting many bundles in load data. " + "(only used in load balancer extension logics)" ) - private double loadBalancerBundleLoadReportPercentage = 10; + private int loadBalancerMaxNumberOfBundlesInBundleLoadReport = 10; @FieldContext( category = CATEGORY_LOAD_BALANCER, doc = "Service units'(bundles) split interval. Broker periodically checks whether " @@ -2582,7 +2593,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "(if the number of bundles is less than loadBalancerNamespaceMaximumBundles). " + "(only used in load balancer extension logics)" ) - private int loadBalancerNamespaceBundleSplitConditionThreshold = 5; + private int loadBalancerNamespaceBundleSplitConditionHitCountThreshold = 3; @FieldContext( category = CATEGORY_LOAD_BALANCER, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 7aefef596e7cb..3e078d0a5eb20 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -85,6 +85,7 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; +import org.slf4j.Logger; @Slf4j public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { @@ -101,6 +102,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private static final long MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS = 200; + private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000; + private PulsarService pulsar; private ServiceConfiguration conf; @@ -108,10 +111,12 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { @Getter private BrokerRegistry brokerRegistry; + @Getter private ServiceUnitStateChannel serviceUnitStateChannel; private AntiAffinityGroupPolicyFilter antiAffinityGroupPolicyFilter; + @Getter private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; private LoadDataStore brokerLoadDataStore; @@ -139,6 +144,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private ScheduledFuture brokerLoadDataReportTask; private ScheduledFuture topBundlesLoadDataReportTask; + + private ScheduledFuture monitorTask; private SplitScheduler splitScheduler; private UnloadManager unloadManager; @@ -148,6 +155,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private boolean started = false; private final AssignCounter assignCounter = new AssignCounter(); + @Getter private final UnloadCounter unloadCounter = new UnloadCounter(); private final SplitCounter splitCounter = new SplitCounter(); @@ -162,14 +170,14 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { lookupRequests = ConcurrentOpenHashMap.>>newBuilder() .build(); - private final CountDownLatch loadStoreInitWaiter = new CountDownLatch(1); + private final CountDownLatch initWaiter = new CountDownLatch(1); public enum Role { Leader, Follower } - private Role role; + private volatile Role role; /** * Life cycle: Constructor -> initialize -> start -> close. @@ -194,6 +202,10 @@ public static ExtensibleLoadManagerImpl get(LoadManager loadManager) { return loadManagerWrapper.get(); } + public static boolean debug(ServiceConfiguration config, Logger log) { + return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); + } + @Override public void start() throws PulsarServerException { if (this.started) { @@ -231,7 +243,6 @@ public void start() throws PulsarServerException { this.brokerLoadDataStore.startTableView(); this.topBundlesLoadDataStore = LoadDataStoreFactory .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); - this.loadStoreInitWaiter.countDown(); } catch (LoadDataStoreException e) { throw new PulsarServerException(e); } @@ -247,7 +258,8 @@ public void start() throws PulsarServerException { this.topBundleLoadDataReporter = new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore); - + this.serviceUnitStateChannel.listen(brokerLoadDataReporter); + this.serviceUnitStateChannel.listen(topBundleLoadDataReporter); var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis(); this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor() .scheduleAtFixedRate(() -> { @@ -273,13 +285,21 @@ public void start() throws PulsarServerException { interval, interval, TimeUnit.MILLISECONDS); + this.monitorTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + monitor(); + }, + MONITOR_INTERVAL_IN_MILLIS, + MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS); + this.unloadScheduler = new UnloadScheduler( - pulsar, pulsar.getLoadManagerExecutor(), unloadManager, - context, serviceUnitStateChannel, antiAffinityGroupPolicyHelper, unloadCounter, unloadMetrics); + pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel, + unloadCounter, unloadMetrics); this.unloadScheduler.start(); this.splitScheduler = new SplitScheduler( pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context); this.splitScheduler.start(); + this.initWaiter.countDown(); this.started = true; } @@ -327,7 +347,7 @@ public CompletableFuture> assign(Optional { if (broker.isEmpty()) { String errorMsg = String.format( - "Failed to look up a broker registry:%s for bundle:%s", broker, bundle); + "Failed to get or assign the owner for bundle:%s", bundle); log.error(errorMsg); throw new IllegalStateException(errorMsg); } @@ -498,6 +518,10 @@ public void close() throws PulsarServerException { topBundlesLoadDataReportTask.cancel(true); } + if (monitorTask != null) { + monitorTask.cancel(true); + } + this.brokerLoadDataStore.close(); this.topBundlesLoadDataStore.close(); this.unloadScheduler.close(); @@ -539,8 +563,8 @@ void playLeader() { int retry = 0; while (true) { try { + initWaiter.await(); serviceUnitStateChannel.scheduleOwnershipMonitor(); - loadStoreInitWaiter.await(); topBundlesLoadDataStore.startTableView(); unloadScheduler.start(); break; @@ -575,8 +599,8 @@ void playFollower() { int retry = 0; while (true) { try { + initWaiter.await(); serviceUnitStateChannel.cancelOwnershipMonitor(); - loadStoreInitWaiter.await(); topBundlesLoadDataStore.closeTableView(); unloadScheduler.close(); break; @@ -631,4 +655,29 @@ public List getMetrics() { return metricsCollection; } + + private void monitor() { + try { + initWaiter.await(); + + // Monitor role + // Periodically check the role in case ZK watcher fails. + var isChannelOwner = serviceUnitStateChannel.isChannelOwner(); + if (isChannelOwner) { + if (role != Leader) { + log.warn("Current role:{} does not match with the channel ownership:{}. " + + "Playing the leader role.", role, isChannelOwner); + playLeader(); + } + } else { + if (role != Follower) { + log.warn("Current role:{} does not match with the channel ownership:{}. " + + "Playing the follower role.", role, isChannelOwner); + playFollower(); + } + } + } catch (Throwable e) { + log.error("Failed to get the channel ownership.", e); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java index 719c72a67b4ea..4782a31fe0c56 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java @@ -64,6 +64,12 @@ public interface ServiceUnitStateChannel extends Closeable { */ CompletableFuture isChannelOwnerAsync(); + /** + * Checks if the current broker is the owner broker of the system topic in this channel. + * @return True if the current broker is the owner. Otherwise, false. + */ + boolean isChannelOwner(); + /** * Handles the metadata session events to track * if the connection between the broker and metadata store is stable or not. @@ -125,6 +131,25 @@ public interface ServiceUnitStateChannel extends Closeable { */ CompletableFuture> getOwnerAsync(String serviceUnit); + /** + * Checks if the target broker is the owner of the service unit. + * + * + * @param serviceUnit (e.g. bundle) + * @param targetBroker + * @return true if the target broker is the owner. false if unknown. + */ + boolean isOwner(String serviceUnit, String targetBroker); + + /** + * Checks if the current broker is the owner of the service unit. + * + * + * @param serviceUnit (e.g. bundle)) + * @return true if the current broker is the owner. false if unknown. + */ + boolean isOwner(String serviceUnit); + /** * Asynchronously publishes the service unit assignment event to the system topic in this channel. * It de-duplicates assignment events if there is any ongoing assignment event for the same service unit. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index ca9e55923e799..e7fb59450b25c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -70,6 +70,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; @@ -233,7 +234,7 @@ public void scheduleOwnershipMonitor() { log.info("Failed to monitor the ownerships. will retry..", e); } }, - ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS); + 0, ownershipMonitorDelayTimeInSecs, SECONDS); log.info("This leader broker:{} started the ownership monitor.", lookupServiceAddress); } @@ -390,7 +391,7 @@ private boolean validateChannelState(ChannelState targetState, boolean checkLowe } private boolean debug() { - return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); + return ExtensibleLoadManagerImpl.debug(config, log); } public CompletableFuture> getChannelOwnerAsync() { @@ -426,7 +427,7 @@ public CompletableFuture isChannelOwnerAsync() { }); } - private boolean isChannelOwner() { + public boolean isChannelOwner() { try { return isChannelOwnerAsync().get( MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS); @@ -437,6 +438,25 @@ private boolean isChannelOwner() { } } + public boolean isOwner(String serviceUnit, String targetBroker) { + if (!validateChannelState(Started, true)) { + throw new IllegalStateException("Invalid channel state:" + channelState.name()); + } + var ownerFuture = getOwnerAsync(serviceUnit); + if (!ownerFuture.isDone() || ownerFuture.isCompletedExceptionally() || ownerFuture.isCancelled()) { + return false; + } + var owner = ownerFuture.join(); + if (owner.isPresent() && StringUtils.equals(targetBroker, owner.get())) { + return true; + } + return false; + } + + public boolean isOwner(String serviceUnit) { + return isOwner(serviceUnit, lookupServiceAddress); + } + public CompletableFuture> getOwnerAsync(String serviceUnit) { if (!validateChannelState(Started, true)) { return CompletableFuture.failedFuture( @@ -951,9 +971,11 @@ private void handleBrokerCreationEvent(String broker) { + " Active cleanup job count:{}", broker, cleanupJobs.size()); } else { - log.info("Failed to cancel the ownership cleanup for broker:{}." - + " There was no scheduled cleanup job. Active cleanup job count:{}", - broker, cleanupJobs.size()); + if (debug()) { + log.info("No needs to cancel the ownership cleanup for broker:{}." + + " There was no scheduled cleanup job. Active cleanup job count:{}", + broker, cleanupJobs.size()); + } } } @@ -989,6 +1011,8 @@ private void scheduleCleanup(String broker, long delayInSecs) { log.error("Failed to run the cleanup job for the broker {}, " + "totalCleanupErrorCnt:{}.", broker, totalCleanupErrorCnt.incrementAndGet(), e); + } finally { + cleanupJobs.remove(broker); } } , delayed); @@ -1057,7 +1081,11 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce double cleanupTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); - // TODO: clean load data stores + + // clean load data stores + getContext().topBundleLoadDataStore().removeAsync(broker); + getContext().brokerLoadDataStore().removeAsync(broker); + log.info("Completed a cleanup for the inactive broker:{} in {} ms. " + "Cleaned up orphan service units: orphanServiceUnitCleanupCnt:{}, " + "approximate cleanupErrorCnt:{}, metrics:{} ", @@ -1066,7 +1094,7 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce orphanServiceUnitCleanupCnt, totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), printCleanupMetrics()); - cleanupJobs.remove(broker); + } private Optional getRollForwardStateData( @@ -1123,7 +1151,11 @@ protected void monitorOwnerships(List brokers) { return; } - log.info("Started the ownership monitor run for activeBrokerCount:{}", brokers.size()); + var debug = debug(); + if (debug) { + log.info("Started the ownership monitor run for activeBrokerCount:{}", brokers.size()); + } + long startTime = System.nanoTime(); Set inactiveBrokers = new HashSet<>(); Set activeBrokers = new HashSet<>(brokers); @@ -1199,27 +1231,32 @@ protected void monitorOwnerships(List brokers) { log.error("Failed to flush the in-flight messages.", e); } + boolean cleaned = false; if (serviceUnitTombstoneCleanupCnt > 0) { this.totalServiceUnitTombstoneCleanupCnt += serviceUnitTombstoneCleanupCnt; + cleaned = true; } if (orphanServiceUnitCleanupCnt > 0) { this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt; + cleaned = true; } - double monitorTime = TimeUnit.NANOSECONDS - .toMillis((System.nanoTime() - startTime)); - log.info("Completed the ownership monitor run in {} ms. " - + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. " - + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. " - + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. " - + "Approximate cleanupErrorCnt:{}, metrics:{}. ", - monitorTime, - inactiveBrokers, inactiveBrokers.size(), - orphanServiceUnitCleanupCnt, - serviceUnitTombstoneCleanupCnt, - totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), - printCleanupMetrics()); + if (debug || cleaned) { + double monitorTime = TimeUnit.NANOSECONDS + .toMillis((System.nanoTime() - startTime)); + log.info("Completed the ownership monitor run in {} ms. " + + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. " + + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. " + + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. " + + "Approximate cleanupErrorCnt:{}, metrics:{}. ", + monitorTime, + inactiveBrokers, inactiveBrokers.size(), + orphanServiceUnitCleanupCnt, + serviceUnitTombstoneCleanupCnt, + totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), + printCleanupMetrics()); + } } @@ -1353,4 +1390,8 @@ public void listen(StateChangeListener listener) { public Set> getOwnershipEntrySet() { return tableview.entrySet(); } + + public static ServiceUnitStateChannel get(PulsarService pulsar) { + return ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get()).getServiceUnitStateChannel(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java index 8b373bc5954b2..48665d39a0d3e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java @@ -75,6 +75,7 @@ public class BrokerLoadData { * The historical resource percentage is configured by loadBalancerHistoryResourcePercentage. */ private double weightedMaxEMA; + private double msgThroughputEMA; private long updatedAt; @Setter @@ -88,6 +89,7 @@ public BrokerLoadData() { bandwidthOut = new ResourceUsage(); maxResourceUsage = DEFAULT_RESOURCE_USAGE; weightedMaxEMA = DEFAULT_RESOURCE_USAGE; + msgThroughputEMA = 0; } /** @@ -142,6 +144,7 @@ public void update(final BrokerLoadData other) { bundleCount = other.bundleCount; topics = other.topics; weightedMaxEMA = other.weightedMaxEMA; + msgThroughputEMA = other.msgThroughputEMA; maxResourceUsage = other.maxResourceUsage; updatedAt = other.updatedAt; reportedAt = other.reportedAt; @@ -161,6 +164,7 @@ private void updateSystemResourceUsage(final ResourceUsage cpu, final ResourceUs private void updateFeatures(ServiceConfiguration conf) { updateMaxResourceUsage(); updateWeightedMaxEMA(conf); + updateMsgThroughputEMA(conf); } private void updateMaxResourceUsage() { @@ -188,6 +192,32 @@ private void updateWeightedMaxEMA(ServiceConfiguration conf) { weightedMaxEMA * historyPercentage + (1 - historyPercentage) * weightedMax; } + private void updateMsgThroughputEMA(ServiceConfiguration conf) { + var historyPercentage = conf.getLoadBalancerHistoryResourcePercentage(); + double msgThroughput = msgThroughputIn + msgThroughputOut; + msgThroughputEMA = updatedAt == 0 ? msgThroughput : + msgThroughputEMA * historyPercentage + (1 - historyPercentage) * msgThroughput; + } + + public void clear() { + cpu = new ResourceUsage(); + memory = new ResourceUsage(); + directMemory = new ResourceUsage(); + bandwidthIn = new ResourceUsage(); + bandwidthOut = new ResourceUsage(); + maxResourceUsage = DEFAULT_RESOURCE_USAGE; + weightedMaxEMA = DEFAULT_RESOURCE_USAGE; + msgThroughputEMA = 0; + msgThroughputIn = 0; + msgThroughputOut = 0; + msgRateIn = 0.0; + msgRateOut = 0.0; + bundleCount = 0; + topics = 0; + updatedAt = 0; + reportedAt = 0; + } + public String toString(ServiceConfiguration conf) { return String.format("cpu= %.2f%%, memory= %.2f%%, directMemory= %.2f%%, " + "bandwithIn= %.2f%%, bandwithOut= %.2f%%, " @@ -195,7 +225,7 @@ public String toString(ServiceConfiguration conf) { + "bandwithInResourceWeight= %f, bandwithOutResourceWeight= %f, " + "msgThroughputIn= %.2f, msgThroughputOut= %.2f, msgRateIn= %.2f, msgRateOut= %.2f, " + "bundleCount= %d, " - + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, " + + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, msgThroughputEMA= %.2f, " + "updatedAt= %d, reportedAt= %d", cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(), @@ -207,7 +237,7 @@ public String toString(ServiceConfiguration conf) { conf.getLoadBalancerBandwithOutResourceWeight(), msgThroughputIn, msgThroughputOut, msgRateIn, msgRateOut, bundleCount, - maxResourceUsage * 100, weightedMaxEMA * 100, + maxResourceUsage * 100, weightedMaxEMA * 100, msgThroughputEMA, updatedAt, reportedAt ); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java index d49361741bec4..2f5c32197c1fd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java @@ -86,7 +86,7 @@ public void update(Map bundleStats, int topk) { topk = Math.min(topk, arr.size()); partitionSort(arr, topk); - for (int i = 0; i < topk; i++) { + for (int i = topk - 1; i >= 0; i--) { var etr = arr.get(i); topKBundlesLoadData.add( new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue())); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java index 37483b58b53e1..4a5d41f7576ba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java @@ -22,8 +22,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; @@ -68,7 +68,7 @@ Overloaded, new AtomicLong(), Underloaded, new AtomicLong(), Admin, new AtomicLong()), Skip, Map.of( - Balanced, new AtomicLong(), + HitCount, new AtomicLong(), NoBundles, new AtomicLong(), CoolDown, new AtomicLong(), OutDatedData, new AtomicLong(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java index e1087ab6e53ba..1301b0708a5bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java @@ -41,7 +41,7 @@ public enum Label { public enum Reason { Overloaded, Underloaded, - Balanced, + HitCount, NoBundles, CoolDown, OutDatedData, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java index 256e52c4554d1..b07acfda7f77d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java @@ -18,15 +18,21 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.reporter; +import com.google.common.annotations.VisibleForTesting; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerHostUsage; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.impl.GenericBrokerHostUsageImpl; import org.apache.pulsar.broker.loadbalance.impl.LinuxBrokerHostUsageImpl; @@ -37,7 +43,9 @@ * The broker load data reporter. */ @Slf4j -public class BrokerLoadDataReporter implements LoadDataReporter { +public class BrokerLoadDataReporter implements LoadDataReporter, StateChangeListener { + + private static final long TOMBSTONE_DELAY_IN_MILLIS = 1000 * 10; private final PulsarService pulsar; @@ -54,6 +62,10 @@ public class BrokerLoadDataReporter implements LoadDataReporter private final BrokerLoadData lastData; + private volatile long lastTombstonedAt; + + private long tombstoneDelayInMillis; + public BrokerLoadDataReporter(PulsarService pulsar, String lookupServiceAddress, LoadDataStore brokerLoadDataStore) { @@ -68,6 +80,7 @@ public BrokerLoadDataReporter(PulsarService pulsar, } this.localData = new BrokerLoadData(); this.lastData = new BrokerLoadData(); + this.tombstoneDelayInMillis = TOMBSTONE_DELAY_IN_MILLIS; } @@ -92,8 +105,11 @@ public BrokerLoadData generateLoadData() { @Override public CompletableFuture reportAsync(boolean force) { BrokerLoadData newLoadData = this.generateLoadData(); + boolean debug = ExtensibleLoadManagerImpl.debug(conf, log); if (force || needBrokerDataUpdate()) { - log.info("publishing load report:{}", localData.toString(conf)); + if (debug) { + log.info("publishing load report:{}", localData.toString(conf)); + } CompletableFuture future = this.brokerLoadDataStore.pushAsync(this.lookupServiceAddress, newLoadData); future.whenComplete((__, ex) -> { @@ -106,7 +122,9 @@ public CompletableFuture reportAsync(boolean force) { }); return future; } else { - log.info("skipping load report:{}", localData.toString(conf)); + if (debug) { + log.info("skipping load report:{}", localData.toString(conf)); + } } return CompletableFuture.completedFuture(null); } @@ -117,10 +135,13 @@ private boolean needBrokerDataUpdate() { final long updateMaxIntervalMillis = TimeUnit.MINUTES .toMillis(loadBalancerReportUpdateMaxIntervalMinutes); long timeSinceLastReportWrittenToStore = System.currentTimeMillis() - localData.getReportedAt(); + boolean debug = ExtensibleLoadManagerImpl.debug(conf, log); if (timeSinceLastReportWrittenToStore > updateMaxIntervalMillis) { - log.info("Writing local data to metadata store because time since last" - + " update exceeded threshold of {} minutes", - loadBalancerReportUpdateMaxIntervalMinutes); + if (debug) { + log.info("Writing local data to metadata store because time since last" + + " update exceeded threshold of {} minutes", + loadBalancerReportUpdateMaxIntervalMinutes); + } // Always update after surpassing the maximum interval. return true; } @@ -133,10 +154,13 @@ private boolean needBrokerDataUpdate() { localData.getMsgThroughputIn() + localData.getMsgThroughputOut()), percentChange(lastData.getBundleCount(), localData.getBundleCount())))); if (maxChange > loadBalancerReportUpdateThresholdPercentage) { - log.info("Writing local data to metadata store because maximum change {}% exceeded threshold {}%; " - + "time since last report written is {} seconds", maxChange, - loadBalancerReportUpdateThresholdPercentage, - timeSinceLastReportWrittenToStore / 1000.0); + if (debug) { + log.info(String.format("Writing local data to metadata store " + + "because maximum change %.2f%% exceeded threshold %d%%. " + + "Time since last report written is %.2f%% seconds", maxChange, + loadBalancerReportUpdateThresholdPercentage, + timeSinceLastReportWrittenToStore / 1000.0)); + } return true; } return false; @@ -152,4 +176,50 @@ protected double percentChange(final double oldValue, final double newValue) { } return 100 * Math.abs((oldValue - newValue) / oldValue); } + + @VisibleForTesting + protected void tombstone() { + var now = System.currentTimeMillis(); + if (now - lastTombstonedAt < tombstoneDelayInMillis) { + return; + } + var lastSuccessfulTombstonedAt = lastTombstonedAt; + lastTombstonedAt = now; // dedup first + brokerLoadDataStore.removeAsync(lookupServiceAddress) + .whenComplete((__, e) -> { + if (e != null) { + log.error("Failed to clean broker load data.", e); + lastTombstonedAt = lastSuccessfulTombstonedAt; + } else { + boolean debug = ExtensibleLoadManagerImpl.debug(conf, log); + if (debug) { + log.info("Cleaned broker load data."); + } + } + } + ); + + } + + @Override + public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) { + if (t != null) { + return; + } + ServiceUnitState state = ServiceUnitStateData.state(data); + switch (state) { + case Releasing, Splitting -> { + if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) { + localData.clear(); + tombstone(); + } + } + case Owned -> { + if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) { + localData.clear(); + tombstone(); + } + } + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java index cc1a39add5d4e..0fa37d3687c20 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java @@ -18,10 +18,16 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.reporter; +import com.google.common.annotations.VisibleForTesting; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; @@ -29,7 +35,9 @@ * The top k highest-loaded bundles' load data reporter. */ @Slf4j -public class TopBundleLoadDataReporter implements LoadDataReporter { +public class TopBundleLoadDataReporter implements LoadDataReporter, StateChangeListener { + + private static final long TOMBSTONE_DELAY_IN_MILLIS = 1000 * 10; private final PulsarService pulsar; @@ -41,6 +49,9 @@ public class TopBundleLoadDataReporter implements LoadDataReporter bundleLoadDataStore) { @@ -49,6 +60,7 @@ public TopBundleLoadDataReporter(PulsarService pulsar, this.bundleLoadDataStore = bundleLoadDataStore; this.lastBundleStatsUpdatedAt = 0; this.topKBundles = new TopKBundles(pulsar); + this.tombstoneDelayInMillis = TOMBSTONE_DELAY_IN_MILLIS; } @Override @@ -60,8 +72,7 @@ public TopBundlesLoadData generateLoadData() { var pulsarStatsUpdatedAt = pulsarStats.getUpdatedAt(); if (pulsarStatsUpdatedAt > lastBundleStatsUpdatedAt) { var bundleStats = pulsar.getBrokerService().getBundleStats(); - double percentage = pulsar.getConfiguration().getLoadBalancerBundleLoadReportPercentage(); - int topk = Math.max(1, (int) (bundleStats.size() * percentage / 100.0)); + int topk = pulsar.getConfiguration().getLoadBalancerMaxNumberOfBundlesInBundleLoadReport(); topKBundles.update(bundleStats, topk); lastBundleStatsUpdatedAt = pulsarStatsUpdatedAt; result = topKBundles.getLoadData(); @@ -74,6 +85,9 @@ public TopBundlesLoadData generateLoadData() { public CompletableFuture reportAsync(boolean force) { var topBundlesLoadData = generateLoadData(); if (topBundlesLoadData != null || force) { + if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) { + log.info("Reporting TopBundlesLoadData:{}", topKBundles.getLoadData()); + } return this.bundleLoadDataStore.pushAsync(lookupServiceAddress, topKBundles.getLoadData()) .exceptionally(e -> { log.error("Failed to report top-bundles load data.", e); @@ -83,4 +97,47 @@ public CompletableFuture reportAsync(boolean force) { return CompletableFuture.completedFuture(null); } } + + @VisibleForTesting + protected void tombstone() { + var now = System.currentTimeMillis(); + if (now - lastTombstonedAt < tombstoneDelayInMillis) { + return; + } + var lastSuccessfulTombstonedAt = lastTombstonedAt; + lastTombstonedAt = now; // dedup first + bundleLoadDataStore.removeAsync(lookupServiceAddress) + .whenComplete((__, e) -> { + if (e != null) { + log.error("Failed to clean broker load data.", e); + lastTombstonedAt = lastSuccessfulTombstonedAt; + } else { + boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log); + if (debug) { + log.info("Cleaned broker load data."); + } + } + } + ); + } + + @Override + public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) { + if (t != null) { + return; + } + ServiceUnitState state = ServiceUnitStateData.state(data); + switch (state) { + case Releasing, Splitting -> { + if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) { + tombstone(); + } + } + case Owned -> { + if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) { + tombstone(); + } + } + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java index 42af396abcad4..421e6240c8f0d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Set; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; @@ -43,4 +44,11 @@ Set findBundlesForUnloading(LoadManagerContext context, Map recentlyUnloadedBundles, Map recentlyUnloadedBrokers); + /** + * Initializes the internals. + * + * @param pulsar The pulsar service instance. + */ + void initialize(PulsarService pulsar); + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java index 589df80fc5c14..816fde0038a58 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.StringJoiner; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -30,6 +31,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager; @@ -98,7 +100,7 @@ public SplitScheduler(PulsarService pulsar, @Override public void execute() { - boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); + boolean debugMode = ExtensibleLoadManagerImpl.debug(conf, log); if (debugMode) { log.info("Load balancer enabled: {}, Split enabled: {}.", conf.isLoadBalancerEnabled(), conf.isLoadBalancerAutoBundleSplitEnabled()); @@ -113,8 +115,10 @@ public void execute() { synchronized (bundleSplitStrategy) { final Set decisions = bundleSplitStrategy.findBundlesToSplit(context, pulsar); + if (debugMode) { + log.info("Split Decisions:", decisions); + } if (!decisions.isEmpty()) { - // currently following the unloading timeout var asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs(); List> futures = new ArrayList<>(); @@ -156,6 +160,14 @@ public void start() { task = loadManagerExecutor.scheduleAtFixedRate(() -> { try { execute(); + var debugMode = ExtensibleLoadManagerImpl.debug(conf, log); + if (debugMode) { + StringJoiner joiner = new StringJoiner("\n"); + joiner.add("### OwnershipEntrySet start ###"); + serviceUnitStateChannel.getOwnershipEntrySet().forEach(e -> joiner.add(e.toString())); + joiner.add("### OwnershipEntrySet end ###"); + log.info(joiner.toString()); + } } catch (Throwable e) { log.error("Failed to run the split job.", e); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index effddaf9c4159..669219daba273 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -20,8 +20,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; @@ -30,19 +30,26 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.MinMaxPriorityQueue; +import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.experimental.Accessors; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -80,17 +87,23 @@ * (loadBalancerMaxNumberOfBrokerTransfersPerCycle). * 9. Print more logs with a debug option(loadBalancerDebugModeEnabled=true). */ +@NoArgsConstructor public class TransferShedder implements NamespaceUnloadStrategy { private static final Logger log = LoggerFactory.getLogger(TransferShedder.class); private static final double KB = 1024; + private static final String CANNOT_CONTINUE_UNLOAD_MSG = "Can't continue the unload cycle."; + private static final String CANNOT_UNLOAD_BROKER_MSG = "Can't unload broker:%s."; + private static final String CANNOT_UNLOAD_BUNDLE_MSG = "Can't unload bundle:%s."; private final LoadStats stats = new LoadStats(); - private final PulsarService pulsar; - private final SimpleResourceAllocationPolicies allocationPolicies; - private final IsolationPoliciesHelper isolationPoliciesHelper; - private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; - - private final Set decisionCache; - private final UnloadCounter counter; + private PulsarService pulsar; + private SimpleResourceAllocationPolicies allocationPolicies; + private IsolationPoliciesHelper isolationPoliciesHelper; + private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; + private Set decisionCache; + @Getter + private UnloadCounter counter; + private ServiceUnitStateChannel channel; + private int unloadConditionHitCount = 0; @VisibleForTesting public TransferShedder(UnloadCounter counter){ @@ -110,9 +123,22 @@ public TransferShedder(PulsarService pulsar, this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); this.counter = counter; this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies); + this.channel = ServiceUnitStateChannelImpl.get(pulsar); this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper; } + @Override + public void initialize(PulsarService pulsar){ + this.pulsar = pulsar; + this.decisionCache = new HashSet<>(); + this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); + var manager = ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get()); + this.counter = manager.getUnloadCounter(); + this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies); + this.channel = ServiceUnitStateChannelImpl.get(pulsar); + this.antiAffinityGroupPolicyHelper = manager.getAntiAffinityGroupPolicyHelper(); + } + @Getter @Accessors(fluent = true) @@ -122,17 +148,14 @@ static class LoadStats { private int totalBrokers; private double avg; private double std; - private MinMaxPriorityQueue minBrokers; - private MinMaxPriorityQueue maxBrokers; private LoadDataStore loadDataStore; - + private List> brokersSortedByLoad; + int maxBrokerIndex; + int minBrokerIndex; + int numberOfBrokerSheddingPerCycle; + int maxNumberOfBrokerSheddingPerCycle; LoadStats() { - this.minBrokers = MinMaxPriorityQueue.orderedBy((a, b) -> Double.compare( - loadDataStore.get((String) b).get().getWeightedMaxEMA(), - loadDataStore.get((String) a).get().getWeightedMaxEMA())).create(); - this.maxBrokers = MinMaxPriorityQueue.orderedBy((a, b) -> Double.compare( - loadDataStore.get((String) a).get().getWeightedMaxEMA(), - loadDataStore.get((String) b).get().getWeightedMaxEMA())).create(); + brokersSortedByLoad = new ArrayList<>(); } private void update(double sum, double sqSum, int totalBrokers) { @@ -143,8 +166,7 @@ private void update(double sum, double sqSum, int totalBrokers) { if (totalBrokers == 0) { this.avg = 0; this.std = 0; - minBrokers.clear(); - maxBrokers.clear(); + } else { this.avg = sum / totalBrokers; this.std = Math.sqrt(sqSum / totalBrokers - avg * avg); @@ -156,34 +178,42 @@ void offload(double max, double min, double offload) { double maxd = Math.max(0, max - offload); double mind = min + offload; sqSum += maxd * maxd + mind * mind; - std = Math.sqrt(sqSum / totalBrokers - avg * avg); + std = Math.sqrt(Math.abs(sqSum / totalBrokers - avg * avg)); + numberOfBrokerSheddingPerCycle++; + minBrokerIndex++; } - void clear(){ + void clear() { sum = 0.0; sqSum = 0.0; totalBrokers = 0; avg = 0.0; std = 0.0; - minBrokers.clear(); - maxBrokers.clear(); + maxBrokerIndex = 0; + minBrokerIndex = 0; + numberOfBrokerSheddingPerCycle = 0; + maxNumberOfBrokerSheddingPerCycle = 0; + brokersSortedByLoad.clear(); + loadDataStore = null; } Optional update(final LoadDataStore loadStore, + final Map availableBrokers, Map recentlyUnloadedBrokers, final ServiceConfiguration conf) { - + maxNumberOfBrokerSheddingPerCycle = conf.getLoadBalancerMaxNumberOfBrokerSheddingPerCycle(); + var debug = ExtensibleLoadManagerImpl.debug(conf, log); UnloadDecision.Reason decisionReason = null; double sum = 0.0; double sqSum = 0.0; int totalBrokers = 0; - int maxTransfers = conf.getLoadBalancerMaxNumberOfBrokerTransfersPerCycle(); long now = System.currentTimeMillis(); + var missingLoadDataBrokers = new HashSet<>(availableBrokers.keySet()); for (Map.Entry entry : loadStore.entrySet()) { BrokerLoadData localBrokerData = entry.getValue(); String broker = entry.getKey(); - + missingLoadDataBrokers.remove(broker); // We don't want to use the outdated load data. if (now - localBrokerData.getUpdatedAt() > conf.getLoadBalancerBrokerLoadDataTTLInSeconds() * 1000) { @@ -196,12 +226,16 @@ Optional update(final LoadDataStore loadS // Also, we should give enough time for each broker to recompute its load after transfers. if (recentlyUnloadedBrokers.containsKey(broker)) { - if (localBrokerData.getUpdatedAt() - recentlyUnloadedBrokers.get(broker) - < conf.getLoadBalanceUnloadDelayInSeconds() * 1000) { - log.warn( - "Broker:{} load data timestamp:{} is too early since " - + "the last transfer timestamp:{}. Stop unloading.", - broker, localBrokerData.getUpdatedAt(), recentlyUnloadedBrokers.get(broker)); + var elapsed = localBrokerData.getUpdatedAt() - recentlyUnloadedBrokers.get(broker); + if (elapsed < conf.getLoadBalanceSheddingDelayInSeconds() * 1000) { + if (debug) { + log.warn( + "Broker:{} load data is too early since " + + "the last transfer. elapsed {} secs < threshold {} secs", + broker, + TimeUnit.MILLISECONDS.toSeconds(elapsed), + conf.getLoadBalanceSheddingDelayInSeconds()); + } update(0.0, 0.0, 0); return Optional.of(CoolDown); } else { @@ -211,25 +245,28 @@ Optional update(final LoadDataStore loadS double load = localBrokerData.getWeightedMaxEMA(); - minBrokers.offer(broker); - if (minBrokers.size() > maxTransfers) { - minBrokers.poll(); - } - maxBrokers.offer(broker); - if (maxBrokers.size() > maxTransfers) { - maxBrokers.poll(); - } sum += load; sqSum += load * load; totalBrokers++; } - if (totalBrokers == 0) { if (decisionReason == null) { decisionReason = NoBrokers; } update(0.0, 0.0, 0); + if (debug) { + log.info("There is no broker load data."); + } + return Optional.of(decisionReason); + } + + if (!missingLoadDataBrokers.isEmpty()) { + decisionReason = NoLoadData; + update(0.0, 0.0, 0); + if (debug) { + log.info("There is missing load data from brokers:{}", missingLoadDataBrokers); + } return Optional.of(decisionReason); } @@ -237,21 +274,35 @@ Optional update(final LoadDataStore loadS return Optional.empty(); } - boolean hasTransferableBrokers() { - return !(maxBrokers.isEmpty() || minBrokers.isEmpty() - || maxBrokers.peekLast().equals(minBrokers().peekLast())); - } - void setLoadDataStore(LoadDataStore loadDataStore) { this.loadDataStore = loadDataStore; + brokersSortedByLoad.addAll(loadDataStore.entrySet()); + brokersSortedByLoad.sort(Comparator.comparingDouble( + a -> a.getValue().getWeightedMaxEMA())); + maxBrokerIndex = brokersSortedByLoad.size() - 1; + minBrokerIndex = 0; + } + + String peekMinBroker() { + return brokersSortedByLoad.get(minBrokerIndex).getKey(); + } + + String pollMaxBroker() { + return brokersSortedByLoad.get(maxBrokerIndex--).getKey(); } @Override public String toString() { return String.format( - "sum:%.2f, sqSum:%.2f, avg:%.2f, std:%.2f, totalBrokers:%d, " - + "minBrokers:%s, maxBrokers:%s", - sum, sqSum, avg, std, totalBrokers, minBrokers, maxBrokers); + "sum:%.2f, sqSum:%.2f, avg:%.2f, std:%.2f, totalBrokers:%d, brokersSortedByLoad:%s", + sum, sqSum, avg, std, totalBrokers, + brokersSortedByLoad.stream().map(v->v.getKey()).collect(Collectors.toList())); + } + + + boolean hasTransferableBrokers() { + return numberOfBrokerSheddingPerCycle < maxNumberOfBrokerSheddingPerCycle + && minBrokerIndex < maxBrokerIndex; } } @@ -263,20 +314,36 @@ public Set findBundlesForUnloading(LoadManagerContext context, final var conf = context.brokerConfiguration(); decisionCache.clear(); stats.clear(); + Map availableBrokers; + try { + availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync() + .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + counter.update(Failure, Unknown); + log.warn("Failed to fetch available brokers. Stop unloading.", e); + return decisionCache; + } try { final var loadStore = context.brokerLoadDataStore(); stats.setLoadDataStore(loadStore); - boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); + boolean debugMode = ExtensibleLoadManagerImpl.debug(conf, log); - var skipReason = stats.update(context.brokerLoadDataStore(), recentlyUnloadedBrokers, conf); + var skipReason = stats.update( + context.brokerLoadDataStore(), availableBrokers, recentlyUnloadedBrokers, conf); if (skipReason.isPresent()) { - log.warn("Failed to update load stat. Reason:{}. Stop unloading.", skipReason.get()); + if (debugMode) { + log.warn(CANNOT_CONTINUE_UNLOAD_MSG + + " Skipped the load stat update. Reason:{}.", + skipReason.get()); + } counter.update(Skip, skipReason.get()); return decisionCache; } counter.updateLoadData(stats.avg, stats.std); + + if (debugMode) { log.info("brokers' load stats:{}", stats); } @@ -287,138 +354,282 @@ public Set findBundlesForUnloading(LoadManagerContext context, final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd(); boolean transfer = conf.isLoadBalancerTransferEnabled(); + if (stats.std() > targetStd || isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) { + unloadConditionHitCount++; + } else { + unloadConditionHitCount = 0; + } - Map availableBrokers; - try { - availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync() - .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); - } catch (ExecutionException | InterruptedException | TimeoutException e) { - counter.update(Skip, Unknown); - log.warn("Failed to fetch available brokers. Reason: Unknown. Stop unloading.", e); + if (unloadConditionHitCount <= conf.getLoadBalancerSheddingConditionHitCountThreshold()) { + if (debugMode) { + log.info(CANNOT_CONTINUE_UNLOAD_MSG + + " Shedding condition hit count:{} is less than or equal to the threshold:{}.", + unloadConditionHitCount, conf.getLoadBalancerSheddingConditionHitCountThreshold()); + } + counter.update(Skip, HitCount); return decisionCache; } while (true) { if (!stats.hasTransferableBrokers()) { if (debugMode) { - log.info("Exhausted target transfer brokers. Stop unloading"); + log.info(CANNOT_CONTINUE_UNLOAD_MSG + + " Exhausted target transfer brokers."); } break; } UnloadDecision.Reason reason; if (stats.std() <= targetStd) { - if (hasMsgThroughput(context, stats.minBrokers.peekLast())) { + if (!isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) { if (debugMode) { - log.info("std:{} <= targetStd:{} and minBroker:{} has msg throughput. Stop unloading.", - stats.std, targetStd, stats.minBrokers.peekLast()); + log.info(CANNOT_CONTINUE_UNLOAD_MSG + + "The overall cluster load meets the target, std:{} <= targetStd:{}," + + " and minBroker:{} is not underloaded.", + stats.std(), targetStd, stats.peekMinBroker()); } break; } else { reason = Underloaded; + if (debugMode) { + log.info(String.format("broker:%s is underloaded:%s although " + + "load std:%.2f <= targetStd:%.2f. " + + "Continuing unload for this underloaded broker.", + stats.peekMinBroker(), + context.brokerLoadDataStore().get(stats.peekMinBroker()).get(), + stats.std(), targetStd)); + } } } else { reason = Overloaded; } - String maxBroker = stats.maxBrokers().pollLast(); - String minBroker = stats.minBrokers().pollLast(); + String maxBroker = stats.pollMaxBroker(); + String minBroker = stats.peekMinBroker(); Optional maxBrokerLoadData = context.brokerLoadDataStore().get(maxBroker); Optional minBrokerLoadData = context.brokerLoadDataStore().get(minBroker); if (maxBrokerLoadData.isEmpty()) { - log.error("maxBroker:{} maxBrokerLoadData is empty. Skip unloading from this max broker.", - maxBroker); + log.error(String.format(CANNOT_UNLOAD_BROKER_MSG + + " MaxBrokerLoadData is empty.", maxBroker)); numOfBrokersWithEmptyLoadData++; continue; } if (minBrokerLoadData.isEmpty()) { - log.error("minBroker:{} minBrokerLoadData is empty. Skip unloading to this min broker.", minBroker); + log.error("Can't transfer load to broker:{}. MinBrokerLoadData is empty.", minBroker); numOfBrokersWithEmptyLoadData++; continue; } - - double max = maxBrokerLoadData.get().getWeightedMaxEMA(); - double min = minBrokerLoadData.get().getWeightedMaxEMA(); - double offload = (max - min) / 2; + double maxLoad = maxBrokerLoadData.get().getWeightedMaxEMA(); + double minLoad = minBrokerLoadData.get().getWeightedMaxEMA(); + double offload = (maxLoad - minLoad) / 2; BrokerLoadData brokerLoadData = maxBrokerLoadData.get(); - double brokerThroughput = brokerLoadData.getMsgThroughputIn() + brokerLoadData.getMsgThroughputOut(); - double offloadThroughput = brokerThroughput * offload; + double maxBrokerThroughput = brokerLoadData.getMsgThroughputIn() + + brokerLoadData.getMsgThroughputOut(); + double minBrokerThroughput = minBrokerLoadData.get().getMsgThroughputIn() + + minBrokerLoadData.get().getMsgThroughputOut(); + double offloadThroughput = maxBrokerThroughput * offload / maxLoad; if (debugMode) { - log.info( - "Attempting to shed load from broker:{}{}, which has the max resource " - + "usage {}%, targetStd:{}," - + " -- Offloading {}%, at least {} KByte/s of traffic, left throughput {} KByte/s", + log.info(String.format( + "Attempting to shed load from broker:%s%s, which has the max resource " + + "usage:%.2f%%, targetStd:%.2f," + + " -- Trying to offload %.2f%%, %.2f KByte/s of traffic.", maxBroker, transfer ? " to broker:" + minBroker : "", - 100 * max, targetStd, - offload * 100, offloadThroughput / KB, (brokerThroughput - offloadThroughput) / KB); + maxLoad * 100, + targetStd, + offload * 100, + offloadThroughput / KB + )); } double trafficMarkedToOffload = 0; - boolean atLeastOneBundleSelected = false; + double trafficMarkedToGain = 0; Optional bundlesLoadData = context.topBundleLoadDataStore().get(maxBroker); if (bundlesLoadData.isEmpty() || bundlesLoadData.get().getTopBundlesLoadData().isEmpty()) { - log.error("maxBroker:{} topBundlesLoadData is empty. Skip unloading from this broker.", maxBroker); + log.error(String.format(CANNOT_UNLOAD_BROKER_MSG + + " TopBundlesLoadData is empty.", maxBroker)); numOfBrokersWithEmptyLoadData++; continue; } - var topBundlesLoadData = bundlesLoadData.get().getTopBundlesLoadData(); - if (topBundlesLoadData.size() > 1) { - int remainingTopBundles = topBundlesLoadData.size(); - for (var e : topBundlesLoadData) { - String bundle = e.bundleName(); - if (!recentlyUnloadedBundles.containsKey(bundle) - && isTransferable(context, availableBrokers, - bundle, maxBroker, Optional.of(minBroker))) { - var bundleData = e.stats(); - double throughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut; - if (remainingTopBundles > 1 - && (trafficMarkedToOffload < offloadThroughput - || !atLeastOneBundleSelected)) { - Unload unload; - if (transfer) { - unload = new Unload(maxBroker, bundle, Optional.of(minBroker)); - } else { - unload = new Unload(maxBroker, bundle); + var maxBrokerTopBundlesLoadData = bundlesLoadData.get().getTopBundlesLoadData(); + if (maxBrokerTopBundlesLoadData.size() == 1) { + numOfBrokersWithFewBundles++; + log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG + + " Sole namespace bundle:%s is overloading the broker. ", + maxBroker, maxBrokerTopBundlesLoadData.iterator().next())); + continue; + } + Optional minBundlesLoadData = context.topBundleLoadDataStore().get(minBroker); + var minBrokerTopBundlesLoadDataIter = + minBundlesLoadData.isPresent() ? minBundlesLoadData.get().getTopBundlesLoadData().iterator() : + null; + + + if (maxBrokerTopBundlesLoadData.isEmpty()) { + numOfBrokersWithFewBundles++; + log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG + + " Broker overloaded despite having no bundles", maxBroker)); + continue; + } + + int remainingTopBundles = maxBrokerTopBundlesLoadData.size(); + for (var e : maxBrokerTopBundlesLoadData) { + String bundle = e.bundleName(); + if (channel != null && !channel.isOwner(bundle, maxBroker)) { + if (debugMode) { + log.warn(String.format(CANNOT_UNLOAD_BUNDLE_MSG + + " MaxBroker:%s is not the owner.", bundle, maxBroker)); + } + continue; + } + if (recentlyUnloadedBundles.containsKey(bundle)) { + if (debugMode) { + log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG + + " Bundle has been recently unloaded at ts:%d.", + bundle, recentlyUnloadedBundles.get(bundle))); + } + continue; + } + if (!isTransferable(context, availableBrokers, bundle, maxBroker, Optional.of(minBroker))) { + if (debugMode) { + log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG + + " This unload can't meet " + + "affinity(isolation) or anti-affinity group policies.", bundle)); + } + continue; + } + if (remainingTopBundles <= 1) { + if (debugMode) { + log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG + + " The remaining bundles in TopBundlesLoadData from the maxBroker:%s is" + + " less than or equal to 1.", + bundle, maxBroker)); + } + break; + } + + var bundleData = e.stats(); + double maxBrokerBundleThroughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut; + boolean swap = false; + List minToMaxUnloads = new ArrayList<>(); + double minBrokerBundleSwapThroughput = 0.0; + if (trafficMarkedToOffload - trafficMarkedToGain + maxBrokerBundleThroughput > offloadThroughput) { + // see if we can swap bundles from min to max broker to balance better. + if (transfer && minBrokerTopBundlesLoadDataIter != null) { + var maxBrokerNewThroughput = + maxBrokerThroughput - trafficMarkedToOffload + trafficMarkedToGain + - maxBrokerBundleThroughput; + var minBrokerNewThroughput = + minBrokerThroughput + trafficMarkedToOffload - trafficMarkedToGain + + maxBrokerBundleThroughput; + while (minBrokerTopBundlesLoadDataIter.hasNext()) { + var minBrokerBundleData = minBrokerTopBundlesLoadDataIter.next(); + if (!isTransferable(context, availableBrokers, + minBrokerBundleData.bundleName(), minBroker, Optional.of(maxBroker))) { + continue; + } + var minBrokerBundleThroughput = + minBrokerBundleData.stats().msgThroughputIn + + minBrokerBundleData.stats().msgThroughputOut; + var maxBrokerNewThroughputTmp = maxBrokerNewThroughput + minBrokerBundleThroughput; + var minBrokerNewThroughputTmp = minBrokerNewThroughput - minBrokerBundleThroughput; + if (maxBrokerNewThroughputTmp < maxBrokerThroughput + && minBrokerNewThroughputTmp < maxBrokerThroughput) { + minToMaxUnloads.add(new Unload(minBroker, + minBrokerBundleData.bundleName(), Optional.of(maxBroker))); + maxBrokerNewThroughput = maxBrokerNewThroughputTmp; + minBrokerNewThroughput = minBrokerNewThroughputTmp; + minBrokerBundleSwapThroughput += minBrokerBundleThroughput; + if (minBrokerNewThroughput <= maxBrokerNewThroughput + && maxBrokerNewThroughput < maxBrokerThroughput * 0.75) { + swap = true; + break; + } + } + } + } + if (!swap) { + if (debugMode) { + log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG + + " The traffic to unload:%.2f - gain:%.2f = %.2f KByte/s is " + + "greater than the target :%.2f KByte/s.", + bundle, + (trafficMarkedToOffload + maxBrokerBundleThroughput) / KB, + trafficMarkedToGain / KB, + (trafficMarkedToOffload - trafficMarkedToGain + maxBrokerBundleThroughput) / KB, + offloadThroughput / KB)); + } + break; + } + } + Unload unload; + if (transfer) { + if (swap) { + minToMaxUnloads.forEach(minToMaxUnload -> { + if (debugMode) { + log.info("Decided to gain bundle:{} from min broker:{}", + minToMaxUnload.serviceUnit(), minToMaxUnload.sourceBroker()); } var decision = new UnloadDecision(); - decision.setUnload(unload); + decision.setUnload(minToMaxUnload); decision.succeed(reason); decisionCache.add(decision); - trafficMarkedToOffload += throughput; - atLeastOneBundleSelected = true; - remainingTopBundles--; + }); + if (debugMode) { + log.info(String.format( + "Total traffic %.2f KByte/s to transfer from min broker:%s to max broker:%s.", + minBrokerBundleSwapThroughput / KB, minBroker, maxBroker)); + trafficMarkedToGain += minBrokerBundleSwapThroughput; } } + unload = new Unload(maxBroker, bundle, Optional.of(minBroker)); + } else { + unload = new Unload(maxBroker, bundle); } - if (!atLeastOneBundleSelected) { - numOfBrokersWithFewBundles++; + var decision = new UnloadDecision(); + decision.setUnload(unload); + decision.succeed(reason); + decisionCache.add(decision); + trafficMarkedToOffload += maxBrokerBundleThroughput; + remainingTopBundles--; + + if (debugMode) { + log.info(String.format("Decided to unload bundle:%s, throughput:%.2f KByte/s." + + " The traffic marked to unload:%.2f - gain:%.2f = %.2f KByte/s." + + " Target:%.2f KByte/s.", + bundle, maxBrokerBundleThroughput / KB, + trafficMarkedToOffload / KB, + trafficMarkedToGain / KB, + (trafficMarkedToOffload - trafficMarkedToGain) / KB, + offloadThroughput / KB)); } - } else if (topBundlesLoadData.size() == 1) { - numOfBrokersWithFewBundles++; - log.warn( - "HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. " - + "No Load Shedding will be done on this broker", - topBundlesLoadData.iterator().next(), maxBroker); - } else { - numOfBrokersWithFewBundles++; - log.warn("Broker {} is overloaded despite having no bundles", maxBroker); } - if (trafficMarkedToOffload > 0) { - stats.offload(max, min, offload); + var adjustedOffload = + (trafficMarkedToOffload - trafficMarkedToGain) * maxLoad / maxBrokerThroughput; + stats.offload(maxLoad, minLoad, adjustedOffload); if (debugMode) { log.info( String.format("brokers' load stats:%s, after offload{max:%.2f, min:%.2f, offload:%.2f}", - stats, max, min, offload)); + stats, maxLoad, minLoad, adjustedOffload)); } + } else { + numOfBrokersWithFewBundles++; + log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG + + " There is no bundle that can be unloaded in top bundles load data. " + + "Consider splitting bundles owned by the broker " + + "to make each bundle serve less traffic " + + "or increasing loadBalancerMaxNumberOfBundlesInBundleLoadReport" + + " to report more bundles in the top bundles load data.", maxBroker)); } - } + + } // while end if (debugMode) { log.info("decisionCache:{}", decisionCache); } + if (decisionCache.isEmpty()) { UnloadDecision.Reason reason; if (numOfBrokersWithEmptyLoadData > 0) { @@ -426,10 +637,13 @@ && isTransferable(context, availableBrokers, } else if (numOfBrokersWithFewBundles > 0) { reason = NoBundles; } else { - reason = Balanced; + reason = HitCount; } counter.update(Skip, reason); + } else { + unloadConditionHitCount = 0; } + } catch (Throwable e) { log.error("Failed to process unloading. ", e); this.counter.update(Failure, Unknown); @@ -438,13 +652,19 @@ && isTransferable(context, availableBrokers, } - private boolean hasMsgThroughput(LoadManagerContext context, String broker) { + private boolean isUnderLoaded(LoadManagerContext context, String broker, double avgLoad) { var brokerLoadDataOptional = context.brokerLoadDataStore().get(broker); if (brokerLoadDataOptional.isEmpty()) { return false; } var brokerLoadData = brokerLoadDataOptional.get(); - return brokerLoadData.getMsgThroughputIn() + brokerLoadData.getMsgThroughputOut() > 0.0; + if (brokerLoadData.getMsgThroughputEMA() < 1) { + return true; + } + + return brokerLoadData.getWeightedMaxEMA() + < avgLoad * Math.min(0.5, Math.max(0.0, + context.brokerConfiguration().getLoadBalancerBrokerLoadTargetStd() / 2)); } @@ -467,7 +687,8 @@ private boolean isTransferable(LoadManagerContext context, return false; } - if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) { + if (antiAffinityGroupPolicyHelper != null + && !antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) { return false; } return true; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java index 31310c5c9cc80..d6c754c90fcf6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java @@ -40,7 +40,6 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; -import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Reflections; @@ -81,11 +80,10 @@ public UnloadScheduler(PulsarService pulsar, UnloadManager unloadManager, LoadManagerContext context, ServiceUnitStateChannel channel, - AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper, UnloadCounter counter, AtomicReference> unloadMetrics) { this(pulsar, loadManagerExecutor, unloadManager, context, channel, - createNamespaceUnloadStrategy(pulsar, antiAffinityGroupPolicyHelper, counter), counter, unloadMetrics); + createNamespaceUnloadStrategy(pulsar), counter, unloadMetrics); } @VisibleForTesting @@ -212,19 +210,22 @@ public void close() { this.recentlyUnloadedBrokers.clear(); } - private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar, - AntiAffinityGroupPolicyHelper helper, - UnloadCounter counter) { + private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar) { ServiceConfiguration conf = pulsar.getConfiguration(); + NamespaceUnloadStrategy unloadStrategy; try { - return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), NamespaceUnloadStrategy.class, + unloadStrategy = Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), + NamespaceUnloadStrategy.class, Thread.currentThread().getContextClassLoader()); + log.info("Created namespace unload strategy:{}", unloadStrategy.getClass().getCanonicalName()); } catch (Exception e) { log.error("Error when trying to create namespace unload strategy: {}", conf.getLoadBalancerLoadPlacementStrategy(), e); + log.error("create namespace unload strategy failed. using TransferShedder instead."); + unloadStrategy = new TransferShedder(); } - log.error("create namespace unload strategy failed. using TransferShedder instead."); - return new TransferShedder(pulsar, counter, helper); + unloadStrategy.initialize(pulsar); + return unloadStrategy; } private boolean isLoadBalancerSheddingEnabled() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java index da04a287f1ac5..15bfdc747f1fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java @@ -32,6 +32,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; @@ -47,15 +48,17 @@ */ @Slf4j public class DefaultNamespaceBundleSplitStrategyImpl implements NamespaceBundleSplitStrategy { + private static final String CANNOT_CONTINUE_SPLIT_MSG = "Can't continue the split cycle."; + private static final String CANNOT_SPLIT_BUNDLE_MSG = "Can't split broker:%s."; private final Set decisionCache; private final Map namespaceBundleCount; - private final Map bundleHighTrafficFrequency; + private final Map splitConditionHitCounts; private final SplitCounter counter; public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) { decisionCache = new HashSet<>(); namespaceBundleCount = new HashMap<>(); - bundleHighTrafficFrequency = new HashMap<>(); + splitConditionHitCounts = new HashMap<>(); this.counter = counter; } @@ -71,22 +74,33 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate(); long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI; long maxSplitCount = conf.getLoadBalancerMaxNumberOfBundlesToSplitPerCycle(); - long splitConditionThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionThreshold(); + long splitConditionHitCountThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(); boolean debug = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled(); + var channel = ServiceUnitStateChannelImpl.get(pulsar); Map bundleStatsMap = pulsar.getBrokerService().getBundleStats(); NamespaceBundleFactory namespaceBundleFactory = pulsar.getNamespaceService().getNamespaceBundleFactory(); - // clean bundleHighTrafficFrequency - bundleHighTrafficFrequency.keySet().retainAll(bundleStatsMap.keySet()); + // clean splitConditionHitCounts + splitConditionHitCounts.keySet().retainAll(bundleStatsMap.keySet()); for (var entry : bundleStatsMap.entrySet()) { final String bundle = entry.getKey(); final NamespaceBundleStats stats = entry.getValue(); if (stats.topics < 2) { if (debug) { - log.info("The count of topics on the bundle {} is less than 2, skip split!", bundle); + log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG + + " The topic count is less than 2.", bundle)); + } + continue; + } + + if (!channel.isOwner(bundle)) { + if (debug) { + log.error(String.format(CANNOT_SPLIT_BUNDLE_MSG + + " This broker is not the owner.", bundle)); + counter.update(Failure, Unknown); } continue; } @@ -96,7 +110,8 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS if (!namespaceBundleFactory .canSplitBundle(namespaceBundleFactory.getBundle(namespaceName, bundleRange))) { if (debug) { - log.info("Can't split the bundle:{}. invalid bundle range:{}. ", bundle, bundleRange); + log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG + + " Invalid bundle range:%s.", bundle, bundleRange)); } counter.update(Failure, Unknown); continue; @@ -117,54 +132,85 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS } if (reason != Unknown) { - bundleHighTrafficFrequency.put(bundle, bundleHighTrafficFrequency.getOrDefault(bundle, 0) + 1); + splitConditionHitCounts.put(bundle, splitConditionHitCounts.getOrDefault(bundle, 0) + 1); } else { - bundleHighTrafficFrequency.remove(bundle); + splitConditionHitCounts.remove(bundle); + } + + if (splitConditionHitCounts.getOrDefault(bundle, 0) <= splitConditionHitCountThreshold) { + if (debug) { + log.info(String.format( + CANNOT_SPLIT_BUNDLE_MSG + + " Split condition hit count: %d is" + + " less than or equal to threshold: %d. " + + "Topics: %d/%d, " + + "Sessions: (%d+%d)/%d, " + + "Message Rate: %.2f/%d (msgs/s), " + + "Message Throughput: %.2f/%d (MB/s).", + bundle, + splitConditionHitCounts.getOrDefault(bundle, 0), + splitConditionHitCountThreshold, + stats.topics, maxBundleTopics, + stats.producerCount, stats.consumerCount, maxBundleSessions, + totalMessageRate, maxBundleMsgRate, + totalMessageThroughput / LoadManagerShared.MIBI, + maxBundleBandwidth / LoadManagerShared.MIBI + )); + } + continue; } - if (bundleHighTrafficFrequency.getOrDefault(bundle, 0) > splitConditionThreshold) { - final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle); - try { - final int bundleCount = pulsar.getNamespaceService() - .getBundleCount(NamespaceName.get(namespace)); - if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0)) - < maxBundleCount) { - if (debug) { - log.info("The bundle {} is considered to split. Topics: {}/{}, Sessions: ({}+{})/{}, " - + "Message Rate: {}/{} (msgs/s), Message Throughput: {}/{} (MB/s)", - bundle, stats.topics, maxBundleTopics, stats.producerCount, stats.consumerCount, - maxBundleSessions, totalMessageRate, maxBundleMsgRate, - totalMessageThroughput / LoadManagerShared.MIBI, - maxBundleBandwidth / LoadManagerShared.MIBI); - } - var decision = new SplitDecision(); - decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId())); - decision.succeed(reason); - decisionCache.add(decision); - int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0); - namespaceBundleCount.put(namespace, bundleNum + 1); - bundleHighTrafficFrequency.remove(bundle); - // Clear namespace bundle-cache - namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName)); - if (decisionCache.size() == maxSplitCount) { - if (debug) { - log.info("Too many bundles to split in this split cycle {} / {}. Stop.", - decisionCache.size(), maxSplitCount); - } - break; - } - } else { - if (debug) { - log.info( - "Could not split namespace bundle {} because namespace {} has too many bundles:" - + "{}", bundle, namespace, bundleCount); - } + final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + try { + final int bundleCount = pulsar.getNamespaceService() + .getBundleCount(NamespaceName.get(namespace)); + if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0)) + >= maxBundleCount) { + if (debug) { + log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG + " Namespace:%s has too many bundles:%d", + bundle, namespace, bundleCount)); } - } catch (Exception e) { - counter.update(Failure, Unknown); - log.warn("Error while computing bundle splits for namespace {}", namespace, e); + continue; + } + } catch (Exception e) { + counter.update(Failure, Unknown); + log.warn("Failed to get bundle count in namespace:{}", namespace, e); + continue; + } + + if (debug) { + log.info(String.format( + "Splitting bundle: %s. " + + "Topics: %d/%d, " + + "Sessions: (%d+%d)/%d, " + + "Message Rate: %.2f/%d (msgs/s), " + + "Message Throughput: %.2f/%d (MB/s)", + bundle, + stats.topics, maxBundleTopics, + stats.producerCount, stats.consumerCount, maxBundleSessions, + totalMessageRate, maxBundleMsgRate, + totalMessageThroughput / LoadManagerShared.MIBI, + maxBundleBandwidth / LoadManagerShared.MIBI + )); + } + var decision = new SplitDecision(); + decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId())); + decision.succeed(reason); + decisionCache.add(decision); + int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0); + namespaceBundleCount.put(namespace, bundleNum + 1); + splitConditionHitCounts.remove(bundle); + // Clear namespace bundle-cache + namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName)); + if (decisionCache.size() == maxSplitCount) { + if (debug) { + log.info(CANNOT_CONTINUE_SPLIT_MSG + + "Too many bundles split in this cycle {} / {}.", + decisionCache.size(), maxSplitCount); } + break; } + } return decisionCache; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java index 678927dac9293..902cfdaf73f5f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java @@ -82,7 +82,7 @@ public Optional select( Set candidates, ServiceUnitId bundleToAssign, LoadManagerContext context) { var conf = context.brokerConfiguration(); if (candidates.isEmpty()) { - log.info("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign); + log.warn("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign); return Optional.empty(); } @@ -131,8 +131,10 @@ public Optional select( if (bestBrokers.isEmpty()) { // Assign randomly as all brokers are overloaded. - log.warn("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, " - + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size()); + if (debugMode) { + log.info("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, " + + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size()); + } for (String broker : candidates) { bestBrokers.add(broker); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index f8a7a9b629f4e..4c57e6b93fa3c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -26,8 +26,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; @@ -671,7 +671,7 @@ public void testGetMetrics() throws Exception { put(Underloaded, new AtomicLong(2)); }}, Skip, new LinkedHashMap<>() {{ - put(Balanced, new AtomicLong(3)); + put(HitCount, new AtomicLong(3)); put(NoBundles, new AtomicLong(4)); put(CoolDown, new AtomicLong(5)); put(OutDatedData, new AtomicLong(6)); @@ -756,7 +756,7 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) dimensions=[{broker=localhost, feature=max, metric=loadBalancing}], metrics=[{brk_lb_resource_usage=0.04}] dimensions=[{broker=localhost, metric=bundleUnloading}], metrics=[{brk_lb_unload_broker_total=2, brk_lb_unload_bundle_total=3}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Failure}], metrics=[{brk_lb_unload_broker_breakdown_total=10}] - dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=HitCount, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=CoolDown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=5}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=OutDatedData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=6}] diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index c05c8ac741565..9eda98e5d842d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -78,6 +78,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -132,6 +133,8 @@ protected void setup() throws Exception { pulsar1 = pulsar; registry = new BrokerRegistryImpl(pulsar); loadManagerContext = mock(LoadManagerContext.class); + doReturn(mock(LoadDataStore.class)).when(loadManagerContext).brokerLoadDataStore(); + doReturn(mock(LoadDataStore.class)).when(loadManagerContext).topBundleLoadDataStore(); brokerSelector = mock(BrokerSelectionStrategy.class); additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf()); pulsar2 = additionalPulsarTestContext.getPulsarService(); @@ -1208,6 +1211,69 @@ public void splitTestWhenDestBrokerFails() } + @Test(priority = 15) + public void testIsOwner() throws IllegalAccessException { + + var owner1 = channel1.isOwner(bundle); + var owner2 = channel2.isOwner(bundle); + + assertFalse(owner1); + assertFalse(owner2); + + owner1 = channel1.isOwner(bundle, lookupServiceAddress2); + owner2 = channel2.isOwner(bundle, lookupServiceAddress1); + + assertFalse(owner1); + assertFalse(owner2); + + channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + owner2 = channel2.isOwner(bundle); + assertFalse(owner2); + + waitUntilOwnerChanges(channel1, bundle, null); + waitUntilOwnerChanges(channel2, bundle, null); + + owner1 = channel1.isOwner(bundle); + owner2 = channel2.isOwner(bundle); + + assertTrue(owner1); + assertFalse(owner2); + + owner1 = channel1.isOwner(bundle, lookupServiceAddress1); + owner2 = channel2.isOwner(bundle, lookupServiceAddress2); + + assertTrue(owner1); + assertFalse(owner2); + + owner1 = channel2.isOwner(bundle, lookupServiceAddress1); + owner2 = channel1.isOwner(bundle, lookupServiceAddress2); + + assertTrue(owner1); + assertFalse(owner2); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, lookupServiceAddress1, 1)); + assertFalse(channel1.isOwner(bundle)); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, lookupServiceAddress1, 1)); + assertTrue(channel1.isOwner(bundle)); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, lookupServiceAddress1, 1)); + assertFalse(channel1.isOwner(bundle)); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, lookupServiceAddress1, 1)); + assertTrue(channel1.isOwner(bundle)); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, lookupServiceAddress1, 1)); + assertFalse(channel1.isOwner(bundle)); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, lookupServiceAddress1, 1)); + assertFalse(channel1.isOwner(bundle)); + + overrideTableView(channel1, bundle, null); + assertFalse(channel1.isOwner(bundle)); + } + + private static ConcurrentOpenHashMap>> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { return (ConcurrentOpenHashMap>>) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java index 5ba7629dd1132..85792a7ba9387 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java @@ -74,6 +74,7 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getTopics(), 6); assertEquals(data.getMaxResourceUsage(), 0.04); // skips memory usage assertEquals(data.getWeightedMaxEMA(), 2); + assertEquals(data.getMsgThroughputEMA(), 3); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); now = System.currentTimeMillis(); @@ -103,6 +104,7 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getTopics(), 10); assertEquals(data.getMaxResourceUsage(), 3.0); assertEquals(data.getWeightedMaxEMA(), 1.875); + assertEquals(data.getMsgThroughputEMA(), 5); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); assertEquals(data.getReportedAt(), 0l); assertEquals(data.toString(conf), "cpu= 300.00%, memory= 100.00%, directMemory= 2.00%, " @@ -111,8 +113,11 @@ public void testUpdateBySystemResourceUsage() { + "bandwithInResourceWeight= 0.500000, bandwithOutResourceWeight= 0.500000, " + "msgThroughputIn= 5.00, msgThroughputOut= 6.00, " + "msgRateIn= 7.00, msgRateOut= 8.00, bundleCount= 9, " - + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, " + + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, msgThroughputEMA= 5.00, " + "updatedAt= " + data.getUpdatedAt() + ", reportedAt= " + data.getReportedAt()); + + data.clear(); + assertEquals(data, new BrokerLoadData()); } @Test @@ -143,6 +148,9 @@ public void testUpdateByBrokerLoadData() { data.update(other); assertEquals(data, other); + + data.clear(); + assertEquals(data, new BrokerLoadData()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java index 9b42163bd664b..472d44df8906d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java @@ -87,15 +87,15 @@ public void testTopBundlesLoadData() { Map bundleStats = new HashMap<>(); var topKBundles = new TopKBundles(pulsar); NamespaceBundleStats stats1 = new NamespaceBundleStats(); - stats1.msgRateIn = 500; + stats1.msgRateIn = 100000; bundleStats.put(bundle1, stats1); NamespaceBundleStats stats2 = new NamespaceBundleStats(); - stats2.msgRateIn = 10000; + stats2.msgRateIn = 500; bundleStats.put(bundle2, stats2); NamespaceBundleStats stats3 = new NamespaceBundleStats(); - stats3.msgRateIn = 100000; + stats3.msgRateIn = 10000; bundleStats.put(bundle3, stats3); NamespaceBundleStats stats4 = new NamespaceBundleStats(); @@ -107,8 +107,8 @@ public void testTopBundlesLoadData() { var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1); var top2 = topKBundles.getLoadData().getTopBundlesLoadData().get(2); - assertEquals(top0.bundleName(), bundle3); - assertEquals(top1.bundleName(), bundle2); + assertEquals(top0.bundleName(), bundle2); + assertEquals(top1.bundleName(), bundle3); assertEquals(top2.bundleName(), bundle1); } @@ -225,8 +225,8 @@ public void testLoadBalancerSheddingBundlesWithPoliciesEnabledConfig() throws Me var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1); - assertEquals(top0.bundleName(), bundle2); - assertEquals(top1.bundleName(), bundle1); + assertEquals(top0.bundleName(), bundle1); + assertEquals(top1.bundleName(), bundle2); configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java index ee7e708667a32..93ab35981e1c4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java @@ -18,28 +18,37 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.reporter; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.PulsarStats; import org.apache.pulsar.broker.stats.BrokerStats; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -52,17 +61,24 @@ public class BrokerLoadDataReporterTest { ServiceConfiguration config; BrokerStats brokerStats; SystemResourceUsage usage; + String broker = "broker1"; + String bundle = "bundle1"; + ScheduledExecutorService executor; @BeforeMethod void setup() { config = new ServiceConfiguration(); + config.setLoadBalancerDebugModeEnabled(true); pulsar = mock(PulsarService.class); store = mock(LoadDataStore.class); brokerService = mock(BrokerService.class); pulsarStats = mock(PulsarStats.class); doReturn(brokerService).when(pulsar).getBrokerService(); doReturn(config).when(pulsar).getConfiguration(); - doReturn(Executors.newSingleThreadScheduledExecutor()).when(pulsar).getLoadManagerExecutor(); + executor = Executors + .newSingleThreadScheduledExecutor(new + ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager")); + doReturn(executor).when(pulsar).getLoadManagerExecutor(); doReturn(pulsarStats).when(brokerService).getPulsarStats(); brokerStats = new BrokerStats(0); brokerStats.topics = 6; @@ -74,6 +90,7 @@ void setup() { doReturn(pulsarStats).when(brokerService).getPulsarStats(); doReturn(brokerStats).when(pulsarStats).getBrokerStats(); doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any()); + doReturn(CompletableFuture.completedFuture(null)).when(store).removeAsync(any()); usage = new SystemResourceUsage(); usage.setCpu(new ResourceUsage(1.0, 100.0)); @@ -83,6 +100,11 @@ void setup() { usage.setBandwidthOut(new ResourceUsage(4.0, 100.0)); } + @AfterMethod + void shutdown(){ + executor.shutdown(); + } + public void testGenerate() throws IllegalAccessException { try (MockedStatic mockLoadManagerShared = Mockito.mockStatic(LoadManagerShared.class)) { mockLoadManagerShared.when(() -> LoadManagerShared.getSystemResourceUsage(any())).thenReturn(usage); @@ -124,4 +146,73 @@ public void testReport() throws IllegalAccessException { } } + @Test + public void testTombstone() throws IllegalAccessException, InterruptedException { + + var target = spy(new BrokerLoadDataReporter(pulsar, broker, store)); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Assigning, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Deleted, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Init, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Free, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), + new RuntimeException()); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(1)).tombstone(); + verify(store, times(1)).removeAsync(eq(broker)); + var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true); + assertEquals(localData, new BrokerLoadData()); + }); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(2)).tombstone(); + verify(store, times(1)).removeAsync(eq(broker)); + var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true); + assertEquals(localData, new BrokerLoadData()); + }); + + FieldUtils.writeDeclaredField(target, "tombstoneDelayInMillis", 0, true); + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-2", broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(3)).tombstone(); + verify(store, times(2)).removeAsync(eq(broker)); + var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true); + assertEquals(localData, new BrokerLoadData()); + }); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Owned, broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(4)).tombstone(); + verify(store, times(3)).removeAsync(eq(broker)); + var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true); + assertEquals(localData, new BrokerLoadData()); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java index b5c415c405fbd..be8c6af2b0404 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java @@ -18,9 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.reporter; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; @@ -29,9 +32,12 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; @@ -42,6 +48,7 @@ import org.apache.pulsar.broker.service.PulsarStats; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; +import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -58,10 +65,13 @@ public class TopBundleLoadDataReporterTest { private LocalPoliciesResources localPoliciesResources; String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF"; String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF"; + String bundle = bundle1; + String broker = "broker-1"; @BeforeMethod void setup() throws MetadataStoreException { config = new ServiceConfiguration(); + config.setLoadBalancerDebugModeEnabled(true); pulsar = mock(PulsarService.class); store = mock(LoadDataStore.class); brokerService = mock(BrokerService.class); @@ -75,6 +85,7 @@ void setup() throws MetadataStoreException { doReturn(config).when(pulsar).getConfiguration(); doReturn(pulsarStats).when(brokerService).getPulsarStats(); doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any()); + doReturn(CompletableFuture.completedFuture(null)).when(store).removeAsync(any()); doReturn(pulsarResources).when(pulsar).getPulsarResources(); doReturn(namespaceResources).when(pulsarResources).getNamespaceResources(); @@ -102,22 +113,23 @@ public void testZeroUpdatedAt() { public void testGenerateLoadData() throws IllegalAccessException { doReturn(1l).when(pulsarStats).getUpdatedAt(); - config.setLoadBalancerBundleLoadReportPercentage(100); + config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(2); var target = new TopBundleLoadDataReporter(pulsar, "", store); var expected = new TopKBundles(pulsar); expected.update(bundleStats, 2); assertEquals(target.generateLoadData(), expected.getLoadData()); - config.setLoadBalancerBundleLoadReportPercentage(50); + config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(1); FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); expected = new TopKBundles(pulsar); expected.update(bundleStats, 1); assertEquals(target.generateLoadData(), expected.getLoadData()); - config.setLoadBalancerBundleLoadReportPercentage(1); + config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(0); FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); + expected = new TopKBundles(pulsar); - expected.update(bundleStats, 1); + expected.update(bundleStats, 0); assertEquals(target.generateLoadData(), expected.getLoadData()); doReturn(new HashMap()).when(brokerService).getBundleStats(); @@ -128,21 +140,83 @@ public void testGenerateLoadData() throws IllegalAccessException { public void testReportForce() { - var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store); + var target = new TopBundleLoadDataReporter(pulsar, broker, store); target.reportAsync(false); verify(store, times(0)).pushAsync(any(), any()); target.reportAsync(true); - verify(store, times(1)).pushAsync("broker-1", new TopBundlesLoadData()); + verify(store, times(1)).pushAsync(broker, new TopBundlesLoadData()); } public void testReport(){ - var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store); + pulsar.getConfiguration().setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(1); + var target = new TopBundleLoadDataReporter(pulsar, broker, store); doReturn(1l).when(pulsarStats).getUpdatedAt(); var expected = new TopKBundles(pulsar); expected.update(bundleStats, 1); target.reportAsync(false); - verify(store, times(1)).pushAsync("broker-1", expected.getLoadData()); + verify(store, times(1)).pushAsync(broker, expected.getLoadData()); } + @Test + public void testTombstone() throws IllegalAccessException { + + var target = spy(new TopBundleLoadDataReporter(pulsar, broker, store)); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Assigning, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Deleted, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Init, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Free, broker, VERSION_ID_INIT), null); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), + new RuntimeException()); + verify(store, times(0)).removeAsync(eq(broker)); + verify(target, times(0)).tombstone(); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(1)).tombstone(); + verify(store, times(1)).removeAsync(eq(broker)); + }); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(2)).tombstone(); + verify(store, times(1)).removeAsync(eq(broker)); + }); + + FieldUtils.writeDeclaredField(target, "tombstoneDelayInMillis", 0, true); + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-2", broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(3)).tombstone(); + verify(store, times(2)).removeAsync(eq(broker)); + }); + + target.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Owned, broker, VERSION_ID_INIT), null); + Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + verify(target, times(4)).tombstone(); + verify(store, times(3)).removeAsync(eq(broker)); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 5ca9345d6def5..8c8d18e202a00 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -18,10 +18,11 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.scheduler; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; -import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; @@ -52,6 +53,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.commons.math3.stat.descriptive.moment.Mean; @@ -59,7 +61,11 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -97,6 +103,10 @@ public class TransferShedderTest { double setupLoadStd = 0.3982762860126121; PulsarService pulsar; + NamespaceService namespaceService; + ExtensibleLoadManagerWrapper loadManagerWrapper; + ExtensibleLoadManagerImpl loadManager; + ServiceUnitStateChannel channel; ServiceConfiguration conf; AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; LocalPoliciesResources localPoliciesResources; @@ -108,13 +118,16 @@ public class TransferShedderTest { @BeforeMethod public void init() throws MetadataStoreException { pulsar = mock(PulsarService.class); + loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class); + loadManager = mock(ExtensibleLoadManagerImpl.class); + channel = mock(ServiceUnitStateChannelImpl.class); conf = new ServiceConfiguration(); conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); var pulsarResources = mock(PulsarResources.class); var namespaceResources = mock(NamespaceResources.class); var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class); var factory = mock(NamespaceBundleFactory.class); - var namespaceService = mock(NamespaceService.class); + namespaceService = mock(NamespaceService.class); localPoliciesResources = mock(LocalPoliciesResources.class); antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class); doReturn(conf).when(pulsar).getConfiguration(); @@ -136,25 +149,30 @@ public void init() throws MetadataStoreException { (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory); }).when(factory).getBundle(anyString(), anyString()); - doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any()); + doReturn(new AtomicReference<>(loadManagerWrapper)).when(pulsar).getLoadManager(); + doReturn(loadManager).when(loadManagerWrapper).get(); + doReturn(channel).when(loadManager).getServiceUnitStateChannel(); + doReturn(true).when(channel).isOwner(any(), any()); } public LoadManagerContext setupContext(){ var ctx = getContext(); - var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2)); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4)); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6)); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80)); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90)); + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2000000, 1000000)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 3000000, 1000000)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 4000000, 2000000)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 6000000, 2000000)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 7000000, 2000000)); + topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000)); + topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000)); + topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 2000000, 4000000)); + topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 2000000, 6000000)); + topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 7000000)); + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1")); + brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2")); + brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3")); + brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4")); + brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5")); return ctx; } @@ -166,8 +184,8 @@ public LoadManagerContext setupContext(int clusterSize) { Random rand = new Random(); for (int i = 0; i < clusterSize; i++) { - int brokerLoad = rand.nextInt(100); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad)); + int brokerLoad = rand.nextInt(1000); + brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); int bundleLoad = rand.nextInt(brokerLoad + 1); topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, bundleLoad, brokerLoad - bundleLoad)); @@ -175,7 +193,7 @@ public LoadManagerContext setupContext(int clusterSize) { return ctx; } - public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) { + public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load, String broker) { var loadData = new BrokerLoadData(); SystemResourceUsage usage1 = new SystemResourceUsage(); var cpu = new ResourceUsage(load, 100.0); @@ -188,8 +206,17 @@ public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - loadData.update(usage1, 1,2,3,4,5,6, - ctx.brokerConfiguration()); + if (ctx.topBundleLoadDataStore() + .get(broker).isPresent()) { + var throughputOut = ctx.topBundleLoadDataStore() + .get(broker).get() + .getTopBundlesLoadData().stream().mapToDouble(v -> v.stats().msgThroughputOut).sum(); + loadData.update(usage1, 1, throughputOut, 3, 4, 5, 6, + ctx.brokerConfiguration()); + } else { + loadData.update(usage1, 1, 2, 3, 4, 5, 6, + ctx.brokerConfiguration()); + } return loadData; } @@ -204,6 +231,29 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int return topKBundles.getLoadData(); } + public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, + int load1, int load2, int load3, int load4, int load5) { + var namespaceBundleStats1 = new NamespaceBundleStats(); + namespaceBundleStats1.msgThroughputOut = load1; + var namespaceBundleStats2 = new NamespaceBundleStats(); + namespaceBundleStats2.msgThroughputOut = load2; + var namespaceBundleStats3 = new NamespaceBundleStats(); + namespaceBundleStats3.msgThroughputOut = load3; + var namespaceBundleStats4 = new NamespaceBundleStats(); + namespaceBundleStats4.msgThroughputOut = load4; + var namespaceBundleStats5 = new NamespaceBundleStats(); + namespaceBundleStats5.msgThroughputOut = load5; + var topKBundles = new TopKBundles(pulsar); + topKBundles.update(Map.of( + bundlePrefix + "/0x00000000_0x1FFFFFFF", namespaceBundleStats1, + bundlePrefix + "/0x1FFFFFFF_0x2FFFFFFF", namespaceBundleStats2, + bundlePrefix + "/0x2FFFFFFF_0x3FFFFFFF", namespaceBundleStats3, + bundlePrefix + "/0x3FFFFFFF_0x4FFFFFFF", namespaceBundleStats4, + bundlePrefix + "/0x4FFFFFFF_0x5FFFFFFF", namespaceBundleStats5 + ), 5); + return topKBundles.getLoadData(); + } + public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) { var namespaceBundleStats1 = new NamespaceBundleStats(); namespaceBundleStats1.msgThroughputOut = load1; @@ -230,6 +280,7 @@ public LoadManagerContext getContext(){ var conf = new ServiceConfiguration(); conf.setLoadBalancerDebugModeEnabled(true); conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); + conf.setLoadBalancerSheddingConditionHitCountThreshold(0); var brokerLoadDataStore = new LoadDataStore() { Map map = new HashMap<>(); @Override @@ -357,29 +408,44 @@ public void testEmptyBrokerLoadData() { UnloadCounter counter = new UnloadCounter(); TransferShedder transferShedder = new TransferShedder(counter); var ctx = getContext(); - ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBrokers).get(), 1); } + @Test + public void testNoOwnerLoadData() throws IllegalAccessException { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + FieldUtils.writeDeclaredField(transferShedder, "channel", channel, true); + var ctx = setupContext(); + doReturn(false).when(channel).isOwner(any(), any()); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + } + @Test public void testEmptyTopBundlesLoadData() { UnloadCounter counter = new UnloadCounter(); TransferShedder transferShedder = new TransferShedder(counter); var ctx = getContext(); - ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 90)); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 10)); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 20)); + brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1")); + brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2")); + brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3")); + brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4")); + brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5")); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); assertEquals(counter.getBreakdownCounters().get(Skip).get(NoLoadData).get(), 1); - assertEquals(counter.getLoadAvg(), 0.39999999999999997); - assertEquals(counter.getLoadStd(), 0.35590260840104376); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); } @Test @@ -445,16 +511,18 @@ public void testRecentlyUnloadedBundles() { recentlyUnloadedBundles.put(bundleD1, now); recentlyUnloadedBundles.put(bundleD2, now); var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of()); - - assertTrue(res.isEmpty()); - assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker3", + "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF", + Optional.of("broker1")), + Success, Overloaded)); + assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); assertEquals(counter.getLoadStd(), setupLoadStd); } @Test public void testGetAvailableBrokersFailed() { - var pulsar = getMockPulsar(); UnloadCounter counter = new UnloadCounter(); AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class); TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper); @@ -462,17 +530,16 @@ public void testGetAvailableBrokersFailed() { BrokerRegistry registry = ctx.brokerRegistry(); doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1); - assertEquals(counter.getLoadAvg(), setupLoadAvg); - assertEquals(counter.getLoadStd(), setupLoadStd); + assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1); + assertEquals(counter.getLoadAvg(), 0.0); + assertEquals(counter.getLoadStd(), 0.0); } @Test(timeOut = 30 * 1000) public void testBundlesWithIsolationPolicies() throws IllegalAccessException { - var pulsar = getMockPulsar(); + doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any()); UnloadCounter counter = new UnloadCounter(); TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper)); - var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true); @@ -484,7 +551,7 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException { var ctx = setupContext(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker1")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -572,29 +639,9 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, }).when(policies).shouldFailoverToSecondaries(eq(namespaceName), anyInt()); } - private PulsarService getMockPulsar() { - var pulsar = mock(PulsarService.class); - var namespaceService = mock(NamespaceService.class); - doReturn(namespaceService).when(pulsar).getNamespaceService(); - NamespaceBundleFactory factory = mock(NamespaceBundleFactory.class); - doReturn(factory).when(namespaceService).getNamespaceBundleFactory(); - doAnswer(answer -> { - String namespace = answer.getArgument(0, String.class); - String bundleRange = answer.getArgument(1, String.class); - String[] boundaries = bundleRange.split("_"); - Long lowerEndpoint = Long.decode(boundaries[0]); - Long upperEndpoint = Long.decode(boundaries[1]); - Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint, - (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); - return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory); - }).when(factory).getBundle(anyString(), anyString()); - return pulsar; - } - @Test public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException { - var pulsar = getMockPulsar(); var counter = new UnloadCounter(); TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper); var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) @@ -606,7 +653,10 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(any()); var ctx = setupContext(); - doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any()); + var antiAffinityGroupPolicyHelperSpy = (AntiAffinityGroupPolicyHelper) + spy(FieldUtils.readDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", true)); + doReturn(false).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), any(), any(), any()); + FieldUtils.writeDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", antiAffinityGroupPolicyHelperSpy, true); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -615,7 +665,7 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me assertEquals(counter.getLoadStd(), setupLoadStd); - doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any()); + doReturn(true).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), eq(bundleE1), any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected2 = new HashSet<>(); expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), @@ -630,11 +680,18 @@ public void testTargetStd() { UnloadCounter counter = new UnloadCounter(); TransferShedder transferShedder = new TransferShedder(counter); var ctx = getContext(); + BrokerRegistry brokerRegistry = mock(BrokerRegistry.class); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", mock(BrokerLookupData.class), + "broker2", mock(BrokerLookupData.class), + "broker3", mock(BrokerLookupData.class) + ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); + doReturn(brokerRegistry).when(ctx).brokerRegistry(); ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10)); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 20)); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 30)); + brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); + brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 20, "broker2")); + brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 30, "broker3")); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); @@ -645,7 +702,7 @@ public void testTargetStd() { var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); - assertEquals(counter.getBreakdownCounters().get(Skip).get(Balanced).get(), 1); + assertEquals(counter.getBreakdownCounters().get(Skip).get(HitCount).get(), 1); assertEquals(counter.getLoadAvg(), 0.2000000063578288); assertEquals(counter.getLoadStd(), 0.08164966587949089); } @@ -669,6 +726,26 @@ public void testSingleTopBundlesLoadData() { assertEquals(counter.getLoadStd(), setupLoadStd); } + @Test + public void testBundleThroughputLargerThanOffloadThreshold() { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = setupContext(); + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 1000000000, 1000000000)); + topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 1000000000, 1000000000)); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker3", + "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF", + Optional.of("broker1")), + Success, Overloaded)); + assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + } + @Test public void testTargetStdAfterTransfer() { @@ -676,8 +753,8 @@ public void testTargetStdAfterTransfer() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55)); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65)); + brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); + brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); @@ -688,19 +765,186 @@ public void testTargetStdAfterTransfer() { assertEquals(counter.getLoadStd(), 0.27644891028904417); } + @Test + public void testUnloadBundlesGreaterThanTargetThroughput() throws IllegalAccessException { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = getContext(); + + var brokerRegistry = mock(BrokerRegistry.class); + doReturn(brokerRegistry).when(ctx).brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", mock(BrokerLookupData.class), + "broker2", mock(BrokerLookupData.class) + ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); + + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoad("my-tenant/my-namespaceB", 100000000, 180000000, 220000000, 250000000, 250000000)); + + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); + brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2")); + + + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + var expected = new HashSet(); + expected.add(new UnloadDecision( + new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x1FFFFFFF", Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision( + new Unload("broker2", "my-tenant/my-namespaceB/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision( + new Unload("broker2", "my-tenant/my-namespaceB/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision( + new Unload("broker1", "my-tenant/my-namespaceA/0x00000000_0x1FFFFFFF", Optional.of("broker2")), + Success, Overloaded)); + expected.add(new UnloadDecision( + new Unload("broker1", "my-tenant/my-namespaceA/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker2")), + Success, Overloaded)); + expected.add(new UnloadDecision( + new Unload("broker1","my-tenant/my-namespaceA/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker2")), + Success, Overloaded)); + expected.add(new UnloadDecision( + new Unload("broker1","my-tenant/my-namespaceA/0x3FFFFFFF_0x4FFFFFFF", Optional.of("broker2")), + Success, Overloaded)); + assertEquals(counter.getLoadAvg(), 5.05); + assertEquals(counter.getLoadStd(), 4.95); + assertEquals(res, expected); + var stats = (TransferShedder.LoadStats) + FieldUtils.readDeclaredField(transferShedder, "stats", true); + assertEquals(stats.std(), 0.050000004900021836); + } + + @Test + public void testSkipBundlesGreaterThanTargetThroughputAfterSplit() { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = getContext(); + ctx.brokerConfiguration().setLoadBalancerBrokerLoadTargetStd(0.20); + + var brokerRegistry = mock(BrokerRegistry.class); + doReturn(brokerRegistry).when(ctx).brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", mock(BrokerLookupData.class), + "broker2", mock(BrokerLookupData.class) + ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); + + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + topBundlesLoadDataStore.pushAsync("broker1", + getTopBundlesLoad("my-tenant/my-namespaceA", 1, 500000000)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoad("my-tenant/my-namespaceB", 500000000, 500000000)); + + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 50, "broker1")); + brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2")); + + + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + } + + + @Test + public void testUnloadBundlesLessThanTargetThroughputAfterSplit() throws IllegalAccessException { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = getContext(); + + var brokerRegistry = mock(BrokerRegistry.class); + doReturn(brokerRegistry).when(ctx).brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", mock(BrokerLookupData.class), + "broker2", mock(BrokerLookupData.class) + ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); + + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 490000000, 510000000)); + + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); + brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2")); + + + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + var expected = new HashSet(); + expected.add(new UnloadDecision( + new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x0FFFFFFF", Optional.of("broker1")), + Success, Overloaded)); + assertEquals(counter.getLoadAvg(), 5.05); + assertEquals(counter.getLoadStd(), 4.95); + assertEquals(res, expected); + var stats = (TransferShedder.LoadStats) + FieldUtils.readDeclaredField(transferShedder, "stats", true); + assertEquals(stats.std(), 0.050000004900021836); + + } + + + @Test + public void testUnloadBundlesGreaterThanTargetThroughputAfterSplit() throws IllegalAccessException { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = getContext(); + + var brokerRegistry = mock(BrokerRegistry.class); + doReturn(brokerRegistry).when(ctx).brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", mock(BrokerLookupData.class), + "broker2", mock(BrokerLookupData.class) + ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); + + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2400000, 2400000)); + topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 5000000, 5000000)); + + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 48, "broker1")); + brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2")); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + var expected = new HashSet(); + expected.add(new UnloadDecision( + new Unload("broker1", + res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker1")).findFirst().get() + .getUnload().serviceUnit(), Optional.of("broker2")), + Success, Overloaded)); + expected.add(new UnloadDecision( + new Unload("broker2", + res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker2")).findFirst().get() + .getUnload().serviceUnit(), Optional.of("broker1")), + Success, Overloaded)); + assertEquals(counter.getLoadAvg(), 0.74); + assertEquals(counter.getLoadStd(), 0.26); + assertEquals(res, expected); + var stats = (TransferShedder.LoadStats) + FieldUtils.readDeclaredField(transferShedder, "stats", true); + assertEquals(stats.std(), 2.5809568279517847E-8); + } + + @Test public void testMinBrokerWithZeroTraffic() throws IllegalAccessException { UnloadCounter counter = new UnloadCounter(); TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55)); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65)); - var load = getCpuLoad(ctx, 4); - FieldUtils.writeDeclaredField(load,"msgThroughputIn", 0, true); - FieldUtils.writeDeclaredField(load,"msgThroughputOut", 0, true); + var load = getCpuLoad(ctx, 4, "broker2"); + FieldUtils.writeDeclaredField(load,"msgThroughputEMA", 0, true); brokerLoadDataStore.pushAsync("broker2", load); + brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); + brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -714,13 +958,63 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException { assertEquals(counter.getLoadStd(), 0.27644891028904417); } + @Test + public void testMinBrokerWithLowerLoadThanAvg() throws IllegalAccessException { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = setupContext(); + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + + var load = getCpuLoad(ctx, 3 , "broker2"); + brokerLoadDataStore.pushAsync("broker2", load); + brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); + brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); + + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Underloaded)); + assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), 0.262); + assertEquals(counter.getLoadStd(), 0.2780935094532054); + } + @Test public void testMaxNumberOfTransfersPerShedderCycle() { UnloadCounter counter = new UnloadCounter(); TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); ctx.brokerConfiguration() - .setLoadBalancerMaxNumberOfBrokerTransfersPerCycle(10); + .setLoadBalancerMaxNumberOfBrokerSheddingPerCycle(10); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Overloaded)); + assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + } + + @Test + public void testLoadBalancerSheddingConditionHitCountThreshold() { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = setupContext(); + int max = 3; + ctx.brokerConfiguration() + .setLoadBalancerSheddingConditionHitCountThreshold(max); + for (int i = 0; i < max; i++) { + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(HitCount).get(), i+1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + } var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), @@ -738,7 +1032,7 @@ public void testRemainingTopBundles() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000)); + topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 3000000)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); @@ -751,6 +1045,33 @@ public void testRemainingTopBundles() { assertEquals(counter.getLoadStd(), setupLoadStd); } + @Test + public void testLoadMoreThan100() throws IllegalAccessException { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = setupContext(); + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 200, "broker4")); + brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 1000, "broker5")); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + Success, Overloaded)); + expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + Success, Overloaded)); + assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), 2.4240000000000004); + assertEquals(counter.getLoadStd(), 3.8633332758124816); + + + var stats = (TransferShedder.LoadStats) + FieldUtils.readDeclaredField(transferShedder, "stats", true); + assertEquals(stats.avg(), 2.4240000000000004); + assertEquals(stats.std(), 2.781643776903451); + } + @Test public void testRandomLoad() throws IllegalAccessException { UnloadCounter counter = new UnloadCounter(); @@ -767,7 +1088,7 @@ public void testRandomLoad() throws IllegalAccessException { } @Test - public void testLoadStats() { + public void testRandomLoadStats() { int numBrokers = 10; double delta = 0.0001; for (int t = 0; t < 5; t++) { @@ -776,8 +1097,13 @@ public void testLoadStats() { var loadStore = ctx.brokerLoadDataStore(); stats.setLoadDataStore(loadStore); var conf = ctx.brokerConfiguration(); - stats.update(loadStore, Map.of(), conf); double[] loads = new double[numBrokers]; + final Map availableBrokers = new HashMap<>(); + for (int i = 0; i < loads.length; i++) { + availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); + } + stats.update(loadStore, availableBrokers, Map.of(), conf); + var brokerLoadDataStore = ctx.brokerLoadDataStore(); for (int i = 0; i < loads.length; i++) { loads[i] = loadStore.get("broker" + i).get().getWeightedMaxEMA(); @@ -785,16 +1111,16 @@ public void testLoadStats() { int i = 0; int j = loads.length - 1; Arrays.sort(loads); - for (int k = 0; k < conf.getLoadBalancerMaxNumberOfBrokerTransfersPerCycle(); k++) { + for (int k = 0; k < conf.getLoadBalancerMaxNumberOfBrokerSheddingPerCycle(); k++) { double minLoad = loads[i]; double maxLoad = loads[j]; double offload = (maxLoad - minLoad) / 2; Mean mean = new Mean(); StandardDeviation std = new StandardDeviation(false); assertEquals(minLoad, - loadStore.get(stats.minBrokers().pollLast()).get().getWeightedMaxEMA()); + loadStore.get(stats.peekMinBroker()).get().getWeightedMaxEMA()); assertEquals(maxLoad, - loadStore.get(stats.maxBrokers().pollLast()).get().getWeightedMaxEMA()); + loadStore.get(stats.pollMaxBroker()).get().getWeightedMaxEMA()); assertEquals(stats.totalBrokers(), numBrokers); assertEquals(stats.avg(), mean.evaluate(loads), delta); assertEquals(stats.std(), std.evaluate(loads), delta); @@ -804,4 +1130,41 @@ public void testLoadStats() { } } } + + @Test + public void testHighVarianceLoadStats() { + int[] loads = {1, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100}; + var ctx = getContext(); + TransferShedder.LoadStats stats = new TransferShedder.LoadStats(); + var loadStore = ctx.brokerLoadDataStore(); + stats.setLoadDataStore(loadStore); + var conf = ctx.brokerConfiguration(); + final Map availableBrokers = new HashMap<>(); + for (int i = 0; i < loads.length; i++) { + availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); + loadStore.pushAsync("broker" + i, getCpuLoad(ctx, loads[i], "broker" + i)); + } + stats.update(loadStore, availableBrokers, Map.of(), conf); + + assertEquals(stats.avg(), 0.9417647058823528); + assertEquals(stats.std(), 0.23294117647058868); + } + + @Test + public void testLowVarianceLoadStats() { + int[] loads = {390, 391, 392, 393, 394, 395, 396, 397, 398, 399}; + var ctx = getContext(); + TransferShedder.LoadStats stats = new TransferShedder.LoadStats(); + var loadStore = ctx.brokerLoadDataStore(); + stats.setLoadDataStore(loadStore); + var conf = ctx.brokerConfiguration(); + final Map availableBrokers = new HashMap<>(); + for (int i = 0; i < loads.length; i++) { + availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); + loadStore.pushAsync("broker" + i, getCpuLoad(ctx, loads[i], "broker" + i)); + } + stats.update(loadStore, availableBrokers, Map.of(), conf); + assertEquals(stats.avg(), 3.9449999999999994); + assertEquals(stats.std(), 0.028722813232795824); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java index 7d9cf556360e4..38d4e9904e649 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java @@ -54,8 +54,8 @@ @Test(groups = "broker") public class UnloadSchedulerTest { - - private ScheduledExecutorService loadManagerExecutor; + private PulsarService pulsar; + private ScheduledExecutorService loadManagerExecutor; public LoadManagerContext setupContext(){ var ctx = getContext(); @@ -65,8 +65,12 @@ public LoadManagerContext setupContext(){ @BeforeMethod public void setUp() { - this.loadManagerExecutor = Executors - .newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager")); + this.pulsar = mock(PulsarService.class); + loadManagerExecutor = Executors + .newSingleThreadScheduledExecutor(new + ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager")); + doReturn(loadManagerExecutor) + .when(pulsar).getLoadManagerExecutor(); } @AfterMethod diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java index deff6fb00fafb..0aa055b58acd3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java @@ -31,10 +31,15 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; @@ -50,6 +55,9 @@ public class DefaultNamespaceBundleSplitStrategyTest { PulsarService pulsar; + ExtensibleLoadManagerWrapper loadManagerWrapper; + ExtensibleLoadManagerImpl loadManager; + ServiceUnitStateChannel channel; BrokerService brokerService; PulsarStats pulsarStats; Map bundleStats; @@ -76,7 +84,7 @@ void setup() { config.setLoadBalancerNamespaceBundleMaxMsgRate(100); config.setLoadBalancerNamespaceBundleMaxBandwidthMbytes(100); config.setLoadBalancerMaxNumberOfBundlesToSplitPerCycle(1); - config.setLoadBalancerNamespaceBundleSplitConditionThreshold(3); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(3); pulsar = mock(PulsarService.class); brokerService = mock(BrokerService.class); @@ -85,8 +93,9 @@ void setup() { namespaceBundleFactory = mock(NamespaceBundleFactory.class); loadManagerContext = mock(LoadManagerContext.class); brokerRegistry = mock(BrokerRegistry.class); - - + loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class); + loadManager = mock(ExtensibleLoadManagerImpl.class); + channel = mock(ServiceUnitStateChannelImpl.class); doReturn(brokerService).when(pulsar).getBrokerService(); doReturn(config).when(pulsar).getConfiguration(); @@ -96,6 +105,10 @@ void setup() { doReturn(true).when(namespaceBundleFactory).canSplitBundle(any()); doReturn(brokerRegistry).when(loadManagerContext).brokerRegistry(); doReturn(broker).when(brokerRegistry).getBrokerId(); + doReturn(new AtomicReference(loadManagerWrapper)).when(pulsar).getLoadManager(); + doReturn(loadManager).when(loadManagerWrapper).get(); + doReturn(channel).when(loadManager).getServiceUnitStateChannel(); + doReturn(true).when(channel).isOwner(any()); bundleStats = new LinkedHashMap<>(); @@ -109,7 +122,7 @@ void setup() { } public void testNamespaceBundleSplitConditionThreshold() { - config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0); bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter()); var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); @@ -118,7 +131,7 @@ public void testNamespaceBundleSplitConditionThreshold() { public void testNotEnoughTopics() { - config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0); bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter()); bundleStats.values().forEach(v -> v.topics = 1); @@ -128,7 +141,7 @@ public void testNotEnoughTopics() { } public void testNamespaceMaximumBundles() throws Exception { - config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0); bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter()); doReturn(config.getLoadBalancerNamespaceMaximumBundles()).when(namespaceService).getBundleCount(any()); @@ -138,7 +151,7 @@ public void testNamespaceMaximumBundles() throws Exception { } public void testEmptyBundleStats() { - config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0); bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter()); bundleStats.clear(); @@ -147,9 +160,21 @@ public void testEmptyBundleStats() { assertEquals(actual, expected); } + public void testNoBundleOwner() { + var counter = spy(new SplitCounter()); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0); + bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1); + doReturn(false).when(channel).isOwner(any()); + var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); + var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); + var expected = Set.of(); + assertEquals(actual, expected); + verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); + } + public void testError() throws Exception { var counter = spy(new SplitCounter()); - config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0); bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); doThrow(new RuntimeException()).when(namespaceService).getBundleCount(any()); @@ -162,7 +187,7 @@ public void testError() throws Exception { public void testMaxMsgRate() { var counter = spy(new SplitCounter()); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); - int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold(); + int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(); bundleStats.values().forEach(v -> { v.msgRateOut = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1; v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1; @@ -193,7 +218,7 @@ public void testMaxMsgRate() { public void testMaxTopics() { var counter = spy(new SplitCounter()); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); - int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold(); + int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(); bundleStats.values().forEach(v -> v.topics = config.getLoadBalancerNamespaceBundleMaxTopics() + 1); for (int i = 0; i < threshold + 2; i++) { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); @@ -221,7 +246,7 @@ public void testMaxTopics() { public void testMaxSessions() { var counter = spy(new SplitCounter()); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); - int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold(); + int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(); bundleStats.values().forEach(v -> { v.producerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1; v.consumerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1; @@ -252,7 +277,7 @@ public void testMaxSessions() { public void testMaxBandwidthMbytes() { var counter = spy(new SplitCounter()); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); - int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold(); + int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(); bundleStats.values().forEach(v -> { v.msgThroughputOut = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1; v.msgThroughputIn = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1; diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java index dc1cdd3d5cbce..cd364990842a1 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java @@ -20,10 +20,12 @@ import java.io.Serializable; import lombok.EqualsAndHashCode; +import lombok.ToString; /** */ @EqualsAndHashCode +@ToString public class NamespaceBundleStats implements Comparable, Serializable { public double msgRateIn; diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java index eed8b33b39178..ae12e931071bc 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java @@ -19,11 +19,13 @@ package org.apache.pulsar.policies.data.loadbalancer; import lombok.EqualsAndHashCode; +import lombok.ToString; /** * POJO used to represent any system specific resource usage this is the format that load manager expects it in. */ @EqualsAndHashCode +@ToString public class ResourceUsage { public final double usage; public final double limit; From 38485e09ce1b01803bd6d5ab95901f6284428685 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 29 Mar 2023 17:55:48 +0800 Subject: [PATCH 043/280] [improve][build] Create source jar for pulsar-client-all shaded jar (#19956) Signed-off-by: tison --- pulsar-client-all/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 00da6e4895097..73621954e1fb8 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -141,6 +141,8 @@ shade + true + true true true false From 07acdbc8541c1eeb724361713e7fd136d4c93fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 29 Mar 2023 14:38:02 +0200 Subject: [PATCH 044/280] [fix][sec] Fix transitive critical CVEs in file-system tiered storage (#19957) --- pom.xml | 6 +++--- tiered-storage/file-system/pom.xml | 25 ------------------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index ac640f1a6a05f..4c489355281ba 100644 --- a/pom.xml +++ b/pom.xml @@ -180,8 +180,8 @@ flexible messaging model and an intuitive client API. 0.3.2-patch11 2.7.5 0.4.4-hotfix1 - 3.3.3 - 2.4.7 + 3.3.5 + 2.4.10 1.2.4 8.5.2 363 @@ -257,7 +257,7 @@ flexible messaging model and an intuitive client API. 3.1 4.2.0 1.2.22 - 1.5.3 + 1.5.4 5.4.0 2.33.2 diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 1d8aab48cbff8..745da1a95c70d 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -53,31 +53,6 @@ - - com.sun.jersey - jersey-json - - 1.19 - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-xc - - - - org.apache.avro avro From 55523ac8f31fd6d54aacba326edef1f53028877e Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 29 Mar 2023 13:55:22 -0700 Subject: [PATCH 045/280] [improve][io] KCA: flag to force optional primitive schemas (#19951) Motivation Kafka's schema has "Optional" flag that used there to validate data/allow nulls. Pulsar's schema does not have such info which makes conversion to kafka schema lossy. Modifications Added a config parameter that lets one force primitive schemas into optional ones. KV schema is always optional. Default is false, to match existing behavior. --- .../io/kafka/connect/KafkaConnectSink.java | 16 +- .../connect/PulsarKafkaConnectSinkConfig.java | 6 + .../schema/PulsarSchemaToKafkaSchema.java | 55 ++++++- .../kafka/connect/KafkaConnectSinkTest.java | 41 +++-- .../PulsarSchemaToKafkaSchemaTest.java | 149 +++++++++++++----- 5 files changed, 203 insertions(+), 64 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java index 475724cc4e545..10efc91ccdaad 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java @@ -90,6 +90,7 @@ public class KafkaConnectSink implements Sink { private PulsarKafkaConnectSinkConfig kafkaSinkConfig; protected String topicName; + protected boolean useOptionalPrimitives; private boolean sanitizeTopicName = false; // Thi is a workaround for https://github.com/apache/pulsar/issues/19922 @@ -164,6 +165,7 @@ public void open(Map config, SinkContext ctx) throws Exception { unwrapKeyValueIfAvailable = kafkaSinkConfig.isUnwrapKeyValueIfAvailable(); sanitizeTopicName = kafkaSinkConfig.isSanitizeTopicName(); collapsePartitionedTopics = kafkaSinkConfig.isCollapsePartitionedTopics(); + useOptionalPrimitives = kafkaSinkConfig.isUseOptionalPrimitives(); useIndexAsOffset = kafkaSinkConfig.isUseIndexAsOffset(); maxBatchBitsForOffset = kafkaSinkConfig.getMaxBatchBitsForOffset(); @@ -446,8 +448,11 @@ protected SinkRecord toSinkRecord(Record sourceRecord) { && sourceRecord.getSchema().getSchemaInfo() != null && sourceRecord.getSchema().getSchemaInfo().getType() == SchemaType.KEY_VALUE) { KeyValueSchema kvSchema = (KeyValueSchema) sourceRecord.getSchema(); - keySchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(kvSchema.getKeySchema()); - valueSchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(kvSchema.getValueSchema()); + // Assume Key_Value schema's key and value are always optional + keySchema = PulsarSchemaToKafkaSchema + .getOptionalKafkaConnectSchema(kvSchema.getKeySchema(), useOptionalPrimitives); + valueSchema = PulsarSchemaToKafkaSchema + .getOptionalKafkaConnectSchema(kvSchema.getValueSchema(), useOptionalPrimitives); Object nativeObject = sourceRecord.getValue().getNativeObject(); @@ -464,12 +469,13 @@ protected SinkRecord toSinkRecord(Record sourceRecord) { } else { if (sourceRecord.getMessage().get().hasBase64EncodedKey()) { key = sourceRecord.getMessage().get().getKeyBytes(); - keySchema = Schema.BYTES_SCHEMA; + keySchema = useOptionalPrimitives ? Schema.OPTIONAL_BYTES_SCHEMA : Schema.BYTES_SCHEMA; } else { key = sourceRecord.getKey().orElse(null); - keySchema = Schema.STRING_SCHEMA; + keySchema = useOptionalPrimitives ? Schema.OPTIONAL_STRING_SCHEMA : Schema.STRING_SCHEMA; } - valueSchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(sourceRecord.getSchema()); + valueSchema = PulsarSchemaToKafkaSchema + .getKafkaConnectSchema(sourceRecord.getSchema(), useOptionalPrimitives); value = KafkaConnectData.getKafkaConnectData(sourceRecord.getValue().getNativeObject(), valueSchema); } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java index 2525081a41ebb..96519e63e0afa 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java @@ -99,6 +99,12 @@ public class PulsarKafkaConnectSinkConfig implements Serializable { help = "Supply kafka record with topic name without -partition- suffix for partitioned topics.") private boolean collapsePartitionedTopics = false; + @FieldDoc( + defaultValue = "false", + help = "Pulsar schema does not contain information whether the Schema is optional, Kafka's does. \n" + + "This provides a way to force all primitive schemas to be optional for Kafka. \n") + private boolean useOptionalPrimitives = false; + public static PulsarKafkaConnectSinkConfig load(String yamlFile) throws IOException { ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); return mapper.readValue(new File(yamlFile), PulsarKafkaConnectSinkConfig.class); diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java index faf28585e8aed..21a0a0f42e025 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java @@ -115,11 +115,15 @@ public Schema schema() { } private static final ImmutableMap pulsarSchemaTypeToKafkaSchema; + private static final ImmutableMap pulsarSchemaTypeToOptionalKafkaSchema; private static final ImmutableSet kafkaLogicalSchemas; private static final AvroData avroData = new AvroData(1000); private static final Cache schemaCache = CacheBuilder.newBuilder().maximumSize(10000) .expireAfterAccess(30, TimeUnit.MINUTES).build(); + private static final Cache optionalSchemaCache = + CacheBuilder.newBuilder().maximumSize(1000) + .expireAfterAccess(30, TimeUnit.MINUTES).build(); static { pulsarSchemaTypeToKafkaSchema = ImmutableMap.builder() @@ -134,6 +138,17 @@ public Schema schema() { .put(SchemaType.BYTES, Schema.BYTES_SCHEMA) .put(SchemaType.DATE, Date.SCHEMA) .build(); + pulsarSchemaTypeToOptionalKafkaSchema = ImmutableMap.builder() + .put(SchemaType.BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA) + .put(SchemaType.INT8, Schema.OPTIONAL_INT8_SCHEMA) + .put(SchemaType.INT16, Schema.OPTIONAL_INT16_SCHEMA) + .put(SchemaType.INT32, Schema.OPTIONAL_INT32_SCHEMA) + .put(SchemaType.INT64, Schema.OPTIONAL_INT64_SCHEMA) + .put(SchemaType.FLOAT, Schema.OPTIONAL_FLOAT32_SCHEMA) + .put(SchemaType.DOUBLE, Schema.OPTIONAL_FLOAT64_SCHEMA) + .put(SchemaType.STRING, Schema.OPTIONAL_STRING_SCHEMA) + .put(SchemaType.BYTES, Schema.OPTIONAL_BYTES_SCHEMA) + .build(); kafkaLogicalSchemas = ImmutableSet.builder() .add(Timestamp.LOGICAL_NAME) .add(Date.LOGICAL_NAME) @@ -153,12 +168,33 @@ private static org.apache.avro.Schema parseAvroSchema(String schemaJson) { return parser.parse(schemaJson); } - public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) { - Schema s = getKafkaConnectSchema(pulsarSchema); - return new OptionalForcingSchema(s); + public static Schema makeOptional(Schema s) { + if (s == null || s.isOptional()) { + return s; + } + + String logicalSchemaName = s.name(); + if (kafkaLogicalSchemas.contains(logicalSchemaName)) { + return s; + } + + try { + return optionalSchemaCache.get(s, () -> new OptionalForcingSchema(s)); + } catch (ExecutionException | UncheckedExecutionException | ExecutionError ee) { + String msg = "Failed to create optional schema for " + s; + log.error(msg); + throw new IllegalStateException(msg, ee); + } } - public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) { + public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema, + boolean useOptionalPrimitives) { + return makeOptional(getKafkaConnectSchema(pulsarSchema, useOptionalPrimitives)); + + } + + public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema, + boolean useOptionalPrimitives) { if (pulsarSchema == null || pulsarSchema.getSchemaInfo() == null) { throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Schema is required.", null); } @@ -191,6 +227,11 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p throw new IllegalStateException("Unsupported Kafka Logical Schema " + logicalSchemaName); } + if (useOptionalPrimitives + && pulsarSchemaTypeToOptionalKafkaSchema.containsKey(pulsarSchema.getSchemaInfo().getType())) { + return pulsarSchemaTypeToOptionalKafkaSchema.get(pulsarSchema.getSchemaInfo().getType()); + } + if (pulsarSchemaTypeToKafkaSchema.containsKey(pulsarSchema.getSchemaInfo().getType())) { return pulsarSchemaTypeToKafkaSchema.get(pulsarSchema.getSchemaInfo().getType()); } @@ -199,8 +240,10 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p return schemaCache.get(pulsarSchema.getSchemaInfo().getSchema(), () -> { if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; - return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()), - getOptionalKafkaConnectSchema(kvSchema.getValueSchema())) + return SchemaBuilder.map( + makeOptional(getKafkaConnectSchema(kvSchema.getKeySchema(), useOptionalPrimitives)), + makeOptional(getKafkaConnectSchema(kvSchema.getValueSchema(), useOptionalPrimitives))) + .optional() .build(); } org.apache.avro.Schema avroSchema = parseAvroSchema( diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index 6ccfa3b71a2f7..5410e0bb8d664 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -162,7 +162,7 @@ public T answer(InvocationOnMock invocationOnMock) throws Throwable { } } - private String offsetTopicName = "persistent://my-property/my-ns/kafka-connect-sink-offset"; + final private String offsetTopicName = "persistent://my-property/my-ns/kafka-connect-sink-offset"; private Path file; private Map props; @@ -797,7 +797,9 @@ public void kafkaLogicalTypesTimestampTest() { .build()); org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema - .getKafkaConnectSchema(schema); + .getKafkaConnectSchema(schema, true); + + Assert.assertFalse(kafkaSchema.isOptional()); java.util.Date date = getDateFromString("12/30/1999 11:12:13"); Object connectData = KafkaConnectData @@ -815,7 +817,9 @@ public void kafkaLogicalTypesTimeTest() { .build()); org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema - .getKafkaConnectSchema(schema); + .getKafkaConnectSchema(schema, true); + + Assert.assertFalse(kafkaSchema.isOptional()); java.util.Date date = getDateFromString("01/01/1970 11:12:13"); Object connectData = KafkaConnectData @@ -833,7 +837,9 @@ public void kafkaLogicalTypesDateTest() { .build()); org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema - .getKafkaConnectSchema(schema); + .getKafkaConnectSchema(schema, true); + + Assert.assertFalse(kafkaSchema.isOptional()); java.util.Date date = getDateFromString("12/31/2022 00:00:00"); Object connectData = KafkaConnectData @@ -854,7 +860,9 @@ public void kafkaLogicalTypesDecimalTest() { .build()); org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema - .getKafkaConnectSchema(schema); + .getKafkaConnectSchema(schema, true); + + Assert.assertFalse(kafkaSchema.isOptional()); Object connectData = KafkaConnectData .getKafkaConnectData(Decimal.fromLogical(kafkaSchema, BigDecimal.valueOf(100L, 10)), kafkaSchema); @@ -874,11 +882,11 @@ public void connectDataComplexAvroSchemaGenericRecordTest() { getGenericRecord(value, pulsarAvroSchema)); org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema - .getKafkaConnectSchema(Schema.KeyValue(pulsarAvroSchema, pulsarAvroSchema)); + .getKafkaConnectSchema(Schema.KeyValue(pulsarAvroSchema, pulsarAvroSchema), false); - Object connectData = KafkaConnectData.getKafkaConnectData(kv, kafkaSchema); - - org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData); + Assert.assertTrue(kafkaSchema.isOptional()); + Assert.assertTrue(kafkaSchema.keySchema().isOptional()); + Assert.assertTrue(kafkaSchema.valueSchema().isOptional()); } @Test @@ -990,7 +998,8 @@ private void testPojoAsAvroAndJsonConversionToConnectData(Object pojo, AvroSchem Object value = pojoAsAvroRecord(pojo, pulsarAvroSchema); org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema - .getKafkaConnectSchema(pulsarAvroSchema); + .getKafkaConnectSchema(pulsarAvroSchema, false); + Assert.assertFalse(kafkaSchema.isOptional()); Object connectData = KafkaConnectData.getKafkaConnectData(value, kafkaSchema); @@ -999,6 +1008,18 @@ private void testPojoAsAvroAndJsonConversionToConnectData(Object pojo, AvroSchem Object jsonNode = pojoAsJsonNode(pojo); connectData = KafkaConnectData.getKafkaConnectData(jsonNode, kafkaSchema); org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData); + + kafkaSchema = PulsarSchemaToKafkaSchema + .getKafkaConnectSchema(pulsarAvroSchema, true); + Assert.assertFalse(kafkaSchema.isOptional()); + + connectData = KafkaConnectData.getKafkaConnectData(value, kafkaSchema); + + org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData); + + jsonNode = pojoAsJsonNode(pojo); + connectData = KafkaConnectData.getKafkaConnectData(jsonNode, kafkaSchema); + org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData); } private JsonNode pojoAsJsonNode(Object pojo) { diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java index 9cc6db034c870..b236365bbb8a1 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java @@ -32,6 +32,7 @@ import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; import org.apache.pulsar.io.kafka.connect.schema.KafkaConnectData; import org.apache.pulsar.io.kafka.connect.schema.PulsarSchemaToKafkaSchema; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.math.BigInteger; @@ -39,6 +40,8 @@ import java.util.Map; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; /** * Test the conversion of PulsarSchema To KafkaSchema\. @@ -132,101 +135,134 @@ static class ComplexStruct { String[] stringArr; } - @Test - public void bytesSchemaTest() { + @DataProvider(name = "useOptionalPrimitives") + public static Object[][] useOptionalPrimitives() { + return new Object[][] { + {true}, + {false} + }; + } + + @Test(dataProvider = "useOptionalPrimitives") + public void bytesSchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTES); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTES, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BYTES); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTEBUFFER); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTEBUFFER, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BYTES); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void stringSchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void stringSchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.STRING); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.STRING, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRING); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void booleanSchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void booleanSchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BOOL); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BOOL, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BOOLEAN); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void int8SchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void int8SchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT8); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT8, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT8); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void int16SchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void int16SchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT16); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT16, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT16); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void int32SchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void int32SchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT32); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT32, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT32); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void int64SchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void int64SchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT64); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT64, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT64); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void float32SchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void float32SchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.FLOAT); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.FLOAT, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.FLOAT32); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void float64SchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void float64SchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DOUBLE); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DOUBLE, useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.FLOAT64); + assertEquals(useOptionalPrimitives, kafkaSchema.isOptional()); } - @Test - public void kvBytesSchemaTest() { + @Test(dataProvider = "useOptionalPrimitives") + public void kvBytesSchemaTest(boolean useOptionalPrimitives) { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.KV_BYTES()); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.KV_BYTES(), useOptionalPrimitives); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.MAP); assertEquals(kafkaSchema.keySchema().type(), org.apache.kafka.connect.data.Schema.Type.BYTES); assertEquals(kafkaSchema.valueSchema().type(), org.apache.kafka.connect.data.Schema.Type.BYTES); + assertTrue(kafkaSchema.isOptional()); + + // key and value are always optional + assertTrue(kafkaSchema.keySchema().isOptional()); + assertTrue(kafkaSchema.valueSchema().isOptional()); } @Test public void kvBytesIntSchemaTests() { Schema pulsarKvSchema = KeyValueSchemaImpl.of(Schema.STRING, Schema.INT64); org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarKvSchema); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarKvSchema, false); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.MAP); assertEquals(kafkaSchema.keySchema().type(), org.apache.kafka.connect.data.Schema.Type.STRING); assertEquals(kafkaSchema.valueSchema().type(), org.apache.kafka.connect.data.Schema.Type.INT64); + assertTrue(kafkaSchema.isOptional()); + + // key and value are always optional + assertTrue(kafkaSchema.keySchema().isOptional()); + assertTrue(kafkaSchema.valueSchema().isOptional()); } @Test public void avroSchemaTest() { AvroSchema pulsarAvroSchema = AvroSchema.of(StructWithAnnotations.class); org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, false); + org.apache.kafka.connect.data.Schema kafkaSchemaOpt = + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, true); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT); assertEquals(kafkaSchema.fields().size(), STRUCT_FIELDS.size()); for (String name: STRUCT_FIELDS) { assertEquals(kafkaSchema.field(name).name(), name); + // set by avro schema + assertEquals(kafkaSchema.field(name).schema().isOptional(), + kafkaSchemaOpt.field(name).schema().isOptional()); } } @@ -234,11 +270,16 @@ public void avroSchemaTest() { public void avroComplexSchemaTest() { AvroSchema pulsarAvroSchema = AvroSchema.of(ComplexStruct.class); org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, false); + org.apache.kafka.connect.data.Schema kafkaSchemaOpt = + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, true); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT); assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size()); for (String name: COMPLEX_STRUCT_FIELDS) { assertEquals(kafkaSchema.field(name).name(), name); + // set by avro schema + assertEquals(kafkaSchema.field(name).schema().isOptional(), + kafkaSchemaOpt.field(name).schema().isOptional()); } } @@ -250,11 +291,16 @@ public void jsonSchemaTest() { .withAlwaysAllowNull(false) .build()); org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, false); + org.apache.kafka.connect.data.Schema kafkaSchemaOpt = + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, true); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT); assertEquals(kafkaSchema.fields().size(), STRUCT_FIELDS.size()); for (String name: STRUCT_FIELDS) { assertEquals(kafkaSchema.field(name).name(), name); + // set by schema + assertEquals(kafkaSchema.field(name).schema().isOptional(), + kafkaSchemaOpt.field(name).schema().isOptional()); } } @@ -266,11 +312,27 @@ public void jsonComplexSchemaTest() { .withAlwaysAllowNull(false) .build()); org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, false); assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT); assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size()); for (String name: COMPLEX_STRUCT_FIELDS) { assertEquals(kafkaSchema.field(name).name(), name); + assertFalse(kafkaSchema.field(name).schema().isOptional()); + } + + kafkaSchema = + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, true); + assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT); + assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size()); + for (String name: COMPLEX_STRUCT_FIELDS) { + assertEquals(kafkaSchema.field(name).name(), name); + assertFalse(kafkaSchema.field(name).schema().isOptional()); + + if (kafkaSchema.field(name).schema().type().isPrimitive()) { + // false because .withAlwaysAllowNull(false), avroschema values are used + assertFalse(kafkaSchema.field(name).schema().isOptional(), + kafkaSchema.field(name).schema().type().getName()); + } } } @@ -308,39 +370,40 @@ public void castToKafkaSchemaTest() { @Test public void dateSchemaTest() { org.apache.kafka.connect.data.Schema kafkaSchema = - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DATE); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DATE, true); assertEquals(kafkaSchema.type(), Date.SCHEMA.type()); + assertFalse(kafkaSchema.isOptional()); } // not supported schemas below: @Test(expectedExceptions = IllegalStateException.class) public void timeSchemaTest() { - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIME); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIME, false); } @Test(expectedExceptions = IllegalStateException.class) public void timestampSchemaTest() { - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIMESTAMP); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIMESTAMP, false); } @Test(expectedExceptions = IllegalStateException.class) public void instantSchemaTest() { - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INSTANT); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INSTANT, false); } @Test(expectedExceptions = IllegalStateException.class) public void localDateSchemaTest() { - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE, false); } @Test(expectedExceptions = IllegalStateException.class) public void localTimeSchemaTest() { - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_TIME); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_TIME, false); } @Test(expectedExceptions = IllegalStateException.class) public void localDatetimeSchemaTest() { - PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE_TIME); + PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE_TIME, false); } } From 68c10eed7604aa3dcc3a6d8b548575e99b94dca2 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 30 Mar 2023 16:32:13 +0800 Subject: [PATCH 046/280] [feat][broker][PIP-195] Add metrics for bucket delayed message tracker (#19716) --- .../pulsar/broker/ServiceConfiguration.java | 2 +- .../delayed/DelayedDeliveryTracker.java | 6 - .../InMemoryDelayedDeliveryTracker.java | 5 - .../BookkeeperBucketSnapshotStorage.java | 15 +- .../bucket/BucketDelayedDeliveryTracker.java | 74 +++++++-- .../BucketDelayedMessageIndexStats.java | 146 ++++++++++++++++++ .../delayed/bucket/ImmutableBucket.java | 28 +++- .../broker/delayed/bucket/MutableBucket.java | 6 +- ...PersistentDispatcherMultipleConsumers.java | 15 +- .../persistent/PersistentSubscription.java | 3 + .../service/persistent/PersistentTopic.java | 9 ++ .../prometheus/AggregatedNamespaceStats.java | 19 +++ .../AggregatedSubscriptionStats.java | 3 + .../prometheus/NamespaceStatsAggregator.java | 13 +- .../broker/stats/prometheus/TopicStats.java | 22 +++ .../delayed/MockBucketSnapshotStorage.java | 2 +- .../BucketDelayedDeliveryTrackerTest.java | 23 +-- .../persistent/BucketDelayedDeliveryTest.java | 143 +++++++++++++++++ .../data/stats/SubscriptionStatsImpl.java | 12 ++ .../policies/data/stats/TopicMetricBean.java | 30 ++++ .../policies/data/stats/TopicStatsImpl.java | 13 ++ 21 files changed, 532 insertions(+), 57 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 03e0fec0a5346..52cebe15f6a5c 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -366,7 +366,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, private int delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds = 300; @FieldContext(category = CATEGORY_SERVER, doc = """ - The max number of delayed message index in per bucket snapshot segment, -1 means no limitation\ + The max number of delayed message index in per bucket snapshot segment, -1 means no limitation, \ after reaching the max number limitation, the snapshot segment will be cut off.""") private int delayedDeliveryMaxIndexesPerBucketSnapshotSegment = 5000; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java index 3cc2da8db1e4d..78229fef25a5a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java @@ -67,12 +67,6 @@ public interface DelayedDeliveryTracker extends AutoCloseable { */ boolean shouldPauseAllDeliveries(); - /** - * Tells whether this DelayedDeliveryTracker contains this message index, - * if the tracker is not supported it or disabled this feature also will return false. - */ - boolean containsMessage(long ledgerId, long entryId); - /** * Reset tick time use zk policies cache. * @param tickTime diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 8de6ee58e2ce5..58358b06a46bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -178,11 +178,6 @@ && getNumberOfDelayedMessages() >= fixedDelayDetectionLookahead && !hasMessageAvailable(); } - @Override - public boolean containsMessage(long ledgerId, long entryId) { - return false; - } - protected long nextDeliveryTime() { return priorityQueue.peekN1(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 08202bb19155d..e7d4f9301dd36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -67,7 +67,7 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntryThenCloseLedger(ledgerHandle, 0, 0). + ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0). thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); } @@ -75,17 +75,13 @@ public CompletableFuture getBucketSnapshotMetadata(long bucket public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntryThenCloseLedger(ledgerHandle, firstSegmentEntryId, + ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries)); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenApply(ledgerHandle -> { - long length = ledgerHandle.getLength(); - closeLedger(ledgerHandle); - return length; - }); + return openLedger(bucketId).thenApply(LedgerHandle::getLength); } @Override @@ -212,8 +208,8 @@ private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data) }); } - CompletableFuture> getLedgerEntryThenCloseLedger(LedgerHandle ledger, - long firstEntryId, long lastEntryId) { + CompletableFuture> getLedgerEntry(LedgerHandle ledger, + long firstEntryId, long lastEntryId) { final CompletableFuture> future = new CompletableFuture<>(); ledger.asyncReadEntries(firstEntryId, lastEntryId, (rc, handle, entries, ctx) -> { @@ -222,7 +218,6 @@ CompletableFuture> getLedgerEntryThenCloseLedger(Ledger } else { future.complete(entries); } - closeLedger(handle); }, null ); return future; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index ef7be187cec3d..a34bd51af98e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -57,6 +57,7 @@ import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.roaringbitmap.RoaringBitmap; @@ -69,6 +70,10 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker static final int AsyncOperationTimeoutSeconds = 60; + private static final Long INVALID_BUCKET_ID = -1L; + + private static final int MAX_MERGE_NUM = 4; + private final long minIndexCountPerBucket; private final long timeStepPerBucketSnapshotSegmentInMillis; @@ -93,9 +98,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final Table snapshotSegmentLastIndexTable; - private static final Long INVALID_BUCKET_ID = -1L; - - private static final int MAX_MERGE_NUM = 4; + private final BucketDelayedMessageIndexStats stats; public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, @@ -125,6 +128,7 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat this.lastMutableBucket = new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(), bucketSnapshotStorage); + this.stats = new BucketDelayedMessageIndexStats(); this.numberDelayedMessages = recoverBucketSnapshot(); } @@ -161,8 +165,9 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException { } try { - FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); + FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 2, TimeUnit.SECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { + log.error("[{}] Failed to recover delayed message index bucket snapshot.", dispatcher.getName(), e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -193,7 +198,7 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException { ImmutableBucket immutableBucket = mapEntry.getValue(); immutableBucketMap.remove(key); // delete asynchronously without waiting for completion - immutableBucket.asyncDeleteBucketSnapshot(); + immutableBucket.asyncDeleteBucketSnapshot(stats); } MutableLong numberDelayedMessages = new MutableLong(0); @@ -246,7 +251,8 @@ private Optional findImmutableBucket(long ledgerId) { return Optional.ofNullable(immutableBuckets.get(ledgerId)); } - private void afterCreateImmutableBucket(Pair immutableBucketDelayedIndexPair) { + private void afterCreateImmutableBucket(Pair immutableBucketDelayedIndexPair, + long startTime) { if (immutableBucketDelayedIndexPair != null) { ImmutableBucket immutableBucket = immutableBucketDelayedIndexPair.getLeft(); immutableBuckets.put(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), @@ -260,14 +266,19 @@ private void afterCreateImmutableBucket(Pair immu CompletableFuture future = createFuture.handle((bucketId, ex) -> { if (ex == null) { immutableBucket.setSnapshotSegments(null); + immutableBucket.asyncUpdateSnapshotLength(); log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(), immutableBucket.bucketKey()); + + stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.create, + System.currentTimeMillis() - startTime); + return bucketId; } - //TODO Record create snapshot failed - log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", - dispatcher.getName(), immutableBucket.bucketKey(), ex); + log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", dispatcher.getName(), + immutableBucket.bucketKey(), ex); + stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.create); // Put indexes back into the shared queue and downgrade to memory mode synchronized (BucketDelayedDeliveryTracker.this) { @@ -311,12 +322,14 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver if (!existBucket && ledgerId > lastMutableBucket.endLedgerId && lastMutableBucket.size() >= minIndexCountPerBucket && !lastMutableBucket.isEmpty()) { + long createStartTime = System.currentTimeMillis(); + stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.create); Pair immutableBucketDelayedIndexPair = lastMutableBucket.sealBucketAndAsyncPersistent( this.timeStepPerBucketSnapshotSegmentInMillis, this.maxIndexesPerBucketSnapshotSegment, this.sharedBucketPriorityQueue); - afterCreateImmutableBucket(immutableBucketDelayedIndexPair); + afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime); lastMutableBucket.resetLastMutableBucketRange(); if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { @@ -374,7 +387,7 @@ private synchronized List selectMergedBuckets(final List= 0) { - return values.subList(minIndex, minIndex + MAX_MERGE_NUM); + return values.subList(minIndex, minIndex + mergeNum); } else if (mergeNum > 2){ return selectMergedBuckets(values, mergeNum - 1); } else { @@ -400,6 +413,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) { immutableBucket.merging = true; } + + long mergeStartTime = System.currentTimeMillis(); + stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.merge); return asyncMergeBucketSnapshot(toBeMergeImmutableBuckets).whenComplete((__, ex) -> { synchronized (this) { for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) { @@ -409,9 +425,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { if (ex != null) { log.error("[{}] Failed to merge bucket snapshot, bucketKeys: {}", dispatcher.getName(), bucketsStr, ex); + + stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.merge); } else { log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}", dispatcher.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size()); + + stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.merge, + System.currentTimeMillis() - mergeStartTime); } }); } @@ -436,6 +457,8 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List { synchronized (BucketDelayedDeliveryTracker.this) { + long createStartTime = System.currentTimeMillis(); + stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.create); Pair immutableBucketDelayedIndexPair = lastMutableBucket.createImmutableBucketAndAsyncPersistent( timeStepPerBucketSnapshotSegmentInMillis, @@ -461,12 +484,12 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List { List> removeFutures = - buckets.stream().map(ImmutableBucket::asyncDeleteBucketSnapshot) + buckets.stream().map(bucket -> bucket.asyncDeleteBucketSnapshot(stats)) .toList(); return FutureUtil.waitForAll(removeFutures); }); @@ -557,15 +580,17 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) { immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); - bucket.asyncDeleteBucketSnapshot(); + bucket.asyncDeleteBucketSnapshot(stats); continue; } + long loadStartTime = System.currentTimeMillis(); + stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load); bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> { if (CollectionUtils.isEmpty(indexList)) { immutableBuckets.asMapOfRanges() .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); - bucket.asyncDeleteBucketSnapshot(); + bucket.asyncDeleteBucketSnapshot(stats); return; } DelayedMessageIndexBucketSnapshotFormat.DelayedIndex @@ -583,9 +608,14 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}", dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); + + stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load); } else { log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); + + stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, + System.currentTimeMillis() - loadStartTime); } }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS); } catch (Exception e) { @@ -645,7 +675,7 @@ private CompletableFuture cleanImmutableBuckets() { Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); while (iterator.hasNext()) { ImmutableBucket bucket = iterator.next(); - futures.add(bucket.clear()); + futures.add(bucket.clear(stats)); numberDelayedMessages -= bucket.getNumberBucketDelayedMessages(); iterator.remove(); } @@ -661,7 +691,6 @@ private boolean removeIndexBit(long ledgerId, long entryId) { .orElse(false); } - @Override public boolean containsMessage(long ledgerId, long entryId) { if (lastMutableBucket.containsMessage(ledgerId, entryId)) { return true; @@ -670,4 +699,15 @@ public boolean containsMessage(long ledgerId, long entryId) { return findImmutableBucket(ledgerId).map(bucket -> bucket.containsMessage(ledgerId, entryId)) .orElse(false); } + + public Map genTopicMetricMap() { + stats.recordNumOfBuckets(immutableBuckets.asMapOfRanges().size() + 1); + stats.recordDelayedMessageIndexLoaded(this.sharedBucketPriorityQueue.size() + this.lastMutableBucket.size()); + MutableLong totalSnapshotLength = new MutableLong(); + immutableBuckets.asMapOfRanges().values().forEach(immutableBucket -> { + totalSnapshotLength.add(immutableBucket.getSnapshotLength()); + }); + stats.recordBucketSnapshotSizeBytes(totalSnapshotLength.longValue()); + return stats.genTopicMetricMap(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java new file mode 100644 index 0000000000000..68788c359d560 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; +import org.apache.bookkeeper.mledger.util.StatsBuckets; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; + +public class BucketDelayedMessageIndexStats { + + private static final long[] BUCKETS = new long[]{50, 100, 500, 1000, 5000, 30000, 60000}; + + enum State { + succeed, + failed, + all + } + + enum Type { + create, + load, + delete, + merge + } + + private static final String BUCKET_TOTAL_NAME = "pulsar_delayed_message_index_bucket_total"; + private static final String INDEX_LOADED_NAME = "pulsar_delayed_message_index_loaded"; + private static final String SNAPSHOT_SIZE_BYTES_NAME = "pulsar_delayed_message_index_bucket_snapshot_size_bytes"; + private static final String OP_COUNT_NAME = "pulsar_delayed_message_index_bucket_op_count"; + private static final String OP_LATENCY_NAME = "pulsar_delayed_message_index_bucket_op_latency_ms"; + + private final AtomicInteger delayedMessageIndexBucketTotal = new AtomicInteger(); + private final AtomicLong delayedMessageIndexLoaded = new AtomicLong(); + private final AtomicLong delayedMessageIndexBucketSnapshotSizeBytes = new AtomicLong(); + private final Map delayedMessageIndexBucketOpLatencyMs = new ConcurrentHashMap<>(); + private final Map delayedMessageIndexBucketOpCount = new ConcurrentHashMap<>(); + + public BucketDelayedMessageIndexStats() { + } + + public Map genTopicMetricMap() { + Map metrics = new HashMap<>(); + + metrics.put(BUCKET_TOTAL_NAME, + new TopicMetricBean(BUCKET_TOTAL_NAME, delayedMessageIndexBucketTotal.get(), null)); + + metrics.put(INDEX_LOADED_NAME, + new TopicMetricBean(INDEX_LOADED_NAME, delayedMessageIndexLoaded.get(), null)); + + metrics.put(SNAPSHOT_SIZE_BYTES_NAME, + new TopicMetricBean(SNAPSHOT_SIZE_BYTES_NAME, delayedMessageIndexBucketSnapshotSizeBytes.get(), null)); + + delayedMessageIndexBucketOpCount.forEach((k, count) -> { + String[] labels = splitKey(k); + String[] labelsAndValues = new String[] {"state", labels[0], "type", labels[1]}; + String key = OP_COUNT_NAME + joinKey(labelsAndValues); + metrics.put(key, new TopicMetricBean(OP_COUNT_NAME, count.sumThenReset(), labelsAndValues)); + }); + + delayedMessageIndexBucketOpLatencyMs.forEach((typeName, statsBuckets) -> { + statsBuckets.refresh(); + long[] buckets = statsBuckets.getBuckets(); + for (int i = 0; i < buckets.length; i++) { + long count = buckets[i]; + if (count == 0L) { + continue; + } + String quantile; + if (i == BUCKETS.length) { + quantile = "overflow"; + } else { + quantile = String.valueOf(BUCKETS[i]); + } + String[] labelsAndValues = new String[] {"type", typeName, "quantile", quantile}; + String key = OP_LATENCY_NAME + joinKey(labelsAndValues); + + metrics.put(key, new TopicMetricBean(OP_LATENCY_NAME, count, labelsAndValues)); + } + String[] labelsAndValues = new String[] {"type", typeName}; + metrics.put(OP_LATENCY_NAME + "_count" + joinKey(labelsAndValues), + new TopicMetricBean(OP_LATENCY_NAME + "_count", statsBuckets.getCount(), labelsAndValues)); + metrics.put(OP_LATENCY_NAME + "_sum" + joinKey(labelsAndValues), + new TopicMetricBean(OP_LATENCY_NAME + "_sum", statsBuckets.getSum(), labelsAndValues)); + }); + + return metrics; + } + + public void recordNumOfBuckets(int numOfBuckets) { + delayedMessageIndexBucketTotal.set(numOfBuckets); + } + + public void recordDelayedMessageIndexLoaded(long num) { + delayedMessageIndexLoaded.set(num); + } + + public void recordBucketSnapshotSizeBytes(long sizeBytes) { + delayedMessageIndexBucketSnapshotSizeBytes.set(sizeBytes); + } + + public void recordTriggerEvent(Type eventType) { + delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.all.name(), eventType.name()), + k -> new LongAdder()).increment(); + } + + public void recordSuccessEvent(Type eventType, long cost) { + delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.succeed.name(), eventType.name()), + k -> new LongAdder()).increment(); + delayedMessageIndexBucketOpLatencyMs.computeIfAbsent(eventType.name(), + k -> new StatsBuckets(BUCKETS)).addValue(cost); + } + + public void recordFailEvent(Type eventType) { + delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.failed.name(), eventType.name()), + k -> new LongAdder()).increment(); + } + + public static String joinKey(String... values) { + return String.join("_", values); + } + + public static String[] splitKey(String key) { + return key.split("_"); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 969d326e28187..82e98cefa5d98 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -131,6 +131,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b List indexList = snapshotSegment.getIndexesList(); this.setCurrentSegmentEntryId(nextSegmentEntryId); + if (isRecover) { + this.asyncUpdateSnapshotLength(); + } return indexList; }); }); @@ -175,7 +178,9 @@ CompletableFuture> }, BucketSnapshotPersistenceException.class, MaxRetryTimes); } - CompletableFuture asyncDeleteBucketSnapshot() { + CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats stats) { + long deleteStartTime = System.currentTimeMillis(); + stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.delete); String bucketKey = bucketKey(); long bucketId = getAndUpdateBucketId(); return removeBucketCursorProperty(bucketKey).thenCompose(__ -> @@ -184,16 +189,33 @@ CompletableFuture asyncDeleteBucketSnapshot() { if (ex != null) { log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", dispatcherName, bucketId, bucketKey, ex); + + stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete); } else { log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", dispatcherName, bucketId, bucketKey); + + stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete, + System.currentTimeMillis() - deleteStartTime); } }); } - CompletableFuture clear() { + CompletableFuture clear(BucketDelayedMessageIndexStats stats) { delayedIndexBitMap.clear(); return getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null) - .thenCompose(__ -> asyncDeleteBucketSnapshot()); + .thenCompose(__ -> asyncDeleteBucketSnapshot(stats)); + } + + protected CompletableFuture asyncUpdateSnapshotLength() { + long bucketId = getAndUpdateBucketId(); + return bucketSnapshotStorage.getBucketSnapshotLength(bucketId).whenComplete((length, ex) -> { + if (ex != null) { + log.error("[{}] Failed to get snapshot length, bucketId: {}, bucketKey: {}", + dispatcherName, bucketId, bucketKey(), ex); + } else { + setSnapshotLength(length); + } + }); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index f8a4ecc7a4ddb..e49ebe9606e01 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -62,8 +62,10 @@ Pair createImmutableBucketAndAsyncPersistent( final long timeStepPerBucketSnapshotSegment, final int maxIndexesPerBucketSnapshotSegment, TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId, final long endLedgerId) { - log.info("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName, - startLedgerId, endLedgerId); + if (log.isDebugEnabled()) { + log.debug("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName, + startLedgerId, endLedgerId); + } if (delayedIndexQueue.isEmpty()) { return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index e5c9e85bac3f2..7ff6e72d02aed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -72,6 +72,7 @@ import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.Codec; import org.slf4j.Logger; @@ -332,7 +333,7 @@ public synchronized void readMoreEntries() { Predicate skipCondition = null; final DelayedDeliveryTracker deliveryTracker = delayedDeliveryTracker.get(); if (deliveryTracker instanceof BucketDelayedDeliveryTracker) { - skipCondition = position -> deliveryTracker + skipCondition = position -> ((BucketDelayedDeliveryTracker) deliveryTracker) .containsMessage(position.getLedgerId(), position.getEntryId()); } cursor.asyncReadEntriesWithSkipOrWait(messagesToRead, bytesToRead, this, ReadType.Normal, @@ -1180,6 +1181,18 @@ public long getDelayedTrackerMemoryUsage() { return 0; } + public Map getBucketDelayedIndexStats() { + if (delayedDeliveryTracker.isEmpty()) { + return Collections.emptyMap(); + } + + if (delayedDeliveryTracker.get() instanceof BucketDelayedDeliveryTracker) { + return ((BucketDelayedDeliveryTracker) delayedDeliveryTracker.get()).genTopicMetricMap(); + } + + return Collections.emptyMap(); + } + public ManagedCursor getCursor() { return cursor; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index bdb3c9fc391ce..4ed191a9b4f61 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -1147,6 +1147,9 @@ public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscri if (dispatcher instanceof PersistentDispatcherMultipleConsumers) { subStats.delayedMessageIndexSizeInBytes = ((PersistentDispatcherMultipleConsumers) dispatcher).getDelayedTrackerMemoryUsage(); + + subStats.bucketDelayedIndexStats = + ((PersistentDispatcherMultipleConsumers) dispatcher).getBucketDelayedIndexStats(); } if (Subscription.isIndividualAckMode(subType)) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 82a4f5312357a..fa08330ff3c35 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -153,6 +153,7 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; @@ -2183,6 +2184,14 @@ public CompletableFuture asyncGetStats(boolean getPreciseBacklog stats.nonContiguousDeletedMessagesRangesSerializedSize += subStats.nonContiguousDeletedMessagesRangesSerializedSize; stats.delayedMessageIndexSizeInBytes += subStats.delayedMessageIndexSizeInBytes; + + subStats.bucketDelayedIndexStats.forEach((k, v) -> { + TopicMetricBean topicMetricBean = + stats.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean()); + topicMetricBean.name = v.name; + topicMetricBean.labelsAndValues = v.labelsAndValues; + topicMetricBean.value += v.value; + }); }); replicators.forEach((cluster, replicator) -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index 0a905daa341f3..ea77bd69302a0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.bookkeeper.mledger.util.StatsBuckets; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.compaction.CompactionRecord; public class AggregatedNamespaceStats { @@ -65,6 +66,8 @@ public class AggregatedNamespaceStats { StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC); int delayedMessageIndexSizeInBytes; + Map bucketDelayedIndexStats = new HashMap<>(); + void updateStats(TopicStats stats) { topicsCount++; @@ -83,6 +86,14 @@ void updateStats(TopicStats stats) { msgOutCounter += stats.msgOutCounter; delayedMessageIndexSizeInBytes += stats.delayedMessageIndexSizeInBytes; + stats.bucketDelayedIndexStats.forEach((k, v) -> { + TopicMetricBean topicMetricBean = + bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean()); + topicMetricBean.name = v.name; + topicMetricBean.labelsAndValues = v.labelsAndValues; + topicMetricBean.value += v.value; + }); + this.ongoingTxnCount += stats.ongoingTxnCount; this.abortedTxnCount += stats.abortedTxnCount; this.committedTxnCount += stats.committedTxnCount; @@ -132,6 +143,13 @@ void updateStats(TopicStats stats) { subsStats.filterRejectedMsgCount += as.filterRejectedMsgCount; subsStats.filterRescheduledMsgCount += as.filterRescheduledMsgCount; subsStats.delayedMessageIndexSizeInBytes += as.delayedMessageIndexSizeInBytes; + as.bucketDelayedIndexStats.forEach((k, v) -> { + TopicMetricBean topicMetricBean = + subsStats.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean()); + topicMetricBean.name = v.name; + topicMetricBean.labelsAndValues = v.labelsAndValues; + topicMetricBean.value += v.value; + }); as.consumerStat.forEach((c, v) -> { AggregatedConsumerStats consumerStats = subsStats.consumerStat.computeIfAbsent(c, k -> new AggregatedConsumerStats()); @@ -172,5 +190,6 @@ public void reset() { replicationStats.clear(); subscriptionStats.clear(); delayedMessageIndexSizeInBytes = 0; + bucketDelayedIndexStats.clear(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java index 383c671754dc1..da0324c55655c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; public class AggregatedSubscriptionStats { @@ -75,4 +76,6 @@ public class AggregatedSubscriptionStats { public Map consumerStat = new HashMap<>(); long delayedMessageIndexSizeInBytes; + + public Map bucketDelayedIndexStats = new HashMap<>(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 918aef539cff4..32fb06ea3ce8c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -28,6 +28,7 @@ import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; +import org.apache.commons.lang3.ArrayUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -155,6 +156,7 @@ private static void aggregateTopicStats(TopicStats stats, SubscriptionStatsImpl subsStats.filterRejectedMsgCount = subscriptionStats.filterRejectedMsgCount; subsStats.filterRescheduledMsgCount = subscriptionStats.filterRescheduledMsgCount; subsStats.delayedMessageIndexSizeInBytes = subscriptionStats.delayedMessageIndexSizeInBytes; + subsStats.bucketDelayedIndexStats = subscriptionStats.bucketDelayedIndexStats; } private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics, @@ -197,6 +199,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include stats.averageMsgSize = tStatus.averageMsgSize; stats.publishRateLimitedTimes = tStatus.publishRateLimitedTimes; stats.delayedMessageIndexSizeInBytes = tStatus.delayedMessageIndexSizeInBytes; + stats.bucketDelayedIndexStats = tStatus.bucketDelayedIndexStats; stats.abortedTxnCount = tStatus.abortedTxnCount; stats.ongoingTxnCount = tStatus.ongoingTxnCount; stats.committedTxnCount = tStatus.committedTxnCount; @@ -379,6 +382,10 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes, cluster, namespace); + stats.bucketDelayedIndexStats.forEach((k, metric) -> { + writeMetric(stream, metric.name, metric.value, cluster, namespace, metric.labelsAndValues); + }); + writePulsarMsgBacklog(stream, stats.msgBacklog, cluster, namespace); stats.managedLedgerStats.storageWriteLatencyBuckets.refresh(); @@ -472,8 +479,10 @@ private static void writeMetric(PrometheusMetricStreams stream, String metricNam } private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, - String namespace) { - stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace); + String namespace, String... extraLabelsAndValues) { + String[] labelsAndValues = new String[]{"cluster", cluster, "namespace", namespace}; + String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues); + stream.writeSample(metricName, value, labels); } private static void writeReplicationStat(PrometheusMetricStreams stream, String metricName, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index abc6979484e58..3a2563a87587b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -25,6 +25,7 @@ import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.lang3.ArrayUtils; import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.compaction.CompactionRecord; import org.apache.pulsar.compaction.CompactorMXBean; @@ -70,6 +71,8 @@ class TopicStats { StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC); public long delayedMessageIndexSizeInBytes; + Map bucketDelayedIndexStats = new HashMap<>(); + public void reset() { subscriptionsCount = 0; producersCount = 0; @@ -107,6 +110,7 @@ public void reset() { compactionCompactedEntriesSize = 0; compactionLatencyBuckets.reset(); delayedMessageIndexSizeInBytes = 0; + bucketDelayedIndexStats.clear(); } public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats, @@ -162,6 +166,11 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + for (TopicMetricBean topicMetricBean : stats.bucketDelayedIndexStats.values()) { + writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace, + topic, splitTopicAndPartitionIndexLabel, topicMetricBean.labelsAndValues); + } + long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); writeMetric(stream, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); @@ -310,6 +319,13 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st subsStats.delayedMessageIndexSizeInBytes, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + final String[] subscriptionLabel = {"subscription", sub}; + for (TopicMetricBean topicMetricBean : subsStats.bucketDelayedIndexStats.values()) { + String[] labelsAndValues = ArrayUtils.addAll(subscriptionLabel, topicMetricBean.labelsAndValues); + writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace, + topic, splitTopicAndPartitionIndexLabel, labelsAndValues); + } + subsStats.consumerStat.forEach((c, consumerStats) -> { writeConsumerMetric(stream, "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver, cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); @@ -409,6 +425,12 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st writeMetric(stream, "pulsar_compaction_latency_count", stats.compactionLatencyBuckets.getCount(), cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + + for (TopicMetricBean topicMetricBean : stats.bucketDelayedIndexStats.values()) { + String[] labelsAndValues = topicMetricBean.labelsAndValues; + writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace, + topic, splitTopicAndPartitionIndexLabel, labelsAndValues); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index cf7310c7067c3..9e924bdeda341 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -174,7 +174,7 @@ public CompletableFuture getBucketSnapshotLength(long bucketId) { long length = 0; List bufList = this.bucketSnapshots.get(bucketId); for (ByteBuf byteBuf : bufList) { - length += byteBuf.array().length; + length += byteBuf.readableBytes(); } return length; }, executorService); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 717bada7705dc..95234d688f6a0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -46,7 +46,6 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest; -import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.MockManagedCursor; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; @@ -158,7 +157,7 @@ public Object[][] provider(Method method) throws Exception { } @Test(dataProvider = "delayedTracker") - public void testContainsMessage(DelayedDeliveryTracker tracker) { + public void testContainsMessage(BucketDelayedDeliveryTracker tracker) { tracker.addMessage(1, 1, 10); tracker.addMessage(2, 2, 20); @@ -191,6 +190,12 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { clockTime.set(1 * 10); + Awaitility.await().untilAsserted(() -> { + Assert.assertTrue( + tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging || + !x.getSnapshotCreateFuture().get().isDone())); + }); + assertTrue(tracker.hasMessageAvailable()); Set scheduledMessages = tracker.getScheduledMessages(100); @@ -202,16 +207,16 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { clockTime.set(30 * 10); - tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, - true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,50); + BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50); - assertFalse(tracker.containsMessage(101, 101)); - assertEquals(tracker.getNumberOfDelayedMessages(), 70); + assertFalse(tracker2.containsMessage(101, 101)); + assertEquals(tracker2.getNumberOfDelayedMessages(), 70); clockTime.set(100 * 10); - assertTrue(tracker.hasMessageAvailable()); - scheduledMessages = tracker.getScheduledMessages(70); + assertTrue(tracker2.hasMessageAvailable()); + scheduledMessages = tracker2.getScheduledMessages(70); assertEquals(scheduledMessages.size(), 70); @@ -221,7 +226,7 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { i++; } - tracker.close(); + tracker2.close(); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index 09b7cbbf1b99d..5480a2e7a704a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -19,18 +19,26 @@ package org.apache.pulsar.broker.service.persistent; import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import com.google.common.collect.Multimap; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.stats.PrometheusMetricsTest; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; @@ -162,4 +170,139 @@ public void testUnsubscribe() throws Exception { } } } + + + @Test + public void testBucketDelayedIndexMetrics() throws Exception { + cleanup(); + setup(); + + String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testBucketDelayedIndexMetrics"); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("test_sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Consumer consumer2 = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("test_sub2") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + final int N = 101; + + for (int i = 0; i < N; i++) { + producer.newMessage() + .value("msg-" + i) + .deliverAfter(3600 + i, TimeUnit.SECONDS) + .sendAsync(); + } + producer.flush(); + + Thread.sleep(2000); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, true, true, output); + String metricsStr = output.toString(StandardCharsets.UTF_8); + Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr); + + List bucketsMetrics = + metricsMap.get("pulsar_delayed_message_index_bucket_total").stream() + .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); + MutableInt bucketsSum = new MutableInt(); + bucketsMetrics.stream().filter(metric -> metric.tags.containsKey("subscription")).forEach(metric -> { + assertEquals(3, metric.value); + bucketsSum.add(metric.value); + }); + assertEquals(6, bucketsSum.intValue()); + Optional bucketsTopicMetric = + bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst(); + assertTrue(bucketsTopicMetric.isPresent()); + assertEquals(bucketsSum.intValue(), bucketsTopicMetric.get().value); + + List loadedIndexMetrics = + metricsMap.get("pulsar_delayed_message_index_loaded").stream() + .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); + MutableInt loadedIndexSum = new MutableInt(); + long count = loadedIndexMetrics.stream().filter(metric -> metric.tags.containsKey("subscription")).peek(metric -> { + assertTrue(metric.value > 0 && metric.value <= N); + loadedIndexSum.add(metric.value); + }).count(); + assertEquals(2, count); + Optional loadedIndexTopicMetrics = + bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst(); + assertTrue(loadedIndexTopicMetrics.isPresent()); + assertEquals(loadedIndexSum.intValue(), loadedIndexTopicMetrics.get().value); + + List snapshotSizeBytesMetrics = + metricsMap.get("pulsar_delayed_message_index_bucket_snapshot_size_bytes").stream() + .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); + MutableInt snapshotSizeBytesSum = new MutableInt(); + count = snapshotSizeBytesMetrics.stream().filter(metric -> metric.tags.containsKey("subscription")) + .peek(metric -> { + assertTrue(metric.value > 0); + snapshotSizeBytesSum.add(metric.value); + }).count(); + assertEquals(2, count); + Optional snapshotSizeBytesTopicMetrics = + snapshotSizeBytesMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst(); + assertTrue(snapshotSizeBytesTopicMetrics.isPresent()); + assertEquals(snapshotSizeBytesSum.intValue(), snapshotSizeBytesTopicMetrics.get().value); + + List opCountMetrics = + metricsMap.get("pulsar_delayed_message_index_bucket_op_count").stream() + .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); + MutableInt opCountMetricsSum = new MutableInt(); + count = opCountMetrics.stream() + .filter(metric -> metric.tags.get("state").equals("succeed") && metric.tags.get("type").equals("create") + && metric.tags.containsKey("subscription")) + .peek(metric -> { + assertTrue(metric.value >= 2); + opCountMetricsSum.add(metric.value); + }).count(); + assertEquals(2, count); + Optional opCountTopicMetrics = + opCountMetrics.stream() + .filter(metric -> metric.tags.get("state").equals("succeed") && metric.tags.get("type") + .equals("create") && !metric.tags.containsKey("subscription")).findFirst(); + assertTrue(opCountTopicMetrics.isPresent()); + assertEquals(opCountMetricsSum.intValue(), opCountTopicMetrics.get().value); + + List opLatencyMetrics = + metricsMap.get("pulsar_delayed_message_index_bucket_op_latency_ms").stream() + .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); + MutableInt opLatencyMetricsSum = new MutableInt(); + count = opLatencyMetrics.stream() + .filter(metric -> metric.tags.get("type").equals("create") + && metric.tags.containsKey("subscription")) + .peek(metric -> { + assertTrue(metric.tags.containsKey("quantile")); + opLatencyMetricsSum.add(metric.value); + }).count(); + assertTrue(count >= 2); + Optional opLatencyTopicMetrics = + opCountMetrics.stream() + .filter(metric -> metric.tags.get("type").equals("create") + && !metric.tags.containsKey("subscription")).findFirst(); + assertTrue(opLatencyTopicMetrics.isPresent()); + assertEquals(opLatencyMetricsSum.intValue(), opLatencyTopicMetrics.get().value); + + ByteArrayOutputStream namespaceOutput = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, false, true, true, namespaceOutput); + Multimap namespaceMetricsMap = PrometheusMetricsTest.parseMetrics(namespaceOutput.toString(StandardCharsets.UTF_8)); + + Optional namespaceMetric = + namespaceMetricsMap.get("pulsar_delayed_message_index_bucket_total").stream().findFirst(); + assertTrue(namespaceMetric.isPresent()); + assertEquals(6, namespaceMetric.get().value); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index 9c7e24ba02118..25fa666523f3c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -134,6 +134,8 @@ public class SubscriptionStatsImpl implements SubscriptionStats { /** The size of InMemoryDelayedDeliveryTracer memory usage. */ public long delayedMessageIndexSizeInBytes; + public Map bucketDelayedIndexStats; + /** SubscriptionProperties (key/value strings) associated with this subscribe. */ public Map subscriptionProperties; @@ -149,6 +151,7 @@ public SubscriptionStatsImpl() { this.consumers = new ArrayList<>(); this.consumersAfterMarkDeletePosition = new LinkedHashMap<>(); this.subscriptionProperties = new HashMap<>(); + this.bucketDelayedIndexStats = new HashMap<>(); } public void reset() { @@ -175,6 +178,7 @@ public void reset() { filterAcceptedMsgCount = 0; filterRejectedMsgCount = 0; filterRescheduledMsgCount = 0; + bucketDelayedIndexStats.clear(); } // if the stats are added for the 1st time, we will need to make a copy of these stats and add it to the current @@ -215,6 +219,14 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { this.filterAcceptedMsgCount += stats.filterAcceptedMsgCount; this.filterRejectedMsgCount += stats.filterRejectedMsgCount; this.filterRescheduledMsgCount += stats.filterRescheduledMsgCount; + stats.bucketDelayedIndexStats.forEach((k, v) -> { + TopicMetricBean topicMetricBean = + this.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean()); + topicMetricBean.name = v.name; + topicMetricBean.labelsAndValues = v.labelsAndValues; + topicMetricBean.value += v.value; + }); + return this; } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java new file mode 100644 index 0000000000000..e01a9d7aa71f3 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data.stats; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@AllArgsConstructor +public class TopicMetricBean { + public String name; + public double value; + public String[] labelsAndValues; +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java index 12d30124f7dcd..c9c4739b904f6 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java @@ -139,6 +139,9 @@ public class TopicStatsImpl implements TopicStats { /** The size of InMemoryDelayedDeliveryTracer memory usage. */ public long delayedMessageIndexSizeInBytes; + /** Map of bucket delayed index statistics. */ + public Map bucketDelayedIndexStats; + /** The compaction stats. */ public CompactionStatsImpl compaction; @@ -182,6 +185,7 @@ public TopicStatsImpl() { this.subscriptions = new HashMap<>(); this.replication = new TreeMap<>(); this.compaction = new CompactionStatsImpl(); + this.bucketDelayedIndexStats = new HashMap<>(); } public void reset() { @@ -214,6 +218,7 @@ public void reset() { this.delayedMessageIndexSizeInBytes = 0; this.compaction.reset(); this.ownerBroker = null; + this.bucketDelayedIndexStats.clear(); } // if the stats are added for the 1st time, we will need to make a copy of these stats and add it to the current @@ -244,6 +249,14 @@ public TopicStatsImpl add(TopicStats ts) { this.abortedTxnCount = stats.abortedTxnCount; this.committedTxnCount = stats.committedTxnCount; + stats.bucketDelayedIndexStats.forEach((k, v) -> { + TopicMetricBean topicMetricBean = + this.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean()); + topicMetricBean.name = v.name; + topicMetricBean.labelsAndValues = v.labelsAndValues; + topicMetricBean.value += v.value; + }); + for (int index = 0; index < stats.getPublishers().size(); index++) { PublisherStats s = stats.getPublishers().get(index); if (s.isSupportsPartialProducer() && s.getProducerName() != null) { From 02147454c425b92f0cd1caefa73b9339db6a0269 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Apr 2023 14:18:02 +0800 Subject: [PATCH 047/280] [feat][client] Support configure MessageCrypto in ProducerBuilder (#19939) Signed-off-by: tison --- .../pulsar/client/api/ProducerBuilder.java | 11 ++++++ .../client/impl/ProducerBuilderImpl.java | 7 ++++ .../client/impl/ProducerBuilderImplTest.java | 39 +++++++++++-------- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java index d2231e71c237a..896c22313247a 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java @@ -389,6 +389,17 @@ public interface ProducerBuilder extends Cloneable { */ ProducerBuilder defaultCryptoKeyReader(Map publicKeys); + /** + * Sets a {@link MessageCrypto}. + * + *

Contains methods to encrypt/decrypt messages for end-to-end encryption. + * + * @param messageCrypto + * MessageCrypto object + * @return the producer builder instance + */ + ProducerBuilder messageCrypto(MessageCrypto messageCrypto); + /** * Add public encryption key, used by producer to encrypt the data key. * diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java index bfdbb63de1855..ecbdfa76b64ae 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java @@ -34,6 +34,7 @@ import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.HashingScheme; +import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.MessageRouter; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -221,6 +222,12 @@ public ProducerBuilder defaultCryptoKeyReader(@NonNull Map pu return cryptoKeyReader(DefaultCryptoKeyReader.builder().publicKeys(publicKeys).build()); } + @Override + public ProducerBuilder messageCrypto(MessageCrypto messageCrypto) { + conf.setMessageCrypto(messageCrypto); + return this; + } + @Override public ProducerBuilder addEncryptionKey(String key) { checkArgument(StringUtils.isNotBlank(key), "Encryption key cannot be blank"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java index 66c552ef7934a..bb3e3fc3accf6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java @@ -18,6 +18,15 @@ */ package org.apache.pulsar.client.impl; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertNotNull; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRouter; import org.apache.pulsar.client.api.MessageRoutingMode; @@ -26,20 +35,10 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TopicMetadata; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertNotNull; - /** * Unit tests of {@link ProducerBuilderImpl}. */ @@ -47,13 +46,13 @@ public class ProducerBuilderImplTest { private static final String TOPIC_NAME = "testTopicName"; private PulsarClientImpl client; - private ProducerBuilderImpl producerBuilderImpl; + private ProducerBuilderImpl producerBuilderImpl; @BeforeClass(alwaysRun = true) public void setup() { - Producer producer = mock(Producer.class); + Producer producer = mock(Producer.class); client = mock(PulsarClientImpl.class); - producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES); + producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); when(client.newProducer()).thenReturn(producerBuilderImpl); when(client.createProducerAsync( @@ -66,8 +65,8 @@ public void testProducerBuilderImpl() throws PulsarClientException { Map properties = new HashMap<>(); properties.put("Test-Key2", "Test-Value2"); - producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES); - Producer producer = producerBuilderImpl.topic(TOPIC_NAME) + producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); + Producer producer = producerBuilderImpl.topic(TOPIC_NAME) .producerName("Test-Producer") .maxPendingMessages(2) .addEncryptionKey("Test-EncryptionKey") @@ -78,6 +77,14 @@ public void testProducerBuilderImpl() throws PulsarClientException { assertNotNull(producer); } + @Test + public void testProducerBuilderImplWhenMessageCryptoSet() throws PulsarClientException { + producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); + producerBuilderImpl.topic(TOPIC_NAME).messageCrypto(new MessageCryptoBc("ctx1", true)); + assertNotNull(producerBuilderImpl.create()); + assertNotNull(producerBuilderImpl.getConf().getMessageCrypto()); + } + @Test public void testProducerBuilderImplWhenMessageRoutingModeAndMessageRouterAreNotSet() throws PulsarClientException { producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES); From be57e9a79b0b6fd40ef02414df7238050b7a885d Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 1 Apr 2023 19:21:58 +0800 Subject: [PATCH 048/280] [fix][schema]Fix AutoProduceBytes producer can not be created to a topic with ProtoBuf schema (#19767) ### Motivation 1. There is a topic1 with a protobuf schema. 2. Create a producer1 with AutoProduceBytes schema. 3. The producer1 will be created failed because the way to get the schema of protobuf schema is not supported. ### ### ### Modification Because the Protobuf schema is implemented from the AvroBaseStructSchema. So we add a way to get Protobuf schema just like the AvroSchema. --- .../org/apache/pulsar/schema/SchemaTest.java | 17 +++++++++++++++++ .../pulsar/client/impl/PulsarClientImpl.java | 9 ++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index d99496f4a967b..0ec72b2ef470a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -70,6 +70,7 @@ import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; +import org.apache.pulsar.client.impl.schema.ProtobufSchema; import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.client.impl.schema.writer.AvroWriter; @@ -113,6 +114,22 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testGetSchemaWhenCreateAutoProduceBytesProducer() throws Exception{ + final String tenant = PUBLIC_TENANT; + final String namespace = "test-namespace-" + randomName(16); + final String topic = tenant + "/" + namespace + "/test-getSchema"; + admin.namespaces().createNamespace( + tenant + "/" + namespace, + Sets.newHashSet(CLUSTER_NAME) + ); + + ProtobufSchema protobufSchema = + ProtobufSchema.of(org.apache.pulsar.client.api.schema.proto.Test.TestMessage.class); + pulsarClient.newProducer(protobufSchema).topic(topic).create(); + pulsarClient.newProducer(org.apache.pulsar.client.api.Schema.AUTO_PRODUCE_BYTES()).topic(topic).create(); + } + @Test public void testMultiTopicSetSchemaProvider() throws Exception { final String tenant = PUBLIC_TENANT; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index ebc11f44ce749..f37709f3d84f6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -70,6 +70,7 @@ import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.AutoProduceBytesSchema; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.client.impl.schema.generic.MultiVersionSchemaInfoProvider; import org.apache.pulsar.client.impl.transaction.TransactionBuilderImpl; import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl; @@ -81,6 +82,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.netty.EventLoopUtil; @@ -350,7 +352,12 @@ public CompletableFuture> createProducerAsync(ProducerConfigurat return lookup.getSchema(TopicName.get(conf.getTopicName())) .thenCompose(schemaInfoOptional -> { if (schemaInfoOptional.isPresent()) { - autoProduceBytesSchema.setSchema(Schema.getSchema(schemaInfoOptional.get())); + SchemaInfo schemaInfo = schemaInfoOptional.get(); + if (schemaInfo.getType() == SchemaType.PROTOBUF) { + autoProduceBytesSchema.setSchema(new GenericAvroSchema(schemaInfo)); + } else { + autoProduceBytesSchema.setSchema(Schema.getSchema(schemaInfo)); + } } else { autoProduceBytesSchema.setSchema(Schema.BYTES); } From 2dcaf0ef692c929ebf6c510d328e0f2775f07f3e Mon Sep 17 00:00:00 2001 From: LinChen Date: Mon, 3 Apr 2023 12:55:16 +0800 Subject: [PATCH 049/280] [fix][broker] Fix the loss of bundle stats data reported to zookeeper, when the updateStats method is executed (#19887) Co-authored-by: lordcheng10 --- .../java/org/apache/pulsar/broker/service/PulsarStats.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java index 9cdf9d1dfc68d..2059aa04350d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java @@ -233,7 +233,7 @@ public synchronized void updateStats( updatedAt = System.currentTimeMillis(); } - public NamespaceBundleStats invalidBundleStats(String bundleName) { + public synchronized NamespaceBundleStats invalidBundleStats(String bundleName) { return bundleStats.remove(bundleName); } @@ -254,7 +254,7 @@ public BrokerOperabilityMetrics getBrokerOperabilityMetrics() { return brokerOperabilityMetrics; } - public Map getBundleStats() { + public synchronized Map getBundleStats() { return bundleStats; } From 67eb0fb4e830f2fb7affc9b3ab77b5423b4ec4ce Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 3 Apr 2023 01:31:35 -0500 Subject: [PATCH 050/280] [cleanup][proxy] ProxyConnection should not call super.exceptionCaught (#19990) ### Motivation While debugging an issue, I noticed that we call `super.exceptionCaught(ctx, cause);` in the `ProxyConnection` class. This leads to the following log line: > An exceptionCaught() event was fired, and it reached at the tail of the pipeline. It usually means the last handler in the pipeline did not handle the exception. io.netty.channel.unix.Errors$NativeIoException: recvAddress(..) failed: Connection reset by peer Because we always handle exceptions, there is no need to forward them to the next handler. ### Modifications * Remove a single method call ### Verifying this change This is a trivial change. Note that we do not call the super method in any other handler implementations in the project. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: skipping PR for this trivial change --- .../java/org/apache/pulsar/proxy/server/ProxyConnection.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 1e19d760c6d7b..b7f5534ee81fb 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -208,7 +208,6 @@ public synchronized void channelInactive(ChannelHandlerContext ctx) throws Excep @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - super.exceptionCaught(ctx, cause); LOG.warn("[{}] Got exception {} : Message: {} State: {}", remoteAddress, cause.getClass().getSimpleName(), cause.getMessage(), state, ClientCnx.isKnownException(cause) ? null : cause); From 5ef3a21a068b837a56a432361aa4b33732a13cfb Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 3 Apr 2023 15:59:17 +0800 Subject: [PATCH 051/280] [fix][build] Client modules should be built with Java 8 (#19991) --- pom.xml | 4 +-- pulsar-client-auth-athenz/pom.xml | 41 ++++++++++++++++++++++++++++++- pulsar-client-auth-sasl/pom.xml | 39 +++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 4c489355281ba..9f6f2909c1af3 100644 --- a/pom.xml +++ b/pom.xml @@ -1966,8 +1966,8 @@ flexible messaging model and an intuitive client API. 8 8 - - + + diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 0b757d747872a..3cdbdb48a2935 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -61,10 +61,49 @@ org.apache.commons commons-lang3 - + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven-enforcer-plugin.version} + + + enforce-bytecode-version + + enforce + + + + + ${pulsar.client.compiler.release} + + test + + + + + + + + + org.codehaus.mojo + extra-enforcer-rules + ${extra-enforcer-rules.version} + + + + org.gaul modernizer-maven-plugin diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 9744a230cf3af..c1143bf69331e 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -71,6 +71,45 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven-enforcer-plugin.version} + + + enforce-bytecode-version + + enforce + + + + + ${pulsar.client.compiler.release} + + test + + + + + + + + + org.codehaus.mojo + extra-enforcer-rules + ${extra-enforcer-rules.version} + + + + org.gaul modernizer-maven-plugin From d1fc7323cbf61a6d2955486fc123fdde5253e72c Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 4 Apr 2023 10:31:33 +0800 Subject: [PATCH 052/280] [fix][broker] Ignore and remove the replicator cursor when the remote cluster is absent (#19972) --- .../service/persistent/PersistentTopic.java | 30 +++++++-- .../persistent/PersistentTopicTest.java | 63 +++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index fa08330ff3c35..18a662c4b7a38 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1696,14 +1696,32 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) { return future; } + private CompletableFuture checkReplicationCluster(String remoteCluster) { + return brokerService.getPulsar().getPulsarResources().getNamespaceResources() + .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) + .thenApply(optPolicies -> optPolicies.map(policies -> policies.replication_clusters) + .orElse(Collections.emptySet()).contains(remoteCluster) + || topicPolicies.getReplicationClusters().get().contains(remoteCluster)); + } + protected CompletableFuture addReplicationCluster(String remoteCluster, ManagedCursor cursor, String localCluster) { return AbstractReplicator.validatePartitionedTopicAsync(PersistentTopic.this.getName(), brokerService) - .thenCompose(__ -> brokerService.pulsar().getPulsarResources().getClusterResources() - .getClusterAsync(remoteCluster) - .thenApply(clusterData -> - brokerService.getReplicationClient(remoteCluster, clusterData))) + .thenCompose(__ -> checkReplicationCluster(remoteCluster)) + .thenCompose(clusterExists -> { + if (!clusterExists) { + log.warn("Remove the replicator because the cluster '{}' does not exist", remoteCluster); + return removeReplicator(remoteCluster).thenApply(__ -> null); + } + return brokerService.pulsar().getPulsarResources().getClusterResources() + .getClusterAsync(remoteCluster) + .thenApply(clusterData -> + brokerService.getReplicationClient(remoteCluster, clusterData)); + }) .thenAccept(replicationClient -> { + if (replicationClient == null) { + return; + } Replicator replicator = replicators.computeIfAbsent(remoteCluster, r -> { try { return new GeoPersistentReplicator(PersistentTopic.this, cursor, localCluster, @@ -1727,8 +1745,8 @@ CompletableFuture removeReplicator(String remoteCluster) { String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster); - replicators.get(remoteCluster).disconnect().thenRun(() -> { - + Optional.ofNullable(replicators.get(remoteCluster)).map(Replicator::disconnect) + .orElse(CompletableFuture.completedFuture(null)).thenRun(() -> { ledger.asyncDeleteCursor(name, new DeleteCursorCallback() { @Override public void deleteCursorComplete(Object ctx) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 80a79e0234de4..c63be7aad01cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -40,15 +40,20 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import lombok.Cleanup; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; @@ -57,6 +62,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -66,7 +72,9 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; import org.junit.Assert; @@ -525,4 +533,59 @@ public void testDeleteTopicFail() throws Exception { makeDeletedFailed.set(false); persistentTopic.delete().get(); } + + @DataProvider(name = "topicLevelPolicy") + public static Object[][] topicLevelPolicy() { + return new Object[][] { { true }, { false } }; + } + + @Test(dataProvider = "topicLevelPolicy") + public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy) throws Exception { + final String namespace = "prop/ns-abc"; + final String topicName = "persistent://" + namespace + + "/testCreateTopicWithZombieReplicatorCursor" + topicLevelPolicy; + final String remoteCluster = "remote"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, conf.getReplicatorPrefix() + "." + remoteCluster, + MessageId.earliest, true); + + admin.clusters().createCluster(remoteCluster, ClusterData.builder() + .serviceUrl("http://localhost:11112") + .brokerServiceUrl("pulsar://localhost:11111") + .build()); + TenantInfo tenantInfo = admin.tenants().getTenantInfo("prop"); + tenantInfo.getAllowedClusters().add(remoteCluster); + admin.tenants().updateTenant("prop", tenantInfo); + + if (topicLevelPolicy) { + admin.topics().setReplicationClusters(topicName, Collections.singletonList(remoteCluster)); + } else { + admin.namespaces().setNamespaceReplicationClustersAsync( + namespace, Collections.singleton(remoteCluster)).get(); + } + + final PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false) + .get(3, TimeUnit.SECONDS).orElse(null); + assertNotNull(topic); + + final Supplier> getCursors = () -> { + final Set cursors = new HashSet<>(); + final Iterable iterable = topic.getManagedLedger().getCursors(); + iterable.forEach(c -> cursors.add(c.getName())); + return cursors; + }; + assertEquals(getCursors.get(), Collections.singleton(conf.getReplicatorPrefix() + "." + remoteCluster)); + + if (topicLevelPolicy) { + admin.topics().setReplicationClusters(topicName, Collections.emptyList()); + } else { + admin.namespaces().setNamespaceReplicationClustersAsync(namespace, Collections.emptySet()).get(); + } + admin.clusters().deleteCluster(remoteCluster); + // Now the cluster and its related policy has been removed but the replicator cursor still exists + + topic.initialize().get(3, TimeUnit.SECONDS); + Awaitility.await().atMost(3, TimeUnit.SECONDS) + .until(() -> !topic.getManagedLedger().getCursors().iterator().hasNext()); + } } From 8c50a6c2e91c81dbf187ce5e66cb39e2758a741e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 4 Apr 2023 15:52:34 +0800 Subject: [PATCH 053/280] [improve][client] PIP-229: Add a common interface to get fields of MessageIdData (#19414) --- .../service/PersistentFailoverE2ETest.java | 8 +- .../broker/service/SubscriptionSeekTest.java | 2 +- .../client/api/CustomMessageIdTest.java | 142 +++++++++++ .../api/PartitionedProducerConsumerTest.java | 2 +- .../pulsar/client/impl/ConsumerAckTest.java | 4 +- .../pulsar/client/impl/MessageIdTest.java | 3 - .../pulsar/client/api/MessageIdAdv.java | 122 ++++++++++ .../pulsar/client/api/TopicMessageId.java | 43 +++- .../impl/AcknowledgmentsGroupingTracker.java | 2 +- .../pulsar/client/impl/BatchMessageAcker.java | 95 -------- .../impl/BatchMessageAckerDisabled.java | 50 ---- .../client/impl/BatchMessageIdImpl.java | 82 +++---- .../client/impl/ChunkMessageIdImpl.java | 9 +- .../pulsar/client/impl/ConsumerBase.java | 15 +- .../pulsar/client/impl/ConsumerImpl.java | 222 +++++++----------- .../pulsar/client/impl/MessageIdAdvUtils.java | 74 ++++++ .../pulsar/client/impl/MessageIdImpl.java | 71 +----- .../pulsar/client/impl/MessageImpl.java | 8 +- .../impl/MessagePayloadContextImpl.java | 9 +- .../client/impl/MultiTopicsConsumerImpl.java | 44 ++-- .../client/impl/NegativeAcksTracker.java | 10 +- ...rsistentAcknowledgmentGroupingTracker.java | 2 +- ...sistentAcknowledgmentsGroupingTracker.java | 158 ++++++------- .../pulsar/client/impl/ResetCursorData.java | 20 +- .../client/impl/TopicMessageIdImpl.java | 39 +-- .../pulsar/client/impl/TopicMessageImpl.java | 2 +- .../client/impl/ZeroQueueConsumerImpl.java | 3 +- .../src/main/resources/findbugsExclude.xml | 5 + .../AcknowledgementsGroupingTrackerTest.java | 13 +- .../impl/BatchMessageAckerDisabledTest.java | 47 ---- .../client/impl/BatchMessageAckerTest.java | 83 ------- .../client/impl/BatchMessageIdImplTest.java | 33 +-- .../impl/MessageIdSerializationTest.java | 3 +- .../functions/utils/FunctionCommon.java | 3 +- .../io/kafka/connect/KafkaConnectSink.java | 36 ++- 35 files changed, 686 insertions(+), 778 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java delete mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java delete mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java delete mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java delete mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java index b263d4448d87a..f5895ec3761bf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java @@ -41,11 +41,11 @@ import org.apache.pulsar.client.api.ConsumerEventListener; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; @@ -370,8 +370,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception { } totalMessages++; consumer1.acknowledge(msg); - MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(msg.getMessageId()); - receivedPtns.add(msgId.getPartitionIndex()); + receivedPtns.add(((MessageIdAdv) msg.getMessageId()).getPartitionIndex()); } assertTrue(Sets.difference(listener1.activePtns, receivedPtns).isEmpty()); @@ -387,8 +386,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception { } totalMessages++; consumer2.acknowledge(msg); - MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(msg.getMessageId()); - receivedPtns.add(msgId.getPartitionIndex()); + receivedPtns.add(((MessageIdAdv) msg.getMessageId()).getPartitionIndex()); } assertTrue(Sets.difference(listener1.inactivePtns, receivedPtns).isEmpty()); assertTrue(Sets.difference(listener2.activePtns, receivedPtns).isEmpty()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java index 93f2a42bcda35..b11946069c9dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java @@ -678,7 +678,7 @@ public void testSeekByFunction() throws Exception { if (message == null) { break; } - received.add(MessageIdImpl.convertToMessageIdImpl(message.getMessageId())); + received.add(message.getMessageId()); } int msgNumFromPartition1 = list.size() / 2; int msgNumFromPartition2 = 1; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java new file mode 100644 index 0000000000000..52bfc9dda37e4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-api") +public class CustomMessageIdTest extends ProducerConsumerBase { + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider + public static Object[][] enableBatching() { + return new Object[][]{ + { true }, + { false } + }; + } + + @Test + public void testSeek() throws Exception { + final var topic = "persistent://my-property/my-ns/test-seek-" + System.currentTimeMillis(); + @Cleanup final var producer = pulsarClient.newProducer(Schema.INT32).topic(topic).create(); + final var msgIds = new ArrayList(); + for (int i = 0; i < 10; i++) { + msgIds.add(new SimpleMessageIdImpl((MessageIdAdv) producer.send(i))); + } + @Cleanup final var consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic).subscriptionName("sub").subscribe(); + consumer.seek(msgIds.get(6)); + final var msg = consumer.receive(3, TimeUnit.SECONDS); + assertNotNull(msg); + assertEquals(msg.getValue(), 7); + } + + @Test(dataProvider = "enableBatching") + public void testAcknowledgment(boolean enableBatching) throws Exception { + final var topic = "persistent://my-property/my-ns/test-ack-" + + enableBatching + System.currentTimeMillis(); + final var producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(enableBatching) + .batchingMaxMessages(10) + .batchingMaxPublishDelay(300, TimeUnit.MILLISECONDS) + .create(); + final var consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName("sub") + .enableBatchIndexAcknowledgment(true) + .isAckReceiptEnabled(true) + .subscribe(); + for (int i = 0; i < 10; i++) { + producer.sendAsync(i); + } + final var msgIds = new ArrayList(); + for (int i = 0; i < 10; i++) { + final var msg = consumer.receive(); + final var msgId = new SimpleMessageIdImpl((MessageIdAdv) msg.getMessageId()); + msgIds.add(msgId); + if (enableBatching) { + assertTrue(msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0); + } else { + assertFalse(msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0); + } + } + consumer.acknowledgeCumulative(msgIds.get(8)); + consumer.redeliverUnacknowledgedMessages(); + final var msg = consumer.receive(3, TimeUnit.SECONDS); + assertNotNull(msg); + assertEquals(msg.getValue(), 9); + } + + private record SimpleMessageIdImpl(long ledgerId, long entryId, int batchIndex, int batchSize) + implements MessageIdAdv { + + public SimpleMessageIdImpl(MessageIdAdv msgId) { + this(msgId.getLedgerId(), msgId.getEntryId(), msgId.getBatchIndex(), msgId.getBatchSize()); + } + + @Override + public byte[] toByteArray() { + return new byte[0]; // never used + } + + @Override + public long getLedgerId() { + return ledgerId; + } + + @Override + public long getEntryId() { + return entryId; + } + + @Override + public int getBatchIndex() { + return batchIndex; + } + + @Override + public int getBatchSize() { + return batchSize; + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java index cd384e587898d..13ae991a0e89c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java @@ -767,7 +767,7 @@ public void testMessageIdForSubscribeToSinglePartition() throws Exception { for (int i = 0; i < totalMessages; i ++) { msg = consumer1.receive(5, TimeUnit.SECONDS); - Assert.assertEquals(MessageIdImpl.convertToMessageIdImpl(msg.getMessageId()).getPartitionIndex(), 2); + Assert.assertEquals(((MessageIdAdv) msg.getMessageId()).getPartitionIndex(), 2); consumer1.acknowledge(msg); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java index 42da60906483c..a83283bc267b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java @@ -176,10 +176,10 @@ private AckTestData prepareDataForAck(String topic) throws PulsarClientException messageIds.add(message.getMessageId()); } MessageId firstEntryMessageId = messageIds.get(0); - MessageId secondEntryMessageId = ((BatchMessageIdImpl) messageIds.get(1)).toMessageIdImpl(); + MessageId secondEntryMessageId = MessageIdAdvUtils.discardBatch(messageIds.get(1)); // Verify messages 2 to N must be in the same entry for (int i = 2; i < messageIds.size(); i++) { - assertEquals(((BatchMessageIdImpl) messageIds.get(i)).toMessageIdImpl(), secondEntryMessageId); + assertEquals(MessageIdAdvUtils.discardBatch(messageIds.get(i)), secondEntryMessageId); } assertTrue(interceptor.individualAckedMessageIdList.isEmpty()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java index ceb5c51e6aa77..375bbff8a4df4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java @@ -118,9 +118,6 @@ public void producerSendAsync(TopicType topicType) throws PulsarClientException, Message message = consumer.receive(); assertEquals(new String(message.getData()), messagePrefix + i); MessageId messageId = message.getMessageId(); - if (topicType == TopicType.PARTITIONED) { - messageId = MessageIdImpl.convertToMessageIdImpl(messageId); - } assertTrue(messageIds.remove(messageId), "Failed to receive message"); } log.info("Remaining message IDs = {}", messageIds); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java new file mode 100644 index 0000000000000..73ecfed0ad059 --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import java.util.BitSet; + +/** + * The {@link MessageId} interface provided for advanced users. + *

+ * All built-in MessageId implementations should be able to be cast to MessageIdAdv. + *

+ */ +public interface MessageIdAdv extends MessageId { + + /** + * Get the ledger ID. + * + * @return the ledger ID + */ + long getLedgerId(); + + /** + * Get the entry ID. + * + * @return the entry ID + */ + long getEntryId(); + + /** + * Get the partition index. + * + * @return -1 if the message is from a non-partitioned topic, otherwise the non-negative partition index + */ + default int getPartitionIndex() { + return -1; + } + + /** + * Get the batch index. + * + * @return -1 if the message is not in a batch + */ + default int getBatchIndex() { + return -1; + } + + /** + * Get the batch size. + * + * @return 0 if the message is not in a batch + */ + default int getBatchSize() { + return 0; + } + + /** + * Get the BitSet that indicates which messages in the batch. + * + * @implNote The message IDs of a batch should share a BitSet. For example, given 3 messages in the same batch whose + * size is 3, all message IDs of them should return "111" (i.e. a BitSet whose size is 3 and all bits are 1). If the + * 1st message has been acknowledged, the returned BitSet should become "011" (i.e. the 1st bit become 0). + * + * @return null if the message is a non-batched message + */ + default BitSet getAckSet() { + return null; + } + + /** + * Get the message ID of the first chunk if the current message ID represents the position of a chunked message. + * + * @implNote A chunked message is distributed across different BookKeeper entries. The message ID of a chunked + * message is composed of two message IDs that represent positions of the first and the last chunk. The message ID + * itself represents the position of the last chunk. + * + * @return null if the message is not a chunked message + */ + default MessageIdAdv getFirstChunkMessageId() { + return null; + } + + /** + * The default implementation of {@link Comparable#compareTo(Object)}. + */ + default int compareTo(MessageId o) { + if (!(o instanceof MessageIdAdv)) { + throw new UnsupportedOperationException("Unknown MessageId type: " + + ((o != null) ? o.getClass().getName() : "null")); + } + final MessageIdAdv other = (MessageIdAdv) o; + int result = Long.compare(this.getLedgerId(), other.getLedgerId()); + if (result != 0) { + return result; + } + result = Long.compare(this.getEntryId(), other.getEntryId()); + if (result != 0) { + return result; + } + // TODO: Correct the following compare logics, see https://github.com/apache/pulsar/pull/18981 + result = Integer.compare(this.getPartitionIndex(), other.getPartitionIndex()); + if (result != 0) { + return result; + } + return Integer.compare(this.getBatchIndex(), other.getBatchIndex()); + } +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java index f6109d5f8e87e..b70267bb0fb8b 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.client.api; +import java.util.BitSet; + /** * The MessageId used for a consumer that subscribes multiple topics or partitioned topics. * @@ -49,13 +51,13 @@ static TopicMessageId create(String topic, MessageId messageId) { /** * The simplest implementation of a TopicMessageId interface. */ - class Impl implements TopicMessageId { + class Impl implements MessageIdAdv, TopicMessageId { private final String topic; - private final MessageId messageId; + private final MessageIdAdv messageId; public Impl(String topic, MessageId messageId) { this.topic = topic; - this.messageId = messageId; + this.messageId = (MessageIdAdv) messageId; } @Override @@ -68,6 +70,41 @@ public String getOwnerTopic() { return topic; } + @Override + public long getLedgerId() { + return messageId.getLedgerId(); + } + + @Override + public long getEntryId() { + return messageId.getEntryId(); + } + + @Override + public int getPartitionIndex() { + return messageId.getPartitionIndex(); + } + + @Override + public int getBatchIndex() { + return messageId.getBatchIndex(); + } + + @Override + public int getBatchSize() { + return messageId.getBatchSize(); + } + + @Override + public BitSet getAckSet() { + return messageId.getAckSet(); + } + + @Override + public MessageIdAdv getFirstChunkMessageId() { + return messageId.getFirstChunkMessageId(); + } + @Override public int compareTo(MessageId o) { return messageId.compareTo(o); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java index d46af1a99e7f0..60d7135e5e4ae 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java @@ -31,7 +31,7 @@ public interface AcknowledgmentsGroupingTracker extends AutoCloseable { boolean isDuplicate(MessageId messageId); - CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType, Map properties); + CompletableFuture addAcknowledgment(MessageId msgId, AckType ackType, Map properties); CompletableFuture addListAcknowledgment(List messageIds, AckType ackType, Map properties); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java deleted file mode 100644 index 1c9b66fd2bad5..0000000000000 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.impl; - -import java.util.BitSet; - -public class BatchMessageAcker { - - private BatchMessageAcker() { - this.bitSet = new BitSet(); - this.batchSize = 0; - } - - static BatchMessageAcker newAcker(int batchSize) { - BitSet bitSet = new BitSet(batchSize); - bitSet.set(0, batchSize); - return new BatchMessageAcker(bitSet, batchSize); - } - - // Use the param bitSet as the BatchMessageAcker's bitSet, don't care about the batchSize. - static BatchMessageAcker newAcker(BitSet bitSet) { - return new BatchMessageAcker(bitSet, -1); - } - - // bitset shared across messages in the same batch. - private final int batchSize; - private final BitSet bitSet; - private volatile boolean prevBatchCumulativelyAcked = false; - - BatchMessageAcker(BitSet bitSet, int batchSize) { - this.bitSet = bitSet; - this.batchSize = batchSize; - } - - BitSet getBitSet() { - return bitSet; - } - - public synchronized int getBatchSize() { - return batchSize; - } - - public synchronized boolean ackIndividual(int batchIndex) { - bitSet.clear(batchIndex); - return bitSet.isEmpty(); - } - - public synchronized int getBitSetSize() { - return bitSet.size(); - } - - public synchronized boolean ackCumulative(int batchIndex) { - // +1 since to argument is exclusive - bitSet.clear(0, batchIndex + 1); - return bitSet.isEmpty(); - } - - // debug purpose - public synchronized int getOutstandingAcks() { - return bitSet.cardinality(); - } - - public void setPrevBatchCumulativelyAcked(boolean acked) { - this.prevBatchCumulativelyAcked = acked; - } - - public boolean isPrevBatchCumulativelyAcked() { - return prevBatchCumulativelyAcked; - } - - @Override - public String toString() { - return "BatchMessageAcker{" - + "batchSize=" + batchSize - + ", bitSet=" + bitSet - + ", prevBatchCumulativelyAcked=" + prevBatchCumulativelyAcked - + '}'; - } -} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java deleted file mode 100644 index b70c928b29650..0000000000000 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.impl; - -import java.util.BitSet; - -class BatchMessageAckerDisabled extends BatchMessageAcker { - - static final BatchMessageAckerDisabled INSTANCE = new BatchMessageAckerDisabled(); - - private BatchMessageAckerDisabled() { - super(new BitSet(), 0); - } - - @Override - public synchronized int getBatchSize() { - return 0; - } - - @Override - public boolean ackIndividual(int batchIndex) { - return true; - } - - @Override - public boolean ackCumulative(int batchIndex) { - return true; - } - - @Override - public int getOutstandingAcks() { - return 0; - } -} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java index ed28082ff6a30..e9cddeb65d7f2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java @@ -18,20 +18,16 @@ */ package org.apache.pulsar.client.impl; -import javax.annotation.Nonnull; -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.TopicMessageId; +import java.util.BitSet; +import org.apache.pulsar.client.api.MessageIdAdv; -/** - */ public class BatchMessageIdImpl extends MessageIdImpl { private static final long serialVersionUID = 1L; - static final int NO_BATCH = -1; private final int batchIndex; private final int batchSize; - private final transient BatchMessageAcker acker; + private final BitSet ackSet; // Private constructor used only for json deserialization @SuppressWarnings("unused") @@ -40,59 +36,35 @@ private BatchMessageIdImpl() { } public BatchMessageIdImpl(long ledgerId, long entryId, int partitionIndex, int batchIndex) { - this(ledgerId, entryId, partitionIndex, batchIndex, 0, BatchMessageAckerDisabled.INSTANCE); + this(ledgerId, entryId, partitionIndex, batchIndex, 0, null); } public BatchMessageIdImpl(long ledgerId, long entryId, int partitionIndex, int batchIndex, int batchSize, - BatchMessageAcker acker) { + BitSet ackSet) { super(ledgerId, entryId, partitionIndex); this.batchIndex = batchIndex; this.batchSize = batchSize; - this.acker = acker; + this.ackSet = ackSet; } - public BatchMessageIdImpl(MessageIdImpl other) { - super(other.ledgerId, other.entryId, other.partitionIndex); - if (other instanceof BatchMessageIdImpl) { - BatchMessageIdImpl otherId = (BatchMessageIdImpl) other; - this.batchIndex = otherId.batchIndex; - this.batchSize = otherId.batchSize; - this.acker = otherId.acker; - } else { - this.batchIndex = NO_BATCH; - this.batchSize = 0; - this.acker = BatchMessageAckerDisabled.INSTANCE; - } + public BatchMessageIdImpl(MessageIdAdv other) { + this(other.getLedgerId(), other.getEntryId(), other.getPartitionIndex(), + other.getBatchIndex(), other.getBatchSize(), other.getAckSet()); } + @Override public int getBatchIndex() { return batchIndex; } - @Override - public int compareTo(@Nonnull MessageId o) { - if (o instanceof MessageIdImpl) { - MessageIdImpl other = (MessageIdImpl) o; - int batchIndex = (o instanceof BatchMessageIdImpl) ? ((BatchMessageIdImpl) o).batchIndex : NO_BATCH; - return messageIdCompare( - this.ledgerId, this.entryId, this.partitionIndex, this.batchIndex, - other.ledgerId, other.entryId, other.partitionIndex, batchIndex - ); - } else if (o instanceof TopicMessageId) { - return compareTo(MessageIdImpl.convertToMessageIdImpl(o)); - } else { - throw new UnsupportedOperationException("Unknown MessageId type: " + o.getClass().getName()); - } - } - @Override public int hashCode() { - return messageIdHashCode(ledgerId, entryId, partitionIndex, batchIndex); + return MessageIdAdvUtils.hashCode(this); } @Override public boolean equals(Object o) { - return super.equals(o); + return MessageIdAdvUtils.equals(this, o); } @Override @@ -106,39 +78,51 @@ public byte[] toByteArray() { return toByteArray(batchIndex, batchSize); } + @Deprecated public boolean ackIndividual() { - return acker.ackIndividual(batchIndex); + return MessageIdAdvUtils.acknowledge(this, true); } + @Deprecated public boolean ackCumulative() { - return acker.ackCumulative(batchIndex); + return MessageIdAdvUtils.acknowledge(this, false); } + @Deprecated public int getOutstandingAcksInSameBatch() { - return acker.getOutstandingAcks(); + return 0; } + @Override public int getBatchSize() { - return acker.getBatchSize(); + return batchSize; } + @Deprecated public int getOriginalBatchSize() { return this.batchSize; } + @Deprecated public MessageIdImpl prevBatchMessageId() { - return new MessageIdImpl( - ledgerId, entryId - 1, partitionIndex); + return (MessageIdImpl) MessageIdAdvUtils.prevMessageId(this); } // MessageIdImpl is widely used as the key of a hash map, in this case, we should convert the batch message id to // have the correct hash code. + @Deprecated public MessageIdImpl toMessageIdImpl() { - return new MessageIdImpl(ledgerId, entryId, partitionIndex); + return (MessageIdImpl) MessageIdAdvUtils.discardBatch(this); } - public BatchMessageAcker getAcker() { - return acker; + @Override + public BitSet getAckSet() { + return ackSet; } + static BitSet newAckSet(int batchSize) { + final BitSet ackSet = new BitSet(batchSize); + ackSet.set(0, batchSize); + return ackSet; + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java index 28d5047c8ef25..29ce160442a3b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java @@ -21,10 +21,10 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.util.Objects; -import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.common.api.proto.MessageIdData; -public class ChunkMessageIdImpl extends MessageIdImpl implements MessageId { +public class ChunkMessageIdImpl extends MessageIdImpl { private final MessageIdImpl firstChunkMsgId; public ChunkMessageIdImpl(MessageIdImpl firstChunkMsgId, MessageIdImpl lastChunkMsgId) { @@ -32,11 +32,12 @@ public ChunkMessageIdImpl(MessageIdImpl firstChunkMsgId, MessageIdImpl lastChunk this.firstChunkMsgId = firstChunkMsgId; } - public MessageIdImpl getFirstChunkMessageId() { + @Override + public MessageIdAdv getFirstChunkMessageId() { return firstChunkMsgId; } - public MessageIdImpl getLastChunkMessageId() { + public MessageIdAdv getLastChunkMessageId() { return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 75d3b2edf6e28..973b3302f4199 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -47,6 +47,7 @@ import org.apache.pulsar.client.api.ConsumerEventListener; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.Messages; import org.apache.pulsar.client.api.PulsarClientException; @@ -82,7 +83,7 @@ public abstract class ConsumerBase extends HandlerState implements Consumer> incomingMessages; - protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap; + protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap; protected final ConcurrentLinkedQueue>> pendingReceives; protected final int maxReceiverQueueSize; private volatile int currentReceiverQueueSize; @@ -128,7 +129,7 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat // Always use growable queue since items can exceed the advertised size this.incomingMessages = new GrowableArrayBlockingQueue<>(); this.unAckedChunkedMessageIdSequenceMap = - ConcurrentOpenHashMap.newBuilder().build(); + ConcurrentOpenHashMap.newBuilder().build(); this.executorProvider = executorProvider; this.externalPinnedExecutor = executorProvider.getExecutor(); this.internalPinnedExecutor = client.getInternalExecutorService(); @@ -223,14 +224,6 @@ protected void trackUnAckedMsgIfNoListener(MessageId messageId, int redeliveryCo } } - protected MessageId normalizeMessageId(MessageId messageId) { - if (messageId instanceof BatchMessageIdImpl) { - // do not add each item in batch message into tracker - return ((BatchMessageIdImpl) messageId).toMessageIdImpl(); - } - return messageId; - } - protected void reduceCurrentReceiverQueueSize() { if (!conf.isAutoScaledReceiverQueueSizeEnabled()) { return; @@ -1131,7 +1124,7 @@ protected void callMessageListener(Message msg) { ? ((TopicMessageImpl) msg).getMessage() : msg)); MessageId id; if (this instanceof ConsumerImpl) { - id = normalizeMessageId(msg.getMessageId()); + id = MessageIdAdvUtils.discardBatch(msg.getMessageId()); } else { id = msg.getMessageId(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index beaa34bf20520..1feef6ca0a642 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -71,6 +72,7 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.Messages; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; @@ -154,12 +156,12 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle @Getter(AccessLevel.PACKAGE) private final int priorityLevel; private final SubscriptionMode subscriptionMode; - private volatile BatchMessageIdImpl startMessageId; + private volatile MessageIdAdv startMessageId; - private volatile BatchMessageIdImpl seekMessageId; + private volatile MessageIdAdv seekMessageId; private final AtomicBoolean duringSeek; - private final BatchMessageIdImpl initialStartMessageId; + private final MessageIdAdv initialStartMessageId; private final long startMessageRollbackDurationInSec; @@ -178,7 +180,7 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final TopicName topicName; private final String topicNameWithoutPartition; - private final Map>> possibleSendToDeadLetterTopicMessages; + private final Map>> possibleSendToDeadLetterTopicMessages; private final DeadLetterPolicy deadLetterPolicy; @@ -258,12 +260,8 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat this.consumerId = client.newConsumerId(); this.subscriptionMode = conf.getSubscriptionMode(); if (startMessageId != null) { - if (startMessageId instanceof ChunkMessageIdImpl) { - this.startMessageId = new BatchMessageIdImpl( - ((ChunkMessageIdImpl) startMessageId).getFirstChunkMessageId()); - } else { - this.startMessageId = new BatchMessageIdImpl((MessageIdImpl) startMessageId); - } + MessageIdAdv firstChunkMessageId = ((MessageIdAdv) startMessageId).getFirstChunkMessageId(); + this.startMessageId = (firstChunkMessageId == null) ? (MessageIdAdv) startMessageId : firstChunkMessageId; } this.initialStartMessageId = this.startMessageId; this.startMessageRollbackDurationInSec = startMessageRollbackDurationInSec; @@ -535,7 +533,6 @@ protected CompletableFuture> internalBatchReceiveAsync() { protected CompletableFuture doAcknowledge(MessageId messageId, AckType ackType, Map properties, TransactionImpl txn) { - messageId = MessageIdImpl.convertToMessageIdImpl(messageId); if (getState() != State.Ready && getState() != State.Connecting) { stats.incrementNumAcksFailed(); PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState()); @@ -551,16 +548,12 @@ protected CompletableFuture doAcknowledge(MessageId messageId, AckType ack return doTransactionAcknowledgeForResponse(messageId, ackType, null, properties, new TxnID(txn.getTxnIdMostBits(), txn.getTxnIdLeastBits())); } - return acknowledgmentsGroupingTracker.addAcknowledgment((MessageIdImpl) messageId, ackType, properties); + return acknowledgmentsGroupingTracker.addAcknowledgment(messageId, ackType, properties); } @Override protected CompletableFuture doAcknowledge(List messageIdList, AckType ackType, Map properties, TransactionImpl txn) { - - for (MessageId messageId : messageIdList) { - checkArgument(messageId instanceof MessageIdImpl); - } if (getState() != State.Ready && getState() != State.Connecting) { stats.incrementNumAcksFailed(); PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState()); @@ -572,7 +565,7 @@ protected CompletableFuture doAcknowledge(List messageIdList, A return FutureUtil.failedFuture(exception); } if (txn != null) { - return doTransactionAcknowledgeForResponse(messageIdList, ackType, null, + return doTransactionAcknowledgeForResponse(messageIdList, ackType, properties, new TxnID(txn.getTxnIdMostBits(), txn.getTxnIdLeastBits())); } else { return this.acknowledgmentsGroupingTracker.addListAcknowledgment(messageIdList, ackType, properties); @@ -592,7 +585,6 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a .InvalidMessageException("Cannot handle message with null messageId")); } - messageId = MessageIdImpl.convertToMessageIdImpl(messageId); if (getState() != State.Ready && getState() != State.Connecting) { stats.incrementNumAcksFailed(); PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState()); @@ -922,7 +914,7 @@ protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize * Clear the internal receiver queue and returns the message id of what was the 1st message in the queue that was * not seen by the application. */ - private BatchMessageIdImpl clearReceiverQueue() { + private MessageIdAdv clearReceiverQueue() { List> currentMessageQueue = new ArrayList<>(incomingMessages.size()); incomingMessages.drainTo(currentMessageQueue); resetIncomingMessageSize(); @@ -934,17 +926,16 @@ private BatchMessageIdImpl clearReceiverQueue() { } if (!currentMessageQueue.isEmpty()) { - MessageIdImpl nextMessageInQueue = (MessageIdImpl) currentMessageQueue.get(0).getMessageId(); - BatchMessageIdImpl previousMessage; - if (nextMessageInQueue instanceof BatchMessageIdImpl) { + MessageIdAdv nextMessageInQueue = (MessageIdAdv) currentMessageQueue.get(0).getMessageId(); + MessageIdAdv previousMessage; + if (MessageIdAdvUtils.isBatch(nextMessageInQueue)) { // Get on the previous message within the current batch previousMessage = new BatchMessageIdImpl(nextMessageInQueue.getLedgerId(), nextMessageInQueue.getEntryId(), nextMessageInQueue.getPartitionIndex(), - ((BatchMessageIdImpl) nextMessageInQueue).getBatchIndex() - 1); + nextMessageInQueue.getBatchIndex() - 1); } else { // Get on previous message in previous entry - previousMessage = new BatchMessageIdImpl(nextMessageInQueue.getLedgerId(), - nextMessageInQueue.getEntryId() - 1, nextMessageInQueue.getPartitionIndex(), -1); + previousMessage = MessageIdAdvUtils.prevMessageId(nextMessageInQueue); } // release messages if they are pooled messages currentMessageQueue.forEach(Message::release); @@ -1126,7 +1117,7 @@ protected MessageImpl newSingleMessage(final int index, final Schema schema, final boolean containMetadata, final BitSetRecyclable ackBitSet, - final BatchMessageAcker acker, + final BitSet ackSetInMessageId, final int redeliveryCount, final long consumerEpoch) { if (log.isDebugEnabled()) { @@ -1161,7 +1152,7 @@ protected MessageImpl newSingleMessage(final int index, } BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(), - messageId.getEntryId(), getPartitionIndex(), index, numMessages, acker); + messageId.getEntryId(), getPartitionIndex(), index, numMessages, ackSetInMessageId); final ByteBuf payloadBuffer = (singleMessagePayload != null) ? singleMessagePayload : payload; final MessageImpl message = MessageImpl.create(topicName.toString(), batchMessageIdImpl, @@ -1525,7 +1516,7 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, possibleToDeadLetter = new ArrayList<>(); } - BatchMessageAcker acker = BatchMessageAcker.newAcker(batchSize); + BitSet ackSetInMessageId = BatchMessageIdImpl.newAckSet(batchSize); BitSetRecyclable ackBitSet = null; if (ackSet != null && ackSet.size() > 0) { ackBitSet = BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet)); @@ -1537,7 +1528,7 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, for (int i = 0; i < batchSize; ++i) { final MessageImpl message = newSingleMessage(i, batchSize, brokerEntryMetadata, msgMetadata, singleMessageMetadata, uncompressedPayload, batchMessage, schema, true, - ackBitSet, acker, redeliveryCount, consumerEpoch); + ackBitSet, ackSetInMessageId, redeliveryCount, consumerEpoch); if (message == null) { skippedMessages++; continue; @@ -1634,7 +1625,7 @@ protected void trackMessage(MessageId messageId) { protected void trackMessage(MessageId messageId, int redeliveryCount) { if (conf.getAckTimeoutMillis() > 0 && messageId instanceof MessageIdImpl) { - MessageId id = normalizeMessageId(messageId); + MessageId id = MessageIdAdvUtils.discardBatch(messageId); if (hasParentConsumer) { //TODO: check parent consumer here // we should no longer track this message, TopicsConsumer will take care from now onwards @@ -1931,8 +1922,6 @@ public void redeliverUnacknowledgedMessages(Set messageIds) { return; } - checkArgument(messageIds.stream().findFirst().get() instanceof MessageIdImpl); - if (conf.getSubscriptionType() != SubscriptionType.Shared && conf.getSubscriptionType() != SubscriptionType.Key_Shared) { // We cannot redeliver single messages if subscription type is not Shared @@ -1942,11 +1931,7 @@ public void redeliverUnacknowledgedMessages(Set messageIds) { ClientCnx cnx = cnx(); if (isConnected() && cnx.getRemoteEndpointProtocolVersion() >= ProtocolVersion.v2.getValue()) { int messagesFromQueue = removeExpiredMessagesFromQueue(messageIds); - Iterable> batches = Iterables.partition( - messageIds.stream() - .map(messageId -> (MessageIdImpl) messageId) - .collect(Collectors.toSet()), MAX_REDELIVER_UNACKNOWLEDGED); - batches.forEach(ids -> { + Iterables.partition(messageIds, MAX_REDELIVER_UNACKNOWLEDGED).forEach(ids -> { getRedeliveryMessageIdData(ids).thenAccept(messageIdData -> { if (!messageIdData.isEmpty()) { ByteBuf cmd = Commands.newRedeliverUnacknowledgedMessages(consumerId, messageIdData); @@ -1985,11 +1970,12 @@ protected void completeOpBatchReceive(OpBatchReceive op) { notifyPendingBatchReceivedCallBack(op.future); } - private CompletableFuture> getRedeliveryMessageIdData(List messageIds) { + private CompletableFuture> getRedeliveryMessageIdData(List messageIds) { if (messageIds == null || messageIds.isEmpty()) { return CompletableFuture.completedFuture(Collections.emptyList()); } - List> futures = messageIds.stream().map(messageId -> { + List> futures = messageIds.stream().map(originalMessageId -> { + final MessageIdAdv messageId = (MessageIdAdv) originalMessageId; CompletableFuture future = processPossibleToDLQ(messageId); return future.thenApply(sendToDLQ -> { if (!sendToDLQ) { @@ -2005,20 +1991,15 @@ private CompletableFuture> getRedeliveryMessageIdData(List processPossibleToDLQ(MessageIdImpl messageId) { + private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) { List> deadLetterMessages = null; if (possibleSendToDeadLetterTopicMessages != null) { - if (messageId instanceof BatchMessageIdImpl) { - messageId = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), - getPartitionIndex()); - } - deadLetterMessages = possibleSendToDeadLetterTopicMessages.get(messageId); + deadLetterMessages = possibleSendToDeadLetterTopicMessages.get(MessageIdAdvUtils.discardBatch(messageId)); } CompletableFuture result = new CompletableFuture<>(); if (deadLetterMessages != null) { initDeadLetterProducerIfNeeded(); List> finalDeadLetterMessages = deadLetterMessages; - MessageIdImpl finalMessageId = messageId; deadLetterProducer.thenAcceptAsync(producerDLQ -> { for (MessageImpl message : finalDeadLetterMessages) { String originMessageIdStr = message.getMessageId().toString(); @@ -2032,12 +2013,12 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId) } typedMessageBuilderNew.sendAsync() .thenAccept(messageIdInDLQ -> { - possibleSendToDeadLetterTopicMessages.remove(finalMessageId); - acknowledgeAsync(finalMessageId).whenComplete((v, ex) -> { + possibleSendToDeadLetterTopicMessages.remove(messageId); + acknowledgeAsync(messageId).whenComplete((v, ex) -> { if (ex != null) { log.warn("[{}] [{}] [{}] Failed to acknowledge the message {} of the original" + " topic but send to the DLQ successfully.", - topicName, subscription, consumerName, finalMessageId, ex); + topicName, subscription, consumerName, messageId, ex); result.complete(false); } else { result.complete(true); @@ -2047,11 +2028,11 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId) if (ex instanceof PulsarClientException.ProducerQueueIsFullError) { log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}: {}", topicName, subscription, consumerName, - deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex.getMessage()); + deadLetterPolicy.getDeadLetterTopic(), messageId, ex.getMessage()); } else { log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}", topicName, subscription, consumerName, - deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex); + deadLetterPolicy.getDeadLetterTopic(), messageId, ex); } result.complete(false); return null; @@ -2154,8 +2135,8 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, final CompletableFuture seekFuture = new CompletableFuture<>(); ClientCnx cnx = cnx(); - BatchMessageIdImpl originSeekMessageId = seekMessageId; - seekMessageId = new BatchMessageIdImpl((MessageIdImpl) seekId); + MessageIdAdv originSeekMessageId = seekMessageId; + seekMessageId = (MessageIdAdv) seekId; duringSeek.set(true); log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); @@ -2193,29 +2174,28 @@ public CompletableFuture seekAsync(long timestamp) { } @Override - public CompletableFuture seekAsync(MessageId originalMessageId) { - final MessageIdImpl messageId = MessageIdImpl.convertToMessageIdImpl(originalMessageId); + public CompletableFuture seekAsync(MessageId messageId) { String seekBy = String.format("the message %s", messageId.toString()); return seekAsyncCheckState(seekBy).orElseGet(() -> { long requestId = client.newRequestId(); - ByteBuf seek = null; - if (messageId instanceof BatchMessageIdImpl) { - BatchMessageIdImpl msgId = (BatchMessageIdImpl) messageId; - // Initialize ack set - BitSetRecyclable ackSet = BitSetRecyclable.create(); - ackSet.set(0, msgId.getBatchSize()); - ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0)); - long[] ackSetArr = ackSet.toLongArray(); - ackSet.recycle(); - - seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr); - } else if (messageId instanceof ChunkMessageIdImpl) { - ChunkMessageIdImpl msgId = (ChunkMessageIdImpl) messageId; - seek = Commands.newSeek(consumerId, requestId, msgId.getFirstChunkMessageId().getLedgerId(), - msgId.getFirstChunkMessageId().getEntryId(), new long[0]); + final MessageIdAdv msgId = (MessageIdAdv) messageId; + final MessageIdAdv firstChunkMsgId = msgId.getFirstChunkMessageId(); + final ByteBuf seek; + if (msgId.getFirstChunkMessageId() != null) { + seek = Commands.newSeek(consumerId, requestId, firstChunkMsgId.getLedgerId(), + firstChunkMsgId.getEntryId(), new long[0]); } else { - seek = Commands.newSeek( - consumerId, requestId, messageId.getLedgerId(), messageId.getEntryId(), new long[0]); + final long[] ackSetArr; + if (MessageIdAdvUtils.isBatch(msgId)) { + final BitSetRecyclable ackSet = BitSetRecyclable.create(); + ackSet.set(0, msgId.getBatchSize()); + ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0)); + ackSetArr = ackSet.toLongArray(); + ackSet.recycle(); + } else { + ackSetArr = new long[0]; + } + seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr); } return seekAsyncInternal(requestId, seek, messageId, seekBy); }); @@ -2247,9 +2227,8 @@ public CompletableFuture hasMessageAvailableAsync() { } future.thenAccept(response -> { - MessageIdImpl lastMessageId = MessageIdImpl.convertToMessageIdImpl(response.lastMessageId); - MessageIdImpl markDeletePosition = MessageIdImpl - .convertToMessageIdImpl(response.markDeletePosition); + MessageIdAdv lastMessageId = (MessageIdAdv) response.lastMessageId; + MessageIdAdv markDeletePosition = (MessageIdAdv) response.markDeletePosition; if (markDeletePosition != null && !(markDeletePosition.getEntryId() < 0 && markDeletePosition.getLedgerId() > lastMessageId.getLedgerId())) { @@ -2434,16 +2413,6 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, } } - private MessageIdImpl getMessageIdImpl(Message msg) { - MessageIdImpl messageId = (MessageIdImpl) msg.getMessageId(); - if (messageId instanceof BatchMessageIdImpl) { - // messageIds contain MessageIdImpl, not BatchMessageIdImpl - messageId = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), getPartitionIndex()); - } - return messageId; - } - - private boolean isMessageUndecryptable(MessageMetadata msgMetadata) { return (msgMetadata.getEncryptionKeysCount() > 0 && conf.getCryptoKeyReader() == null && conf.getCryptoFailureAction() == ConsumerCryptoFailureAction.CONSUME); @@ -2486,7 +2455,7 @@ private int removeExpiredMessagesFromQueue(Set messageIds) { int messagesFromQueue = 0; Message peek = incomingMessages.peek(); if (peek != null) { - MessageIdImpl messageId = getMessageIdImpl(peek); + MessageIdAdv messageId = MessageIdAdvUtils.discardBatch(peek.getMessageId()); if (!messageIds.contains(messageId)) { // first message is not expired, then no message is expired in queue. return 0; @@ -2497,7 +2466,7 @@ private int removeExpiredMessagesFromQueue(Set messageIds) { while (message != null) { decreaseIncomingMessageSize(message); messagesFromQueue++; - MessageIdImpl id = getMessageIdImpl(message); + MessageIdAdv id = MessageIdAdvUtils.discardBatch(message.getMessageId()); if (!messageIds.contains(id)) { messageIds.add(id); break; @@ -2691,33 +2660,26 @@ private void removeChunkMessage(String msgUUID, ChunkedMessageCtx chunkedMsgCtx, private CompletableFuture doTransactionAcknowledgeForResponse(MessageId messageId, AckType ackType, ValidationError validationError, Map properties, TxnID txnID) { - BitSetRecyclable bitSetRecyclable = null; - long ledgerId; - long entryId; - ByteBuf cmd; long requestId = client.newRequestId(); - if (messageId instanceof BatchMessageIdImpl) { - BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; - bitSetRecyclable = BitSetRecyclable.create(); - ledgerId = batchMessageId.getLedgerId(); - entryId = batchMessageId.getEntryId(); + final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId; + final long ledgerId = messageIdAdv.getLedgerId(); + final long entryId = messageIdAdv.getEntryId(); + final ByteBuf cmd; + if (MessageIdAdvUtils.isBatch(messageIdAdv)) { + BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create(); + bitSetRecyclable.set(0, messageIdAdv.getBatchSize()); if (ackType == AckType.Cumulative) { - batchMessageId.ackCumulative(); - bitSetRecyclable.set(0, batchMessageId.getBatchSize()); - bitSetRecyclable.clear(0, batchMessageId.getBatchIndex() + 1); + MessageIdAdvUtils.acknowledge(messageIdAdv, false); + bitSetRecyclable.clear(0, messageIdAdv.getBatchIndex() + 1); } else { - bitSetRecyclable.set(0, batchMessageId.getBatchSize()); - bitSetRecyclable.clear(batchMessageId.getBatchIndex()); + bitSetRecyclable.clear(messageIdAdv.getBatchIndex()); } cmd = Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, ackType, validationError, properties, - txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, batchMessageId.getBatchSize()); + txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, messageIdAdv.getBatchSize()); bitSetRecyclable.recycle(); } else { - MessageIdImpl singleMessage = (MessageIdImpl) messageId; - ledgerId = singleMessage.getLedgerId(); - entryId = singleMessage.getEntryId(); - cmd = Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, ackType, - validationError, properties, txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId); + cmd = Commands.newAck(consumerId, ledgerId, entryId, null, ackType, validationError, properties, + txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId); } if (ackType == AckType.Cumulative) { @@ -2736,58 +2698,42 @@ private CompletableFuture doTransactionAcknowledgeForResponse(MessageId me } private CompletableFuture doTransactionAcknowledgeForResponse(List messageIds, AckType ackType, - ValidationError validationError, Map properties, TxnID txnID) { - BitSetRecyclable bitSetRecyclable = null; - long ledgerId; - long entryId; - ByteBuf cmd; long requestId = client.newRequestId(); List messageIdDataList = new LinkedList<>(); for (MessageId messageId : messageIds) { - if (messageId instanceof BatchMessageIdImpl) { - BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; - bitSetRecyclable = BitSetRecyclable.create(); + final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId; + final MessageIdData messageIdData = new MessageIdData(); + messageIdData.setLedgerId(messageIdAdv.getLedgerId()); + messageIdData.setEntryId(messageIdAdv.getEntryId()); + if (MessageIdAdvUtils.isBatch(messageIdAdv)) { + final BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create(); + bitSetRecyclable.set(0, messageIdAdv.getBatchSize()); if (ackType == AckType.Cumulative) { - batchMessageId.ackCumulative(); - bitSetRecyclable.set(0, batchMessageId.getBatchSize()); - bitSetRecyclable.clear(0, batchMessageId.getBatchIndex() + 1); + MessageIdAdvUtils.acknowledge(messageIdAdv, false); + bitSetRecyclable.clear(0, messageIdAdv.getBatchIndex() + 1); } else { - bitSetRecyclable.set(0, batchMessageId.getBatchSize()); - bitSetRecyclable.clear(batchMessageId.getBatchIndex()); + bitSetRecyclable.clear(messageIdAdv.getBatchIndex()); } - MessageIdData messageIdData = new MessageIdData(); - messageIdData.setLedgerId(batchMessageId.getLedgerId()); - messageIdData.setEntryId(batchMessageId.getEntryId()); - messageIdData.setBatchSize(batchMessageId.getBatchSize()); - long[] as = bitSetRecyclable.toLongArray(); - for (int i = 0; i < as.length; i++) { - messageIdData.addAckSet(as[i]); + for (long x : bitSetRecyclable.toLongArray()) { + messageIdData.addAckSet(x); } bitSetRecyclable.recycle(); - messageIdDataList.add(messageIdData); - } else { - MessageIdImpl singleMessage = (MessageIdImpl) messageId; - ledgerId = singleMessage.getLedgerId(); - entryId = singleMessage.getEntryId(); - MessageIdData messageIdData = new MessageIdData(); - messageIdData.setLedgerId(ledgerId); - messageIdData.setEntryId(entryId); - messageIdDataList.add(messageIdData); } + messageIdDataList.add(messageIdData); if (ackType == AckType.Cumulative) { unAckedMessageTracker.removeMessagesTill(messageId); } else { unAckedMessageTracker.remove(messageId); } } - cmd = Commands.newAck(consumerId, messageIdDataList, ackType, validationError, properties, + final ByteBuf cmd = Commands.newAck(consumerId, messageIdDataList, ackType, null, properties, txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId); return cnx().newAckForReceipt(cmd, requestId); } - public Map>> getPossibleSendToDeadLetterTopicMessages() { + public Map>> getPossibleSendToDeadLetterTopicMessages() { return possibleSendToDeadLetterTopicMessages; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java new file mode 100644 index 0000000000000..c8b18524ec052 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import java.util.BitSet; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; + +public class MessageIdAdvUtils { + + static int hashCode(MessageIdAdv msgId) { + return (int) (31 * (msgId.getLedgerId() + 31 * msgId.getEntryId()) + + (31 * (long) msgId.getPartitionIndex()) + msgId.getBatchIndex()); + } + + static boolean equals(MessageIdAdv lhs, Object o) { + if (!(o instanceof MessageIdAdv)) { + return false; + } + final MessageIdAdv rhs = (MessageIdAdv) o; + return lhs.getLedgerId() == rhs.getLedgerId() + && lhs.getEntryId() == rhs.getEntryId() + && lhs.getPartitionIndex() == rhs.getPartitionIndex() + && lhs.getBatchIndex() == rhs.getBatchIndex(); + } + + static boolean acknowledge(MessageIdAdv msgId, boolean individual) { + if (!isBatch(msgId)) { + return true; + } + final BitSet ackSet = msgId.getAckSet(); + if (ackSet == null) { + // The internal MessageId implementation should never reach here. If users have implemented their own + // MessageId and getAckSet() is not override, return false to avoid acknowledge current entry. + return false; + } + int batchIndex = msgId.getBatchIndex(); + if (individual) { + ackSet.clear(batchIndex); + } else { + ackSet.clear(0, batchIndex + 1); + } + return ackSet.isEmpty(); + } + + static boolean isBatch(MessageIdAdv msgId) { + return msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0; + } + + static MessageIdAdv discardBatch(MessageId messageId) { + MessageIdAdv msgId = (MessageIdAdv) messageId; + return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId(), msgId.getPartitionIndex()); + } + + static MessageIdAdv prevMessageId(MessageIdAdv msgId) { + return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId() - 1, msgId.getPartitionIndex()); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java index 1a0f491a6a7bb..83ee762578390 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java @@ -18,21 +18,17 @@ */ package org.apache.pulsar.client.impl; -import static org.apache.pulsar.client.impl.BatchMessageIdImpl.NO_BATCH; -import com.google.common.collect.ComparisonChain; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.util.concurrent.FastThreadLocal; import java.io.IOException; import java.util.Objects; -import javax.annotation.Nonnull; import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.TopicMessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.common.api.proto.MessageIdData; -import org.apache.pulsar.common.classification.InterfaceStability; import org.apache.pulsar.common.naming.TopicName; -public class MessageIdImpl implements MessageId { +public class MessageIdImpl implements MessageIdAdv { protected final long ledgerId; protected final long entryId; protected final int partitionIndex; @@ -49,28 +45,29 @@ public MessageIdImpl(long ledgerId, long entryId, int partitionIndex) { this.partitionIndex = partitionIndex; } + @Override public long getLedgerId() { return ledgerId; } + @Override public long getEntryId() { return entryId; } + @Override public int getPartitionIndex() { return partitionIndex; } @Override public int hashCode() { - return messageIdHashCode(ledgerId, entryId, partitionIndex, NO_BATCH); + return MessageIdAdvUtils.hashCode(this); } @Override public boolean equals(Object o) { - return (o instanceof MessageId) - && !(o instanceof MultiMessageIdImpl) - && (compareTo((MessageId) o) == 0); + return MessageIdAdvUtils.equals(this, o); } @Override @@ -100,7 +97,7 @@ public static MessageId fromByteArray(byte[] data) throws IOException { if (idData.hasBatchIndex()) { if (idData.hasBatchSize()) { messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(), - idData.getBatchIndex(), idData.getBatchSize(), BatchMessageAcker.newAcker(idData.getBatchSize())); + idData.getBatchIndex(), idData.getBatchSize(), BatchMessageIdImpl.newAckSet(idData.getBatchSize())); } else { messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(), idData.getBatchIndex()); @@ -118,22 +115,6 @@ public static MessageId fromByteArray(byte[] data) throws IOException { return messageId; } - @InterfaceStability.Unstable - public static MessageIdImpl convertToMessageIdImpl(MessageId messageId) { - if (messageId instanceof TopicMessageId) { - if (messageId instanceof TopicMessageIdImpl) { - return (MessageIdImpl) ((TopicMessageIdImpl) messageId).getInnerMessageId(); - } else { - try { - return (MessageIdImpl) MessageId.fromByteArray(messageId.toByteArray()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - return (MessageIdImpl) messageId; - } - public static MessageId fromByteArrayWithTopic(byte[] data, String topicName) throws IOException { return fromByteArrayWithTopic(data, TopicName.get(topicName)); } @@ -152,10 +133,10 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName) if (idData.hasBatchSize()) { messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(), idData.getBatchIndex(), idData.getBatchSize(), - BatchMessageAcker.newAcker(idData.getBatchSize())); + BatchMessageIdImpl.newAckSet(idData.getBatchSize())); } else { messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(), - idData.getBatchIndex(), 0, BatchMessageAckerDisabled.INSTANCE); + idData.getBatchIndex(), 0, null); } } else { messageId = new MessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition()); @@ -207,36 +188,4 @@ public byte[] toByteArray() { // there is no message batch so we pass -1 return toByteArray(-1, 0); } - - @Override - public int compareTo(@Nonnull MessageId o) { - if (o instanceof MessageIdImpl) { - MessageIdImpl other = (MessageIdImpl) o; - int batchIndex = (o instanceof BatchMessageIdImpl) ? ((BatchMessageIdImpl) o).getBatchIndex() : NO_BATCH; - return messageIdCompare( - this.ledgerId, this.entryId, this.partitionIndex, NO_BATCH, - other.ledgerId, other.entryId, other.partitionIndex, batchIndex - ); - } else if (o instanceof TopicMessageId) { - return compareTo(convertToMessageIdImpl(o)); - } else { - throw new UnsupportedOperationException("Unknown MessageId type: " + o.getClass().getName()); - } - } - - static int messageIdHashCode(long ledgerId, long entryId, int partitionIndex, int batchIndex) { - return (int) (31 * (ledgerId + 31 * entryId) + (31 * (long) partitionIndex) + batchIndex); - } - - static int messageIdCompare( - long ledgerId1, long entryId1, int partitionIndex1, int batchIndex1, - long ledgerId2, long entryId2, int partitionIndex2, int batchIndex2 - ) { - return ComparisonChain.start() - .compare(ledgerId1, ledgerId2) - .compare(entryId1, entryId2) - .compare(partitionIndex1, partitionIndex2) - .compare(batchIndex1, batchIndex2) - .result(); - } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java index 0b6fb608ee62b..d369d639a73a0 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java @@ -40,6 +40,7 @@ import lombok.Getter; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.impl.schema.AbstractSchema; @@ -714,9 +715,10 @@ public boolean hasIndex() { @Override public Optional getIndex() { if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) { - if (msgMetadata.hasNumMessagesInBatch() && messageId instanceof BatchMessageIdImpl) { - int batchSize = ((BatchMessageIdImpl) messageId).getBatchSize(); - int batchIndex = ((BatchMessageIdImpl) messageId).getBatchIndex(); + MessageIdAdv messageIdAdv = (MessageIdAdv) messageId; + if (msgMetadata.hasNumMessagesInBatch() && MessageIdAdvUtils.isBatch(messageIdAdv)) { + int batchSize = messageIdAdv.getBatchSize(); + int batchIndex = messageIdAdv.getBatchIndex(); return Optional.of(brokerEntryMetadata.getIndex() - batchSize + batchIndex + 1); } return Optional.of(brokerEntryMetadata.getIndex()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java index dcae86bd01a3b..f4c9aa2707477 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; +import java.util.BitSet; import java.util.List; import lombok.NonNull; import org.apache.pulsar.client.api.Message; @@ -50,7 +51,7 @@ protected MessagePayloadContextImpl newObject(Handle private MessageIdImpl messageId; private ConsumerImpl consumer; private int redeliveryCount; - private BatchMessageAcker acker; + private BitSet ackSetInMessageId; private BitSetRecyclable ackBitSet; private long consumerEpoch; @@ -73,7 +74,7 @@ public static MessagePayloadContextImpl get(final BrokerEntryMetadata brokerEntr context.messageId = messageId; context.consumer = consumer; context.redeliveryCount = redeliveryCount; - context.acker = BatchMessageAcker.newAcker(context.getNumMessages()); + context.ackSetInMessageId = BatchMessageIdImpl.newAckSet(context.getNumMessages()); context.ackBitSet = (ackSet != null && ackSet.size() > 0) ? BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet)) : null; @@ -88,7 +89,7 @@ public void recycle() { consumer = null; redeliveryCount = 0; consumerEpoch = DEFAULT_CONSUMER_EPOCH; - acker = null; + ackSetInMessageId = null; if (ackBitSet != null) { ackBitSet.recycle(); ackBitSet = null; @@ -134,7 +135,7 @@ public Message getMessageAt(int index, schema, containMetadata, ackBitSet, - acker, + ackSetInMessageId, redeliveryCount, consumerEpoch); } finally { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index f993304b0780a..5fe0e4a82b840 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -55,6 +55,7 @@ import org.apache.pulsar.client.api.ConsumerStats; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.Messages; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.NotSupportedException; @@ -100,7 +101,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { private final MultiTopicConsumerStatsRecorderImpl stats; private final ConsumerConfigurationData internalConfig; - private volatile BatchMessageIdImpl startMessageId = null; + private volatile MessageIdAdv startMessageId; private final long startMessageRollbackDurationInSec; MultiTopicsConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf, ExecutorProvider executorProvider, CompletableFuture> subscribeFuture, Schema schema, @@ -139,9 +140,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { this.consumers = new ConcurrentHashMap<>(); this.pausedConsumers = new ConcurrentLinkedQueue<>(); this.allTopicPartitionsNumber = new AtomicInteger(0); - this.startMessageId = startMessageId != null - ? new BatchMessageIdImpl(MessageIdImpl.convertToMessageIdImpl(startMessageId)) - : null; + this.startMessageId = (MessageIdAdv) startMessageId; this.startMessageRollbackDurationInSec = startMessageRollbackDurationInSec; this.paused = conf.isStartPaused(); @@ -454,18 +453,15 @@ protected CompletableFuture doAcknowledge(MessageId messageId, AckType ack return FutureUtil.failedFuture(new PulsarClientException("Consumer already closed")); } - TopicMessageId topicMessageId = (TopicMessageId) messageId; - ConsumerImpl consumer = consumers.get(topicMessageId.getOwnerTopic()); + ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic()); if (consumer == null) { return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException()); } - MessageId innerMessageId = MessageIdImpl.convertToMessageIdImpl(topicMessageId); if (ackType == AckType.Cumulative) { - return consumer.acknowledgeCumulativeAsync(innerMessageId); + return consumer.acknowledgeCumulativeAsync(messageId); } else { - return consumer.doAcknowledgeWithTxn(innerMessageId, ackType, properties, txnImpl) - .thenRun(() -> - unAckedMessageTracker.remove(topicMessageId)); + return consumer.doAcknowledgeWithTxn(messageId, ackType, properties, txnImpl) + .thenRun(() -> unAckedMessageTracker.remove(messageId)); } } @@ -490,10 +486,9 @@ protected CompletableFuture doAcknowledge(List messageIdList, } Map> topicToMessageIdMap = new HashMap<>(); for (MessageId messageId : messageIdList) { - TopicMessageId topicMessageId = (TopicMessageId) messageId; - topicToMessageIdMap.putIfAbsent(topicMessageId.getOwnerTopic(), new ArrayList<>()); - topicToMessageIdMap.get(topicMessageId.getOwnerTopic()) - .add(MessageIdImpl.convertToMessageIdImpl(topicMessageId)); + String ownerTopic = ((TopicMessageId) messageId).getOwnerTopic(); + topicToMessageIdMap.putIfAbsent(ownerTopic, new ArrayList<>()); + topicToMessageIdMap.get(ownerTopic).add(messageId); } final Map, List> consumerToMessageIds = new IdentityHashMap<>(); for (Map.Entry> entry : topicToMessageIdMap.entrySet()) { @@ -549,10 +544,8 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a @Override public void negativeAcknowledge(MessageId messageId) { checkArgument(messageId instanceof TopicMessageId); - TopicMessageId topicMessageId = (TopicMessageId) messageId; - - ConsumerImpl consumer = consumers.get(topicMessageId.getOwnerTopic()); - consumer.negativeAcknowledge(MessageIdImpl.convertToMessageIdImpl(topicMessageId)); + ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic()); + consumer.negativeAcknowledge(messageId); } @Override @@ -705,12 +698,11 @@ public void redeliverUnacknowledgedMessages(Set messageIds) { return; } removeExpiredMessagesFromQueue(messageIds); - messageIds.stream().map(messageId -> (TopicMessageId) messageId) - .collect(Collectors.groupingBy(TopicMessageId::getOwnerTopic, Collectors.toSet())) - .forEach((topicName, messageIds1) -> - consumers.get(topicName) - .redeliverUnacknowledgedMessages(messageIds1.stream() - .map(MessageIdImpl::convertToMessageIdImpl).collect(Collectors.toSet()))); + messageIds.stream() + .collect(Collectors.groupingBy( + msgId -> ((TopicMessageIdImpl) msgId).getOwnerTopic(), Collectors.toSet())) + .forEach((topicName, messageIds1) -> + consumers.get(topicName).redeliverUnacknowledgedMessages(messageIds1)); resumeReceivingFromPausedConsumersIfNeeded(); } @@ -1508,7 +1500,7 @@ public CompletableFuture getLastMessageIdAsync() { public static boolean isIllegalMultiTopicsMessageId(MessageId messageId) { //only support earliest/latest - return !MessageId.earliest.equals(messageId) && !MessageId.latest.equals(messageId); + return !messageId.equals(MessageId.earliest) && !messageId.equals(MessageId.latest); } public void tryAcknowledgeMessage(Message msg) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java index 70d57db3bb691..37f58a0218091 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java @@ -95,14 +95,6 @@ public synchronized void add(Message message) { } private synchronized void add(MessageId messageId, int redeliveryCount) { - messageId = MessageIdImpl.convertToMessageIdImpl(messageId); - - if (messageId instanceof BatchMessageIdImpl) { - BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; - messageId = new MessageIdImpl(batchMessageId.getLedgerId(), batchMessageId.getEntryId(), - batchMessageId.getPartitionIndex()); - } - if (nackedMessages == null) { nackedMessages = new HashMap<>(); } @@ -113,7 +105,7 @@ private synchronized void add(MessageId messageId, int redeliveryCount) { } else { backoffNs = nackDelayNanos; } - nackedMessages.put(messageId, System.nanoTime() + backoffNs); + nackedMessages.put(MessageIdAdvUtils.discardBatch(messageId), System.nanoTime() + backoffNs); if (this.timeout == null) { // Schedule a task and group all the redeliveries for same period. Leave a small buffer to allow for diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java index 32f8fb922304b..e8951cd3d1692 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java @@ -43,7 +43,7 @@ public boolean isDuplicate(MessageId messageId) { return false; } - public CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType, Map addAcknowledgment(MessageId msgId, AckType ackType, Map properties) { // no-op return CompletableFuture.completedFuture(null); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index fef0bcb8906f1..9086ccc4ef0e0 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -23,6 +23,7 @@ import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; @@ -43,6 +44,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Triple; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.util.TimedCompletableFuture; @@ -79,8 +81,8 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments * This is a set of all the individual acks that the application has issued and that were not already sent to * broker. */ - private final ConcurrentSkipListSet pendingIndividualAcks; - private final ConcurrentHashMap pendingIndividualBatchIndexAcks; + private final ConcurrentSkipListSet pendingIndividualAcks; + private final ConcurrentHashMap pendingIndividualBatchIndexAcks; private final ScheduledFuture scheduledTask; private final boolean batchIndexAckEnabled; @@ -113,18 +115,16 @@ public PersistentAcknowledgmentsGroupingTracker(ConsumerImpl consumer, Consum */ @Override public boolean isDuplicate(MessageId messageId) { - if (!(messageId instanceof MessageIdImpl)) { + if (!(messageId instanceof MessageIdAdv)) { throw new IllegalArgumentException("isDuplicated cannot accept " + messageId.getClass().getName() + ": " + messageId); } - if (lastCumulativeAck.compareTo(messageId) >= 0) { + final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId; + if (lastCumulativeAck.compareTo(messageIdAdv) >= 0) { // Already included in a cumulative ack return true; } else { - final MessageIdImpl messageIdImpl = (messageId instanceof BatchMessageIdImpl) - ? ((BatchMessageIdImpl) messageId).toMessageIdImpl() - : (MessageIdImpl) messageId; - return pendingIndividualAcks.contains(messageIdImpl); + return pendingIndividualAcks.contains(MessageIdAdvUtils.discardBatch(messageIdAdv)); } } @@ -135,10 +135,10 @@ public CompletableFuture addListAcknowledgment(List messageIds, if (consumer.isAckReceiptEnabled()) { Set> completableFutureSet = new HashSet<>(); messageIds.forEach(messageId -> - completableFutureSet.add(addAcknowledgment((MessageIdImpl) messageId, ackType, properties))); + completableFutureSet.add(addAcknowledgment(messageId, ackType, properties))); return FutureUtil.waitForAll(new ArrayList<>(completableFutureSet)); } else { - messageIds.forEach(messageId -> addAcknowledgment((MessageIdImpl) messageId, ackType, properties)); + messageIds.forEach(messageId -> addAcknowledgment(messageId, ackType, properties)); return CompletableFuture.completedFuture(null); } } else { @@ -162,46 +162,43 @@ public CompletableFuture addListAcknowledgment(List messageIds, private void addListAcknowledgment(List messageIds) { for (MessageId messageId : messageIds) { - if (messageId instanceof BatchMessageIdImpl) { - BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; - addIndividualAcknowledgment(batchMessageId.toMessageIdImpl(), - batchMessageId, + MessageIdAdv messageIdAdv = (MessageIdAdv) messageId; + if (MessageIdAdvUtils.isBatch(messageIdAdv)) { + addIndividualAcknowledgment(MessageIdAdvUtils.discardBatch(messageIdAdv), + messageIdAdv, this::doIndividualAckAsync, this::doIndividualBatchAckAsync); - } else if (messageId instanceof MessageIdImpl) { - addIndividualAcknowledgment((MessageIdImpl) messageId, + } else { + addIndividualAcknowledgment(messageIdAdv, null, this::doIndividualAckAsync, this::doIndividualBatchAckAsync); - } else { - throw new IllegalStateException("Unsupported message id type in addListAcknowledgement: " - + messageId.getClass().getCanonicalName()); } } } @Override - public CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType, + public CompletableFuture addAcknowledgment(MessageId msgId, AckType ackType, Map properties) { - if (msgId instanceof BatchMessageIdImpl) { - BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) msgId; - return addAcknowledgment(batchMessageId.toMessageIdImpl(), ackType, properties, batchMessageId); + MessageIdAdv msgIdAdv = (MessageIdAdv) msgId; + if (MessageIdAdvUtils.isBatch(msgIdAdv)) { + return addAcknowledgment(MessageIdAdvUtils.discardBatch(msgId), ackType, properties, msgIdAdv); } else { - return addAcknowledgment(msgId, ackType, properties, null); + return addAcknowledgment(msgIdAdv, ackType, properties, null); } } private CompletableFuture addIndividualAcknowledgment( - MessageIdImpl msgId, - @Nullable BatchMessageIdImpl batchMessageId, - Function> individualAckFunction, - Function> batchAckFunction) { + MessageIdAdv msgId, + @Nullable MessageIdAdv batchMessageId, + Function> individualAckFunction, + Function> batchAckFunction) { if (batchMessageId != null) { consumer.onAcknowledge(batchMessageId, null); } else { consumer.onAcknowledge(msgId, null); } - if (batchMessageId == null || batchMessageId.ackIndividual()) { + if (batchMessageId == null || MessageIdAdvUtils.acknowledge(batchMessageId, true)) { consumer.getStats().incrementNumAcksSent((batchMessageId != null) ? batchMessageId.getBatchSize() : 1); consumer.getUnAckedMessageTracker().remove(msgId); if (consumer.getPossibleSendToDeadLetterTopicMessages() != null) { @@ -215,10 +212,10 @@ private CompletableFuture addIndividualAcknowledgment( } } - private CompletableFuture addAcknowledgment(MessageIdImpl msgId, + private CompletableFuture addAcknowledgment(MessageIdAdv msgId, AckType ackType, Map properties, - @Nullable BatchMessageIdImpl batchMessageId) { + @Nullable MessageIdAdv batchMessageId) { switch (ackType) { case Individual: return addIndividualAcknowledgment(msgId, @@ -231,15 +228,12 @@ private CompletableFuture addAcknowledgment(MessageIdImpl msgId, } else { consumer.onAcknowledgeCumulative(msgId, null); } - if (batchMessageId == null || batchMessageId.ackCumulative()) { + if (batchMessageId == null || MessageIdAdvUtils.acknowledge(batchMessageId, false)) { return doCumulativeAck(msgId, properties, null); } else if (batchIndexAckEnabled) { return doCumulativeBatchIndexAck(batchMessageId, properties); } else { - if (!batchMessageId.getAcker().isPrevBatchCumulativelyAcked()) { - doCumulativeAck(batchMessageId.prevBatchMessageId(), properties, null); - batchMessageId.getAcker().setPrevBatchCumulativelyAcked(true); - } + doCumulativeAck(MessageIdAdvUtils.prevMessageId(batchMessageId), properties, null); return CompletableFuture.completedFuture(null); } default: @@ -247,7 +241,7 @@ private CompletableFuture addAcknowledgment(MessageIdImpl msgId, } } - private CompletableFuture doIndividualAck(MessageIdImpl messageId, Map properties) { + private CompletableFuture doIndividualAck(MessageIdAdv messageId, Map properties) { if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) { // We cannot group acks if the delay is 0 or when there are properties attached to it. Fortunately that's an // uncommon condition since it's only used for the compaction subscription. @@ -267,13 +261,13 @@ private CompletableFuture doIndividualAck(MessageIdImpl messageId, Map doIndividualAckAsync(MessageIdImpl messageId) { + private CompletableFuture doIndividualAckAsync(MessageIdAdv messageId) { pendingIndividualAcks.add(messageId); pendingIndividualBatchIndexAcks.remove(messageId); return CompletableFuture.completedFuture(null); } - private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMessageId, + private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId, Map properties) { if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) { return doImmediateBatchIndexAck(batchMessageId, batchMessageId.getBatchIndex(), @@ -283,7 +277,7 @@ private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMes } } - private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMessageId) { + private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId) { Optional readLock = acquireReadLock(); try { doIndividualBatchAckAsync(batchMessageId); @@ -296,7 +290,7 @@ private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMes } } - private CompletableFuture doCumulativeAck(MessageIdImpl messageId, Map properties, + private CompletableFuture doCumulativeAck(MessageIdAdv messageId, Map properties, BitSetRecyclable bitSet) { consumer.getStats().incrementNumAcksSent(consumer.getUnAckedMessageTracker().removeMessagesTill(messageId)); if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) { @@ -314,29 +308,29 @@ private CompletableFuture doCumulativeAck(MessageIdImpl messageId, Map doIndividualBatchAckAsync(BatchMessageIdImpl batchMessageId) { + private CompletableFuture doIndividualBatchAckAsync(MessageIdAdv msgId) { ConcurrentBitSetRecyclable bitSet = pendingIndividualBatchIndexAcks.computeIfAbsent( - batchMessageId.toMessageIdImpl(), __ -> { - ConcurrentBitSetRecyclable value; - if (batchMessageId.getAcker() != null - && !(batchMessageId.getAcker() instanceof BatchMessageAckerDisabled)) { - value = ConcurrentBitSetRecyclable.create(batchMessageId.getAcker().getBitSet()); + MessageIdAdvUtils.discardBatch(msgId), __ -> { + final BitSet ackSet = msgId.getAckSet(); + final ConcurrentBitSetRecyclable value; + if (ackSet != null && !ackSet.isEmpty()) { + value = ConcurrentBitSetRecyclable.create(ackSet); } else { value = ConcurrentBitSetRecyclable.create(); - value.set(0, batchMessageId.getOriginalBatchSize()); + value.set(0, msgId.getBatchSize()); } return value; }); - bitSet.clear(batchMessageId.getBatchIndex()); + bitSet.clear(msgId.getBatchIndex()); return CompletableFuture.completedFuture(null); } - private void doCumulativeAckAsync(MessageIdImpl msgId, BitSetRecyclable bitSet) { + private void doCumulativeAckAsync(MessageIdAdv msgId, BitSetRecyclable bitSet) { // Handle concurrent updates from different threads lastCumulativeAck.update(msgId, bitSet); } - private CompletableFuture doCumulativeBatchIndexAck(BatchMessageIdImpl batchMessageId, + private CompletableFuture doCumulativeBatchIndexAck(MessageIdAdv batchMessageId, Map properties) { if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) { return doImmediateBatchIndexAck(batchMessageId, batchMessageId.getBatchIndex(), @@ -349,7 +343,7 @@ private CompletableFuture doCumulativeBatchIndexAck(BatchMessageIdImpl bat } } - private CompletableFuture doImmediateAck(MessageIdImpl msgId, AckType ackType, Map properties, + private CompletableFuture doImmediateAck(MessageIdAdv msgId, AckType ackType, Map properties, BitSetRecyclable bitSet) { ClientCnx cnx = consumer.getClientCnx(); @@ -360,7 +354,7 @@ private CompletableFuture doImmediateAck(MessageIdImpl msgId, AckType ackT return newImmediateAckAndFlush(consumer.consumerId, msgId, bitSet, ackType, properties, cnx); } - private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgId, int batchIndex, int batchSize, + private CompletableFuture doImmediateBatchIndexAck(MessageIdAdv msgId, int batchIndex, int batchSize, AckType ackType, Map properties) { ClientCnx cnx = consumer.getClientCnx(); @@ -369,8 +363,8 @@ private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgI .ConnectException("Consumer connect fail! consumer state:" + consumer.getState())); } BitSetRecyclable bitSet; - if (msgId.getAcker() != null && !(msgId.getAcker() instanceof BatchMessageAckerDisabled)) { - bitSet = BitSetRecyclable.valueOf(msgId.getAcker().getBitSet().toLongArray()); + if (msgId.getAckSet() != null) { + bitSet = BitSetRecyclable.valueOf(msgId.getAckSet().toLongArray()); } else { bitSet = BitSetRecyclable.create(); bitSet.set(0, batchSize); @@ -382,7 +376,7 @@ private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgI } CompletableFuture completableFuture = newMessageAckCommandAndWrite(cnx, consumer.consumerId, - msgId.ledgerId, msgId.entryId, bitSet, ackType, properties, true, null, null); + msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, properties, true, null, null); bitSet.recycle(); return completableFuture; } @@ -414,7 +408,7 @@ private void flushAsync(ClientCnx cnx) { boolean shouldFlush = false; if (lastCumulativeAckToFlush != null) { shouldFlush = true; - final MessageIdImpl messageId = lastCumulativeAckToFlush.getMessageId(); + final MessageIdAdv messageId = lastCumulativeAckToFlush.getMessageId(); newMessageAckCommandAndWrite(cnx, consumer.consumerId, messageId.getLedgerId(), messageId.getEntryId(), lastCumulativeAckToFlush.getBitSetRecyclable(), AckType.Cumulative, Collections.emptyMap(), false, @@ -429,7 +423,7 @@ private void flushAsync(ClientCnx cnx) { if (Commands.peerSupportsMultiMessageAcknowledgment(cnx.getRemoteEndpointProtocolVersion())) { // We can send 1 single protobuf command with all individual acks while (true) { - MessageIdImpl msgId = pendingIndividualAcks.pollFirst(); + MessageIdAdv msgId = pendingIndividualAcks.pollFirst(); if (msgId == null) { break; } @@ -452,7 +446,7 @@ private void flushAsync(ClientCnx cnx) { } else { // When talking to older brokers, send the acknowledgements individually while (true) { - MessageIdImpl msgId = pendingIndividualAcks.pollFirst(); + MessageIdAdv msgId = pendingIndividualAcks.pollFirst(); if (msgId == null) { break; } @@ -465,12 +459,13 @@ private void flushAsync(ClientCnx cnx) { } if (!pendingIndividualBatchIndexAcks.isEmpty()) { - Iterator> iterator = + Iterator> iterator = pendingIndividualBatchIndexAcks.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - entriesToAck.add(Triple.of(entry.getKey().ledgerId, entry.getKey().entryId, entry.getValue())); + Map.Entry entry = iterator.next(); + entriesToAck.add(Triple.of( + entry.getKey().getLedgerId(), entry.getKey().getEntryId(), entry.getValue())); iterator.remove(); } } @@ -509,7 +504,7 @@ public void close() { } } - private CompletableFuture newImmediateAckAndFlush(long consumerId, MessageIdImpl msgId, + private CompletableFuture newImmediateAckAndFlush(long consumerId, MessageIdAdv msgId, BitSetRecyclable bitSet, AckType ackType, Map map, ClientCnx cnx) { MessageIdImpl[] chunkMsgIds = this.consumer.unAckedChunkedMessageIdSequenceMap.remove(msgId); @@ -535,7 +530,7 @@ private CompletableFuture newImmediateAckAndFlush(long consumerId, Message completableFuture = CompletableFuture.completedFuture(null); } } else { - completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.ledgerId, msgId.getEntryId(), + completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, map, true, null, null); } return completableFuture; @@ -621,13 +616,13 @@ protected LastCumulativeAck initialValue() { return new LastCumulativeAck(); } }; - public static final MessageIdImpl DEFAULT_MESSAGE_ID = (MessageIdImpl) MessageIdImpl.earliest; + public static final MessageIdAdv DEFAULT_MESSAGE_ID = (MessageIdAdv) MessageId.earliest; - private volatile MessageIdImpl messageId = DEFAULT_MESSAGE_ID; + private volatile MessageIdAdv messageId = DEFAULT_MESSAGE_ID; private BitSetRecyclable bitSetRecyclable = null; private boolean flushRequired = false; - public synchronized void update(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) { + public synchronized void update(final MessageIdAdv messageId, final BitSetRecyclable bitSetRecyclable) { if (compareTo(messageId) < 0) { if (this.bitSetRecyclable != null && this.bitSetRecyclable != bitSetRecyclable) { this.bitSetRecyclable.recycle(); @@ -662,25 +657,22 @@ public synchronized void reset() { flushRequired = false; } - public synchronized int compareTo(MessageId messageId) { - if (this.messageId instanceof BatchMessageIdImpl && (!(messageId instanceof BatchMessageIdImpl))) { - final BatchMessageIdImpl lhs = (BatchMessageIdImpl) this.messageId; - final MessageIdImpl rhs = (MessageIdImpl) messageId; - return MessageIdImpl.messageIdCompare( - lhs.getLedgerId(), lhs.getEntryId(), lhs.getPartitionIndex(), lhs.getBatchIndex(), - rhs.getLedgerId(), rhs.getEntryId(), rhs.getPartitionIndex(), Integer.MAX_VALUE); - } else if (messageId instanceof BatchMessageIdImpl && (!(this.messageId instanceof BatchMessageIdImpl))){ - final MessageIdImpl lhs = this.messageId; - final BatchMessageIdImpl rhs = (BatchMessageIdImpl) messageId; - return MessageIdImpl.messageIdCompare( - lhs.getLedgerId(), lhs.getEntryId(), lhs.getPartitionIndex(), Integer.MAX_VALUE, - rhs.getLedgerId(), rhs.getEntryId(), rhs.getPartitionIndex(), rhs.getBatchIndex()); - } else { - return this.messageId.compareTo(messageId); + public synchronized int compareTo(MessageIdAdv messageId) { + int result = Long.compare(this.messageId.getLedgerId(), messageId.getLedgerId()); + if (result != 0) { + return result; + } + result = Long.compare(this.messageId.getEntryId(), messageId.getEntryId()); + if (result != 0) { + return result; } + return Integer.compare( + (this.messageId.getBatchIndex() >= 0) ? this.messageId.getBatchIndex() : Integer.MAX_VALUE, + (messageId.getBatchIndex() >= 0) ? messageId.getBatchIndex() : Integer.MAX_VALUE + ); } - private synchronized void set(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) { + private synchronized void set(final MessageIdAdv messageId, final BitSetRecyclable bitSetRecyclable) { this.messageId = messageId; this.bitSetRecyclable = bitSetRecyclable; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java index 06f79024c4f24..1c4230470dbc2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java @@ -22,6 +22,8 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; +import org.apache.pulsar.client.api.TopicMessageId; @Data @NoArgsConstructor @@ -67,18 +69,12 @@ private ResetCursorData(String position) { } public ResetCursorData(MessageId messageId) { - if (messageId instanceof BatchMessageIdImpl) { - BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; - this.ledgerId = batchMessageId.getLedgerId(); - this.entryId = batchMessageId.getEntryId(); - this.batchIndex = batchMessageId.getBatchIndex(); - this.partitionIndex = batchMessageId.partitionIndex; - } else if (messageId instanceof MessageIdImpl) { - MessageIdImpl messageIdImpl = (MessageIdImpl) messageId; - this.ledgerId = messageIdImpl.getLedgerId(); - this.entryId = messageIdImpl.getEntryId(); - this.partitionIndex = messageIdImpl.partitionIndex; - } else if (messageId instanceof TopicMessageIdImpl) { + MessageIdAdv messageIdAdv = (MessageIdAdv) messageId; + this.ledgerId = messageIdAdv.getLedgerId(); + this.entryId = messageIdAdv.getEntryId(); + this.batchIndex = messageIdAdv.getBatchIndex(); + this.partitionIndex = messageIdAdv.getPartitionIndex(); + if (messageId instanceof TopicMessageId) { throw new IllegalArgumentException("Not supported operation on partitioned-topic"); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java index 941f18cf65a2c..189dc1c608379 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java @@ -21,16 +21,12 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.TopicMessageId; -public class TopicMessageIdImpl implements TopicMessageId { +public class TopicMessageIdImpl extends TopicMessageId.Impl { - /** This topicPartitionName is get from ConsumerImpl, it contains partition part. */ - private final String topicPartitionName; private final String topicName; - private final MessageId messageId; public TopicMessageIdImpl(String topicPartitionName, String topicName, MessageId messageId) { - this.messageId = messageId; - this.topicPartitionName = topicPartitionName; + super(topicPartitionName, messageId); this.topicName = topicName; } @@ -49,40 +45,21 @@ public String getTopicName() { */ @Deprecated public String getTopicPartitionName() { - return this.topicPartitionName; + return getOwnerTopic(); } + @Deprecated public MessageId getInnerMessageId() { - return messageId; - } - - @Override - public String toString() { - return messageId.toString(); - } - - @Override - public byte[] toByteArray() { - return messageId.toByteArray(); - } - - @Override - public int hashCode() { - return messageId.hashCode(); + return new MessageIdImpl(getLedgerId(), getEntryId(), getPartitionIndex()); } @Override public boolean equals(Object obj) { - return messageId.equals(obj); + return super.equals(obj); } @Override - public int compareTo(MessageId o) { - return messageId.compareTo(o); - } - - @Override - public String getOwnerTopic() { - return topicPartitionName; + public int hashCode() { + return super.hashCode(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java index c3fcb0a16a383..d24ecbd6aa917 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java @@ -70,7 +70,7 @@ public MessageId getMessageId() { @Deprecated public MessageId getInnerMessageId() { - return MessageIdImpl.convertToMessageIdImpl(messageId); + return messageId.getInnerMessageId(); } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java index 42eb197d632d3..ae874b4da6d6b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java @@ -174,7 +174,8 @@ private void triggerZeroQueueSizeListener(final Message message) { } waitingOnListenerForZeroQueueSize = true; trackMessage(message); - unAckedMessageTracker.add(normalizeMessageId(message.getMessageId()), message.getRedeliveryCount()); + unAckedMessageTracker.add( + MessageIdAdvUtils.discardBatch(message.getMessageId()), message.getRedeliveryCount()); listener.received(ZeroQueueConsumerImpl.this, beforeConsume(message)); } catch (Throwable t) { log.error("[{}][{}] Message listener error in processing unqueued message: {}", topic, subscription, diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml index e5f8babe841b8..92ec9e934ee1e 100644 --- a/pulsar-client/src/main/resources/findbugsExclude.xml +++ b/pulsar-client/src/main/resources/findbugsExclude.xml @@ -1007,4 +1007,9 @@ + + + + + diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java index ddca6951e49e1..0418a54c772cc 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java @@ -38,6 +38,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.util.TimedCompletableFuture; @@ -61,7 +62,7 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { eventLoopGroup = new NioEventLoopGroup(1); consumer = mock(ConsumerImpl.class); consumer.unAckedChunkedMessageIdSequenceMap = - ConcurrentOpenHashMap.newBuilder().build(); + ConcurrentOpenHashMap.newBuilder().build(); cnx = spy(new ClientCnxTest(new ClientConfigurationData(), eventLoopGroup)); PulsarClientImpl client = mock(PulsarClientImpl.class); doReturn(client).when(consumer).getClient(); @@ -391,21 +392,21 @@ public void testBatchAckTrackerMultiAck(boolean isNeedReceipt) throws Exception public void testDoIndividualBatchAckAsync() throws Exception{ ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); AcknowledgmentsGroupingTracker tracker = new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup); - MessageId messageId1 = new BatchMessageIdImpl(5, 1, 0, 3, 10, BatchMessageAckerDisabled.INSTANCE); + MessageId messageId1 = new BatchMessageIdImpl(5, 1, 0, 3, 10, null); BitSet bitSet = new BitSet(20); for(int i = 0; i < 20; i ++) { bitSet.set(i, true); } - MessageId messageId2 = new BatchMessageIdImpl(3, 2, 0, 5, 20, BatchMessageAcker.newAcker(bitSet)); + MessageId messageId2 = new BatchMessageIdImpl(3, 2, 0, 5, 20, bitSet); Method doIndividualBatchAckAsync = PersistentAcknowledgmentsGroupingTracker.class - .getDeclaredMethod("doIndividualBatchAckAsync", BatchMessageIdImpl.class); + .getDeclaredMethod("doIndividualBatchAckAsync", MessageIdAdv.class); doIndividualBatchAckAsync.setAccessible(true); doIndividualBatchAckAsync.invoke(tracker, messageId1); doIndividualBatchAckAsync.invoke(tracker, messageId2); Field pendingIndividualBatchIndexAcks = PersistentAcknowledgmentsGroupingTracker.class.getDeclaredField("pendingIndividualBatchIndexAcks"); pendingIndividualBatchIndexAcks.setAccessible(true); - ConcurrentHashMap batchIndexAcks = - (ConcurrentHashMap) pendingIndividualBatchIndexAcks.get(tracker); + ConcurrentHashMap batchIndexAcks = + (ConcurrentHashMap) pendingIndividualBatchIndexAcks.get(tracker); MessageIdImpl position1 = new MessageIdImpl(5, 1, 0); MessageIdImpl position2 = new MessageIdImpl(3, 2, 0); assertTrue(batchIndexAcks.containsKey(position1)); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java deleted file mode 100644 index 1b3795d878cd8..0000000000000 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.impl; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; - -import org.testng.annotations.Test; - -public class BatchMessageAckerDisabledTest { - - @Test - public void testAckIndividual() { - for (int i = 0; i < 10; i++) { - assertTrue(BatchMessageAckerDisabled.INSTANCE.ackIndividual(i)); - } - } - - @Test - public void testAckCumulative() { - for (int i = 0; i < 10; i++) { - assertTrue(BatchMessageAckerDisabled.INSTANCE.ackCumulative(i)); - } - } - - @Test - public void testGetOutstandingAcks() { - assertEquals(0, BatchMessageAckerDisabled.INSTANCE.getOutstandingAcks()); - } - -} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java deleted file mode 100644 index d31fd18cba971..0000000000000 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.impl; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.util.BitSet; - -public class BatchMessageAckerTest { - - private static final int BATCH_SIZE = 10; - - private BatchMessageAcker acker; - - @BeforeMethod - public void setup() { - acker = BatchMessageAcker.newAcker(10); - } - - @Test - public void testAckers() { - assertEquals(BATCH_SIZE, acker.getOutstandingAcks()); - assertEquals(BATCH_SIZE, acker.getBatchSize()); - - assertFalse(acker.ackIndividual(4)); - for (int i = 0; i < BATCH_SIZE; i++) { - if (4 == i) { - assertFalse(acker.getBitSet().get(i)); - } else { - assertTrue(acker.getBitSet().get(i)); - } - } - - assertFalse(acker.ackCumulative(6)); - for (int i = 0; i < BATCH_SIZE; i++) { - if (i <= 6) { - assertFalse(acker.getBitSet().get(i)); - } else { - assertTrue(acker.getBitSet().get(i)); - } - } - - for (int i = BATCH_SIZE - 1; i >= 8; i--) { - assertFalse(acker.ackIndividual(i)); - assertFalse(acker.getBitSet().get(i)); - } - - assertTrue(acker.ackIndividual(7)); - assertEquals(0, acker.getOutstandingAcks()); - } - - @Test - public void testBitSetAcker() { - BitSet bitSet = BitSet.valueOf(acker.getBitSet().toLongArray()); - BatchMessageAcker bitSetAcker = BatchMessageAcker.newAcker(bitSet); - - Assert.assertEquals(acker.getBitSet(), bitSetAcker.getBitSet()); - Assert.assertEquals(acker.getOutstandingAcks(), bitSetAcker.getOutstandingAcks()); - } - -} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java index 6bf9cd943483f..10d805cdc4db3 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java @@ -20,13 +20,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectWriter; import java.io.IOException; import java.util.Collections; -import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.annotations.Test; public class BatchMessageIdImplTest { @@ -123,36 +118,10 @@ public void hashCodeUnbatchedTest() { assertEquals(batchMsgId2.hashCode(), msgId2.hashCode()); } - @Test - public void deserializationTest() { - // initialize BitSet with null - BatchMessageAcker ackerDisabled = new BatchMessageAcker(null, 0); - BatchMessageIdImpl batchMsgId = new BatchMessageIdImpl(0, 0, 0, 0, 0, ackerDisabled); - - ObjectWriter writer = ObjectMapperFactory.create().writerWithDefaultPrettyPrinter(); - - try { - writer.writeValueAsString(batchMsgId); - fail("Shouldn't be deserialized"); - } catch (JsonProcessingException e) { - // expected - assertTrue(e.getCause() instanceof NullPointerException); - } - - // use the default BatchMessageAckerDisabled - BatchMessageIdImpl batchMsgIdToDeserialize = new BatchMessageIdImpl(0, 0, 0, 0); - - try { - writer.writeValueAsString(batchMsgIdToDeserialize); - } catch (JsonProcessingException e) { - fail("Should be successful"); - } - } - @Test public void serializeAndDeserializeTest() throws IOException { BatchMessageIdImpl batchMessageId = new BatchMessageIdImpl(1, 1, 0, - 1, 10, BatchMessageAcker.newAcker(10)); + 1, 10, BatchMessageIdImpl.newAckSet(10)); byte[] serialized = batchMessageId.toByteArray(); BatchMessageIdImpl deserialized = (BatchMessageIdImpl) MessageIdImpl.fromByteArray(serialized); assertEquals(deserialized.getBatchSize(), batchMessageId.getBatchSize()); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java index 7f029635241de..4173d6439b931 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java @@ -43,8 +43,7 @@ public void testProtobufSerialization2() throws Exception { @Test public void testBatchSizeNotSet() throws Exception { - MessageId id = new BatchMessageIdImpl(1L, 2L, 3, 4, -1, - BatchMessageAckerDisabled.INSTANCE); + MessageId id = new BatchMessageIdImpl(1L, 2L, 3, 4, -1, null); byte[] serialized = id.toByteArray(); assertEquals(MessageId.fromByteArray(serialized), id); assertEquals(MessageId.fromByteArrayWithTopic(serialized, "my-topic"), id); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index 28cce0fe62209..7df173da0f195 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -48,6 +48,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic; @@ -325,7 +326,7 @@ public static String getFullyQualifiedInstanceId(String tenant, String namespace } public static final long getSequenceId(MessageId messageId) { - MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(messageId); + MessageIdAdv msgId = (MessageIdAdv) messageId; long ledgerId = msgId.getLedgerId(); long entryId = msgId.getEntryId(); diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java index 10efc91ccdaad..ff0bfd391e80e 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java @@ -26,6 +26,7 @@ import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -389,6 +390,23 @@ static class BatchMessageSequenceRef { int batchIdx; } + private static Method getMethodOfMessageId(MessageId messageId, String name) throws NoSuchMethodException { + Class clazz = messageId.getClass(); + NoSuchMethodException firstException = null; + while (clazz != null) { + try { + return clazz.getDeclaredMethod(name); + } catch (NoSuchMethodException e) { + if (firstException == null) { + firstException = e; + } + clazz = clazz.getSuperclass(); + } + } + assert firstException != null; + throw firstException; + } + @VisibleForTesting static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId messageId) { long ledgerId; @@ -396,23 +414,17 @@ static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId me int batchIdx; try { try { - messageId = (MessageId) messageId.getClass().getDeclaredMethod("getInnerMessageId").invoke(messageId); - } catch (NoSuchMethodException noSuchMethodException) { - // not a TopicMessageIdImpl - } - - try { - batchIdx = (int) messageId.getClass().getDeclaredMethod("getBatchIndex").invoke(messageId); + batchIdx = (int) getMethodOfMessageId(messageId, "getBatchIndex").invoke(messageId); + if (batchIdx < 0) { + return null; + } } catch (NoSuchMethodException noSuchMethodException) { // not a BatchMessageIdImpl, returning null to use the standard sequenceId return null; } - // if getBatchIndex exists it means messageId is a 'BatchMessageIdImpl' instance. - final Class messageIdImplClass = messageId.getClass().getSuperclass(); - - ledgerId = (long) messageIdImplClass.getDeclaredMethod("getLedgerId").invoke(messageId); - entryId = (long) messageIdImplClass.getDeclaredMethod("getEntryId").invoke(messageId); + ledgerId = (long) getMethodOfMessageId(messageId, "getLedgerId").invoke(messageId); + entryId = (long) getMethodOfMessageId(messageId, "getEntryId").invoke(messageId); } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException ex) { log.error("Unexpected error while retrieving sequenceId, messageId class: {}, error: {}", messageId.getClass().getName(), ex.getMessage(), ex); From d0916754ddeb1a4315e4429601941fdc6a210f30 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 4 Apr 2023 23:09:43 +0300 Subject: [PATCH 054/280] [fix][build] Suppress Guava CVE-2020-8908 in OWASP dependency check (#20005) --- src/owasp-dependency-check-suppressions.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 2f73564649445..84ed2a3332cd4 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -463,5 +463,12 @@ CVE-2020-17516 CVE-2021-44521 + + + CVE-2020-8908 + From 0d1fe1821e030c0c0b53a7d47a2bf89151783eb4 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Wed, 5 Apr 2023 15:45:59 +0900 Subject: [PATCH 055/280] [refactor][broker] Suppress error logging when message expiration fails (#19778) --- .../admin/impl/PersistentTopicsBase.java | 85 +++++++++++++------ 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 035e32542ed71..7347d6dbf20a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2098,11 +2098,15 @@ protected void internalExpireMessagesForAllSubscriptions(AsyncResponse asyncResp FutureUtil.waitForAll(futures).handle((result, exception) -> { if (exception != null) { - Throwable t = exception.getCause(); - log.error("[{}] Failed to expire messages up to {} on {}", - clientAppId(), expireTimeInSeconds, - topicName, t); - asyncResponse.resume(new RestException(t)); + Throwable t = FutureUtil.unwrapCompletionException(exception); + if (t instanceof PulsarAdminException) { + log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(), + expireTimeInSeconds, topicName, t.toString()); + } else { + log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(), + expireTimeInSeconds, topicName, t); + } + resumeAsyncResponseExceptionally(asyncResponse, t); return null; } asyncResponse.resume(Response.noContent().build()); @@ -2169,9 +2173,14 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy FutureUtil.waitForAll(futures).handle((result, exception) -> { if (exception != null) { Throwable throwable = FutureUtil.unwrapCompletionException(exception); - log.error("[{}] Failed to expire messages for all subscription up to {} on {}", - clientAppId(), expireTimeInSeconds, topicName, throwable); - asyncResponse.resume(new RestException(throwable)); + if (throwable instanceof RestException) { + log.warn("[{}] Failed to expire messages for all subscription up to {} on {}: {}", + clientAppId(), expireTimeInSeconds, topicName, throwable.toString()); + } else { + log.error("[{}] Failed to expire messages for all subscription up to {} on {}", + clientAppId(), expireTimeInSeconds, topicName, throwable); + } + resumeAsyncResponseExceptionally(asyncResponse, throwable); return null; } asyncResponse.resume(Response.noContent().build()); @@ -3927,17 +3936,24 @@ protected void internalExpireMessagesByTimestamp(AsyncResponse asyncResponse, St FutureUtil.waitForAll(futures).handle((result, exception) -> { if (exception != null) { - Throwable t = exception.getCause(); + Throwable t = FutureUtil.unwrapCompletionException(exception); if (t instanceof NotFoundException) { asyncResponse.resume(new RestException(Status.NOT_FOUND, getSubNotFoundErrorMessage(topicName.toString(), subName))); return null; } else { - log.error("[{}] Failed to expire messages up " - + "to {} on {}", clientAppId(), - expireTimeInSeconds, topicName, t); - asyncResponse.resume(new RestException(t)); + if (t instanceof PulsarAdminException) { + log.warn("[{}] Failed to expire messages up " + + "to {} on {}: {}", clientAppId(), + expireTimeInSeconds, topicName, + t.toString()); + } else { + log.error("[{}] Failed to expire messages up " + + "to {} on {}", clientAppId(), + expireTimeInSeconds, topicName, t); + } + resumeAsyncResponseExceptionally(asyncResponse, t); return null; } } @@ -3955,12 +3971,18 @@ protected void internalExpireMessagesByTimestamp(AsyncResponse asyncResponse, St })) ).exceptionally(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(), - expireTimeInSeconds, topicName, ex); + if (!isRedirectException(cause)) { + if (cause instanceof RestException) { + log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(), expireTimeInSeconds, + topicName, cause.toString()); + } else { + log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(), expireTimeInSeconds, + topicName, cause); + } } - resumeAsyncResponseExceptionally(asyncResponse, ex); + resumeAsyncResponseExceptionally(asyncResponse, cause); return null; }); } @@ -4068,7 +4090,7 @@ protected void internalExpireMessagesByPosition(AsyncResponse asyncResponse, Str future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES, subName)) .thenCompose(__ -> { - log.info("[{}][{}] received expire messages on subscription {} to position {}", clientAppId(), + log.info("[{}][{}] Received expire messages on subscription {} to position {}", clientAppId(), topicName, subName, messageId); return internalExpireMessagesNonPartitionedTopicByPosition(asyncResponse, subName, messageId, isExcluded, batchIndex); @@ -4134,16 +4156,22 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit + topicName + " for subscription " + subName + " due to ongoing" + " message expiration not finished or invalid message position provided."); } + } catch (RestException exception) { + throw exception; } catch (Exception exception) { - log.error("[{}] Failed to expire messages up to {} on {} with subscription {} {}", - clientAppId(), position, topicName, subName, exception); throw new RestException(exception); } asyncResponse.resume(Response.noContent().build()); }).exceptionally(e -> { - log.error("[{}] Failed to expire messages up to {} on {} with subscription {} {}", - clientAppId(), messageId, topicName, subName, e); - asyncResponse.resume(e); + Throwable throwable = FutureUtil.unwrapCompletionException(e); + if (throwable instanceof RestException) { + log.warn("[{}] Failed to expire messages up to {} on {} with subscription {}: {}", + clientAppId(), messageId, topicName, subName, throwable.toString()); + } else { + log.error("[{}] Failed to expire messages up to {} on {} with subscription {}", clientAppId(), + messageId, topicName, subName, throwable); + } + resumeAsyncResponseExceptionally(asyncResponse, throwable); return null; }); } catch (Exception e) { @@ -4152,9 +4180,14 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit resumeAsyncResponseExceptionally(asyncResponse, e); } }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - log.error("[{}] Failed to expire messages up to {} on subscription {} to position {}", clientAppId(), - topicName, subName, messageId, cause); + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof RestException) { + log.warn("[{}] Failed to expire messages up to {} on subscription {} to position {}: {}", clientAppId(), + topicName, subName, messageId, cause.toString()); + } else { + log.error("[{}] Failed to expire messages up to {} on subscription {} to position {}", clientAppId(), + topicName, subName, messageId, cause); + } resumeAsyncResponseExceptionally(asyncResponse, cause); return null; }); From 7813daba863c45f19ab5d8718c961b8aa97137e7 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 5 Apr 2023 17:41:54 +0800 Subject: [PATCH 056/280] [improve][broker] PIP-192: Filter the transfer dest broker (#19958) --- .../extensions/ExtensibleLoadManagerImpl.java | 14 +- .../filter/AntiAffinityGroupPolicyFilter.java | 6 - .../extensions/filter/BrokerFilter.java | 6 - .../filter/BrokerIsolationPoliciesFilter.java | 11 +- .../filter/BrokerMaxTopicCountFilter.java | 6 - .../filter/BrokerVersionFilter.java | 6 - .../AntiAffinityGroupPolicyHelper.java | 35 +--- .../policies/IsolationPoliciesHelper.java | 5 + .../extensions/scheduler/TransferShedder.java | 92 +++++------ ...tiAffinityNamespaceGroupExtensionTest.java | 80 +--------- .../ExtensibleLoadManagerImplTest.java | 10 -- .../BrokerIsolationPoliciesFilterTest.java | 12 +- .../scheduler/TransferShedderTest.java | 151 ++++++++++++++---- 13 files changed, 195 insertions(+), 239 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 3e078d0a5eb20..7e84fa5969a6d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -64,6 +64,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter; import org.apache.pulsar.broker.loadbalance.extensions.scheduler.LoadManagerScheduler; @@ -75,6 +76,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceName; @@ -119,6 +121,9 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { @Getter private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; + @Getter + private IsolationPoliciesHelper isolationPoliciesHelper; + private LoadDataStore brokerLoadDataStore; private LoadDataStore topBundlesLoadDataStore; @@ -185,7 +190,6 @@ public enum Role { public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline = new ArrayList<>(); this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); - this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); // TODO: Make brokerSelectionStrategy configurable. this.brokerSelectionStrategy = new LeastResourceUsageWithWeight(); @@ -236,6 +240,9 @@ public void start() throws PulsarServerException { antiAffinityGroupPolicyHelper.listenFailureDomainUpdate(); this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper); this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter); + SimpleResourceAllocationPolicies policies = new SimpleResourceAllocationPolicies(pulsar); + this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies); + this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper)); try { this.brokerLoadDataStore = LoadDataStoreFactory @@ -293,8 +300,8 @@ public void start() throws PulsarServerException { MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS); this.unloadScheduler = new UnloadScheduler( - pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel, - unloadCounter, unloadMetrics); + pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, + serviceUnitStateChannel, unloadCounter, unloadMetrics); this.unloadScheduler.start(); this.splitScheduler = new SplitScheduler( pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context); @@ -307,7 +314,6 @@ public void start() throws PulsarServerException { public void initialize(PulsarService pulsar) { this.pulsar = pulsar; this.conf = pulsar.getConfiguration(); - this.brokerFilterPipeline.forEach(brokerFilter -> brokerFilter.initialize(pulsar)); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java index 358f985f83e12..462f8f0e3597a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; @@ -50,9 +49,4 @@ public Map filter( public String name() { return FILTER_NAME; } - - @Override - public void initialize(PulsarService pulsar) { - return; - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index 30d25f559b11e..d9cbfdc391ed4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -35,11 +34,6 @@ public interface BrokerFilter { */ String name(); - /** - * Initialize this broker filter using the given pulsar service. - */ - void initialize(PulsarService pulsar); - /** * Filter out unqualified brokers based on implementation. * diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java index b28c77f76f3eb..eeb0d9d3a3309 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -21,12 +21,10 @@ import java.util.Map; import java.util.Set; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; -import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -37,14 +35,13 @@ public class BrokerIsolationPoliciesFilter implements BrokerFilter { private IsolationPoliciesHelper isolationPoliciesHelper; - @Override - public String name() { - return FILTER_NAME; + public BrokerIsolationPoliciesFilter(IsolationPoliciesHelper helper) { + this.isolationPoliciesHelper = helper; } @Override - public void initialize(PulsarService pulsar) { - this.isolationPoliciesHelper = new IsolationPoliciesHelper(new SimpleResourceAllocationPolicies(pulsar)); + public String name() { + return FILTER_NAME; } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index b98edd3d425e5..0bceae36bb8c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -20,7 +20,6 @@ import java.util.Map; import java.util.Optional; -import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; @@ -36,11 +35,6 @@ public String name() { return FILTER_NAME; } - @Override - public void initialize(PulsarService pulsar) { - // No-op - } - @Override public Map filter(Map brokers, ServiceUnitId serviceUnit, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index b7332a5ff10a0..7420fcc211309 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -22,7 +22,6 @@ import java.util.Iterator; import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; @@ -148,9 +147,4 @@ public Version getLatestVersionNumber(Map brokerMap) public String name() { return FILTER_NAME; } - - @Override - public void initialize(PulsarService pulsar) { - // No-op - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java index 69e3302bebd50..44360bc77d83f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -49,38 +48,10 @@ public void filter( channel.getOwnershipEntrySet(), brokerToFailureDomainMap); } - public boolean canUnload( - Map brokers, - String bundle, - String srcBroker, - Optional dstBroker) { - + public boolean hasAntiAffinityGroupPolicy(String bundle) { try { - var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup( - pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle)); - if (antiAffinityGroupOptional.isEmpty()) { - return true; - } - - // bundle has anti-affinityGroup - if (!pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) { - return false; - } - - // copy to retain the input brokers - Map candidates = new HashMap<>(brokers); - - filter(candidates, bundle); - - candidates.remove(srcBroker); - - // unload case - if (dstBroker.isEmpty()) { - return !candidates.isEmpty(); - } - - // transfer case - return candidates.containsKey(dstBroker.get()); + return LoadManagerShared.getNamespaceAntiAffinityGroup( + pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle)).isPresent(); } catch (MetadataStoreException e) { log.error("Failed to check unload candidates. Assumes that bundle:{} cannot unload ", bundle, e); return false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java index 4d7a5bf22d661..67dc702cc0c9f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -26,6 +26,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; @Slf4j @@ -65,4 +66,8 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) { return brokerCandidateCache; } + public boolean hasIsolationPolicy(NamespaceName namespaceName) { + return policies.areIsolationPoliciesPresent(namespaceName); + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 669219daba273..98e05296d605a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -32,6 +32,7 @@ import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -46,6 +47,7 @@ import lombok.experimental.Accessors; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -53,6 +55,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; @@ -60,7 +63,6 @@ import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; -import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.common.naming.NamespaceBundle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,9 +98,10 @@ public class TransferShedder implements NamespaceUnloadStrategy { private static final String CANNOT_UNLOAD_BUNDLE_MSG = "Can't unload bundle:%s."; private final LoadStats stats = new LoadStats(); private PulsarService pulsar; - private SimpleResourceAllocationPolicies allocationPolicies; private IsolationPoliciesHelper isolationPoliciesHelper; private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; + private List brokerFilterPipeline; + private Set decisionCache; @Getter private UnloadCounter counter; @@ -109,7 +112,6 @@ public class TransferShedder implements NamespaceUnloadStrategy { public TransferShedder(UnloadCounter counter){ this.pulsar = null; this.decisionCache = new HashSet<>(); - this.allocationPolicies = null; this.counter = counter; this.isolationPoliciesHelper = null; this.antiAffinityGroupPolicyHelper = null; @@ -117,26 +119,28 @@ public TransferShedder(UnloadCounter counter){ public TransferShedder(PulsarService pulsar, UnloadCounter counter, + List brokerFilterPipeline, + IsolationPoliciesHelper isolationPoliciesHelper, AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper){ this.pulsar = pulsar; this.decisionCache = new HashSet<>(); - this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); this.counter = counter; - this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies); - this.channel = ServiceUnitStateChannelImpl.get(pulsar); + this.isolationPoliciesHelper = isolationPoliciesHelper; this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper; + this.channel = ServiceUnitStateChannelImpl.get(pulsar); + this.brokerFilterPipeline = brokerFilterPipeline; } @Override public void initialize(PulsarService pulsar){ this.pulsar = pulsar; this.decisionCache = new HashSet<>(); - this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); var manager = ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get()); this.counter = manager.getUnloadCounter(); - this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies); - this.channel = ServiceUnitStateChannelImpl.get(pulsar); + this.isolationPoliciesHelper = manager.getIsolationPoliciesHelper(); this.antiAffinityGroupPolicyHelper = manager.getAntiAffinityGroupPolicyHelper(); + this.channel = ServiceUnitStateChannelImpl.get(pulsar); + this.brokerFilterPipeline = manager.getBrokerFilterPipeline(); } @@ -673,7 +677,7 @@ private boolean isTransferable(LoadManagerContext context, String bundle, String srcBroker, Optional dstBroker) { - if (pulsar == null || allocationPolicies == null) { + if (pulsar == null) { return true; } @@ -682,54 +686,50 @@ private boolean isTransferable(LoadManagerContext context, NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespace, bundleRange); - if (!canTransferWithIsolationPoliciesToBroker( - context, availableBrokers, namespaceBundle, srcBroker, dstBroker)) { + if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled(context, namespaceBundle)) { return false; } - if (antiAffinityGroupPolicyHelper != null - && !antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) { - return false; - } - return true; - } - - /** - * Check the gave bundle and broker can be transfer or unload with isolation policies applied. - * - * @param context The load manager context. - * @param availableBrokers The available brokers. - * @param namespaceBundle The bundle try to unload or transfer. - * @param currentBroker The current broker. - * @param targetBroker The broker will be transfer to. - * @return Can be transfer/unload or not. - */ - private boolean canTransferWithIsolationPoliciesToBroker(LoadManagerContext context, - Map availableBrokers, - NamespaceBundle namespaceBundle, - String currentBroker, - Optional targetBroker) { - if (isolationPoliciesHelper == null - || !allocationPolicies.areIsolationPoliciesPresent(namespaceBundle.getNamespaceObject())) { - return true; + Map candidates = new HashMap<>(availableBrokers); + for (var filter : brokerFilterPipeline) { + try { + filter.filter(candidates, namespaceBundle, context); + } catch (BrokerFilterException e) { + log.error("Failed to filter brokers with filter: {}", filter.getClass().getName(), e); + return false; + } } - // bundle has isolation policies. - if (!context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) { - return false; + if (dstBroker.isPresent()) { + if (!candidates.containsKey(dstBroker.get())) { + return false; + } } - boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled(); - Set candidates = isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, namespaceBundle); - // Remove the current bundle owner broker. - candidates.remove(currentBroker); + candidates.remove(srcBroker); + boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled(); // Unload: Check if there are any more candidates available for selection. - if (targetBroker.isEmpty() || !transfer) { + if (dstBroker.isEmpty() || !transfer) { return !candidates.isEmpty(); } // Transfer: Check if this broker is among the candidates. - return candidates.contains(targetBroker.get()); + return candidates.containsKey(dstBroker.get()); + } + + protected boolean isLoadBalancerSheddingBundlesWithPoliciesEnabled(LoadManagerContext context, + NamespaceBundle namespaceBundle) { + if (isolationPoliciesHelper != null + && isolationPoliciesHelper.hasIsolationPolicy(namespaceBundle.getNamespaceObject())) { + return context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled(); + } + + if (antiAffinityGroupPolicyHelper != null + && antiAffinityGroupPolicyHelper.hasAntiAffinityGroupPolicy(namespaceBundle.toString())) { + return context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled(); + } + + return true; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java index 16b87195b1511..3469dbe5a5499 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java @@ -22,8 +22,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; import java.util.AbstractMap; import java.util.HashMap; import java.util.HashSet; @@ -84,83 +82,7 @@ protected void selectBrokerForNamespace( } protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, String bundle) { - try { - String namespaceBundle = namespace + "/" + bundle; - var antiAffinityGroupPolicyHelper = - (AntiAffinityGroupPolicyHelper) - FieldUtils.readDeclaredField( - primaryLoadManager, "antiAffinityGroupPolicyHelper", true); - var brokerRegistry = - (BrokerRegistry) - FieldUtils.readDeclaredField( - primaryLoadManager, "brokerRegistry", true); - var brokers = brokerRegistry - .getAvailableBrokerLookupDataAsync().get(5, TimeUnit.SECONDS); - var serviceUnitStateChannel = (ServiceUnitStateChannel) - FieldUtils.readDeclaredField( - primaryLoadManager, "serviceUnitStateChannel", true); - var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle) - .get(5, TimeUnit.SECONDS).get(); - var brokersCopy = new HashMap<>(brokers); - brokersCopy.remove(srcBroker); - var dstBroker = brokersCopy.entrySet().iterator().next().getKey(); - - // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = true - conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); - assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, - "not-enabled-" + namespace + "/" + bundle, - srcBroker, Optional.of(dstBroker))); - - assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, - "not-enabled-" + namespace + "/" + bundle, - srcBroker, Optional.empty())); - - assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - srcBroker, Optional.of(dstBroker))); - - assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - dstBroker, Optional.of(srcBroker))); - - assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - srcBroker, Optional.empty())); - - assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - dstBroker, Optional.empty())); - - // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = false - conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); - assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, - "not-enabled-" + namespace + "/" + bundle, - srcBroker, Optional.of(dstBroker))); - - assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers, - "not-enabled-" + namespace + "/" + bundle, - srcBroker, Optional.empty())); - - assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - srcBroker, Optional.of(dstBroker))); - - assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - dstBroker, Optional.of(srcBroker))); - - assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - srcBroker, Optional.empty())); - - assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers, - namespaceBundle, - dstBroker, Optional.empty())); - - - } catch (Exception e) { - throw new RuntimeException(e); - } + // No-op } protected boolean isLoadManagerUpdatedDomainCache(Object loadManager) throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 4c57e6b93fa3c..502e02e465e64 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -284,11 +284,6 @@ public String name() { return "Mock broker filter"; } - @Override - public void initialize(PulsarService pulsar) { - // No-op - } - @Override public Map filter(Map brokers, ServiceUnitId serviceUnit, @@ -821,11 +816,6 @@ public String name() { return "Mock-broker-filter"; } - @Override - public void initialize(PulsarService pulsar) { - // No-op - } - } private void setPrimaryLoadManager() throws IllegalAccessException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index a079a23bcea04..5bc1e436bae25 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -31,7 +31,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import org.apache.commons.lang.reflect.FieldUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; @@ -71,13 +70,13 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); - BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(); var policies = mock(SimpleResourceAllocationPolicies.class); // 1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1 setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 1); IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); - FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper); // a. available-brokers: broker1, broker2, broker3 => result: broker1 Map result = filter.filter(new HashMap<>(Map.of( @@ -128,13 +127,14 @@ public void testFilterWithPersistentOrNonPersistentDisabled() doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); - BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(); - var policies = mock(SimpleResourceAllocationPolicies.class); doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); doReturn(true).when(policies).isSharedBroker(any()); IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); - FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper); + + Map result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 8c8d18e202a00..3279cb4e475d3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -44,9 +44,11 @@ import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; @@ -60,6 +62,7 @@ import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; @@ -69,6 +72,9 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter; import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; @@ -86,6 +92,7 @@ import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.policies.data.LocalPolicies; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -108,6 +115,7 @@ public class TransferShedderTest { ExtensibleLoadManagerImpl loadManager; ServiceUnitStateChannel channel; ServiceConfiguration conf; + IsolationPoliciesHelper isolationPoliciesHelper; AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper; LocalPoliciesResources localPoliciesResources; String bundleD1 = "my-tenant/my-namespaceD/0x00000000_0x0FFFFFFF"; @@ -129,6 +137,7 @@ public void init() throws MetadataStoreException { var factory = mock(NamespaceBundleFactory.class); namespaceService = mock(NamespaceService.class); localPoliciesResources = mock(LocalPoliciesResources.class); + isolationPoliciesHelper = mock(IsolationPoliciesHelper.class); antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class); doReturn(conf).when(pulsar).getConfiguration(); doReturn(namespaceService).when(pulsar).getNamespaceService(); @@ -158,8 +167,6 @@ public void init() throws MetadataStoreException { public LoadManagerContext setupContext(){ var ctx = getContext(); - - var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000)); topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000)); @@ -389,13 +396,12 @@ public void startTableView() throws LoadDataStoreException { BrokerRegistry brokerRegistry = mock(BrokerRegistry.class); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class), - "broker3", mock(BrokerLookupData.class), - "broker4", mock(BrokerLookupData.class), - "broker5", mock(BrokerLookupData.class) - ))) - .when(brokerRegistry).getAvailableBrokerLookupDataAsync(); + "broker1", getMockBrokerLookupData(), + "broker2", getMockBrokerLookupData(), + "broker3", getMockBrokerLookupData(), + "broker4", getMockBrokerLookupData(), + "broker5", getMockBrokerLookupData() + ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); doReturn(conf).when(ctx).brokerConfiguration(); doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore(); doReturn(topBundleLoadDataStore).when(ctx).topBundleLoadDataStore(); @@ -403,6 +409,14 @@ public void startTableView() throws LoadDataStoreException { return ctx; } + + BrokerLookupData getMockBrokerLookupData() { + BrokerLookupData brokerLookupData = mock(BrokerLookupData.class); + doReturn(true).when(brokerLookupData).persistentTopicsEnabled(); + doReturn(true).when(brokerLookupData).nonPersistentTopicsEnabled(); + return brokerLookupData; + } + @Test public void testEmptyBrokerLoadData() { UnloadCounter counter = new UnloadCounter(); @@ -524,31 +538,33 @@ public void testRecentlyUnloadedBundles() { @Test public void testGetAvailableBrokersFailed() { UnloadCounter counter = new UnloadCounter(); - AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class); - TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper); + TransferShedder transferShedder = new TransferShedder(pulsar, counter, null, + isolationPoliciesHelper, antiAffinityGroupPolicyHelper); var ctx = setupContext(); BrokerRegistry registry = ctx.brokerRegistry(); doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync(); - var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1); assertEquals(counter.getLoadAvg(), 0.0); assertEquals(counter.getLoadStd(), 0.0); } @Test(timeOut = 30 * 1000) - public void testBundlesWithIsolationPolicies() throws IllegalAccessException { - doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any()); - UnloadCounter counter = new UnloadCounter(); - TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper)); - var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) - spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); - FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true); + public void testBundlesWithIsolationPolicies() { + List filters = new ArrayList<>(); + var allocationPoliciesSpy = mock(SimpleResourceAllocationPolicies.class); IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPoliciesSpy); - FieldUtils.writeDeclaredField(transferShedder, "isolationPoliciesHelper", isolationPoliciesHelper, true); + BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper); + filters.add(filter); + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, filters, + isolationPoliciesHelper, antiAffinityGroupPolicyHelper)); setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", Set.of("broker5"), Set.of(), Set.of(), 1); var ctx = setupContext(); + ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); + doReturn(ctx.brokerConfiguration()).when(pulsar).getConfiguration(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker1")), @@ -641,22 +657,25 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, @Test - public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException { + public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { + var filters = new ArrayList(); + AntiAffinityGroupPolicyFilter filter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper); + filters.add(filter); var counter = new UnloadCounter(); - TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper); - var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) - spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); - doReturn(false).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); - FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true); + TransferShedder transferShedder = new TransferShedder(pulsar, counter, filters, + isolationPoliciesHelper, antiAffinityGroupPolicyHelper); LocalPolicies localPolicies = new LocalPolicies(null, null, "namespaceAntiAffinityGroup"); doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(any()); var ctx = setupContext(); - var antiAffinityGroupPolicyHelperSpy = (AntiAffinityGroupPolicyHelper) - spy(FieldUtils.readDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", true)); - doReturn(false).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), any(), any(), any()); - FieldUtils.writeDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", antiAffinityGroupPolicyHelperSpy, true); + ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); + + doAnswer(invocationOnMock -> { + Map brokers = invocationOnMock.getArgument(0); + brokers.clear(); + return brokers; + }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -664,8 +683,16 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me assertEquals(counter.getLoadAvg(), setupLoadAvg); assertEquals(counter.getLoadStd(), setupLoadStd); + doAnswer(invocationOnMock -> { + Map brokers = invocationOnMock.getArgument(0); + String bundle = invocationOnMock.getArgument(1, String.class); - doReturn(true).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), eq(bundleE1), any(), any()); + if (bundle.equalsIgnoreCase(bundleE1)) { + return brokers; + } + brokers.clear(); + return brokers; + }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected2 = new HashSet<>(); expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), @@ -675,6 +702,68 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me assertEquals(counter.getLoadStd(), setupLoadStd); } + @Test + public void testFilterHasException() throws MetadataStoreException { + var filters = new ArrayList(); + BrokerFilter filter = new BrokerFilter() { + @Override + public String name() { + return "Test-Filter"; + } + + @Override + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + throw new BrokerFilterException("test"); + } + }; + filters.add(filter); + var counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(pulsar, counter, filters, + isolationPoliciesHelper, antiAffinityGroupPolicyHelper); + + var ctx = setupContext(); + ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + assertTrue(res.isEmpty()); + assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + } + + @Test + public void testIsLoadBalancerSheddingBundlesWithPoliciesEnabled() { + var counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(pulsar, counter, new ArrayList<>(), + isolationPoliciesHelper, antiAffinityGroupPolicyHelper); + + var ctx = setupContext(); + + NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); + doReturn("bundle").when(namespaceBundle).toString(); + + boolean[][] expects = { + {true, true, true, true}, + {true, true, false, false}, + {true, false, true, true}, + {true, false, false, false}, + {false, true, true, true}, + {false, true, false, false}, + {false, false, true, true}, + {false, false, false, true} + }; + + for (boolean[] expect : expects) { + doReturn(expect[0]).when(isolationPoliciesHelper).hasIsolationPolicy(any()); + doReturn(expect[1]).when(antiAffinityGroupPolicyHelper).hasAntiAffinityGroupPolicy(any()); + ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(expect[2]); + assertEquals(transferShedder.isLoadBalancerSheddingBundlesWithPoliciesEnabled(ctx, namespaceBundle), + expect[3]); + } + } + @Test public void testTargetStd() { UnloadCounter counter = new UnloadCounter(); From f568c8f49828be42b8a7e81abea45a80cf4f93f4 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 5 Apr 2023 19:12:30 +0800 Subject: [PATCH 057/280] [improve][sql] Remove unnecessary future encapsulation (#19959) Signed-off-by: tison Co-authored-by: tison --- .../presto/PulsarSqlSchemaInfoProvider.java | 33 ++++----- .../sql/presto/TestPulsarRecordCursor.java | 67 ++++++++++--------- 2 files changed, 46 insertions(+), 54 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java index d8f7db96b83d7..e2d030d2d7f1b 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java @@ -27,8 +27,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.SchemaInfoProvider; import org.apache.pulsar.common.naming.TopicName; @@ -50,10 +50,13 @@ public class PulsarSqlSchemaInfoProvider implements SchemaInfoProvider { private final PulsarAdmin pulsarAdmin; - private final LoadingCache cache = CacheBuilder.newBuilder().maximumSize(100000) - .expireAfterAccess(30, TimeUnit.MINUTES).build(new CacheLoader() { + private final LoadingCache> cache = CacheBuilder.newBuilder() + .maximumSize(100000) + .expireAfterAccess(30, TimeUnit.MINUTES) + .build(new CacheLoader<>() { + @Nonnull @Override - public SchemaInfo load(BytesSchemaVersion schemaVersion) throws Exception { + public CompletableFuture load(@Nonnull BytesSchemaVersion schemaVersion) { return loadSchema(schemaVersion); } }); @@ -69,7 +72,7 @@ public CompletableFuture getSchemaByVersion(byte[] schemaVersion) { if (null == schemaVersion) { return completedFuture(null); } - return completedFuture(cache.get(BytesSchemaVersion.of(schemaVersion))); + return cache.get(BytesSchemaVersion.of(schemaVersion)); } catch (ExecutionException e) { LOG.error("Can't get generic schema for topic {} schema version {}", topicName.toString(), new String(schemaVersion, StandardCharsets.UTF_8), e); @@ -79,14 +82,7 @@ public CompletableFuture getSchemaByVersion(byte[] schemaVersion) { @Override public CompletableFuture getLatestSchema() { - try { - return completedFuture(pulsarAdmin.schemas() - .getSchemaInfo(topicName.toString())); - } catch (PulsarAdminException e) { - LOG.error("Can't get current schema for topic {}", - topicName.toString(), e); - return FutureUtil.failedFuture(e.getCause()); - } + return pulsarAdmin.schemas().getSchemaInfoAsync(topicName.toString()); } @Override @@ -94,24 +90,19 @@ public String getTopicName() { return topicName.getLocalName(); } - private SchemaInfo loadSchema(BytesSchemaVersion bytesSchemaVersion) throws PulsarAdminException { + private CompletableFuture loadSchema(BytesSchemaVersion bytesSchemaVersion) { ClassLoader originalContextLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(InjectionManagerFactory.class.getClassLoader()); long version = ByteBuffer.wrap(bytesSchemaVersion.get()).getLong(); - SchemaInfo schemaInfo = pulsarAdmin.schemas().getSchemaInfo(topicName.toString(), version); - if (schemaInfo == null) { - throw new RuntimeException( - "The specific version (" + version + ") schema of the topic " + topicName + " is null"); - } - return schemaInfo; + return pulsarAdmin.schemas().getSchemaInfoAsync(topicName.toString(), version); } finally { Thread.currentThread().setContextClassLoader(originalContextLoader); } } - public static SchemaInfo defaultSchema(){ + public static SchemaInfo defaultSchema() { return Schema.BYTES.getSchemaInfo(); } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 7eaa2da498f45..40ced8e4f8e32 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -18,6 +18,22 @@ */ package org.apache.pulsar.sql.presto; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; import io.airlift.log.Logger; import io.netty.buffer.ByteBuf; @@ -26,7 +42,18 @@ import io.trino.spi.type.RowType; import io.trino.spi.type.Type; import io.trino.spi.type.VarcharType; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import lombok.Data; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -57,34 +84,6 @@ import org.mockito.stubbing.Answer; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import static java.util.concurrent.CompletableFuture.completedFuture; -import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - public class TestPulsarRecordCursor extends TestPulsarConnector { private static final Logger log = Logger.get(TestPulsarRecordCursor.class); @@ -490,8 +489,8 @@ public void testGetSchemaInfo() throws Exception { // If the schemaVersion of the message is not null, try to get the schema. Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO); Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes()); - Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L))) - .thenReturn(Schema.AVRO(Foo.class).getSchemaInfo()); + Mockito.when(schemas.getSchemaInfoAsync(anyString(), eq(0L))) + .thenReturn(CompletableFuture.completedFuture(Schema.AVRO(Foo.class).getSchemaInfo())); schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); assertEquals(SchemaType.AVRO, schemaInfo.getType()); @@ -516,19 +515,21 @@ public void testGetSchemaInfo() throws Exception { // If the specific version schema is null, throw runtime exception. Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes()); - Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null); + Mockito.when(schemas.getSchemaInfoAsync(schemaTopic, 1)) + .thenReturn(CompletableFuture.completedFuture(null)); try { schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); fail("The specific version " + 1 + " schema is null, should fail."); } catch (InvocationTargetException e) { String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString(); assertTrue(e.getCause() instanceof RuntimeException); - assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null")); + assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null")); } // Get the specific version schema. Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes()); - Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo()); + Mockito.when(schemas.getSchemaInfoAsync(schemaTopic, 2)) + .thenReturn(CompletableFuture.completedFuture(Schema.AVRO(Foo.class).getSchemaInfo())); schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo); } From 836f03202dacd078c77dbe9dd2bcbf35f1d1a99e Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Wed, 5 Apr 2023 19:40:24 +0800 Subject: [PATCH 058/280] [fix][client] Fix DeadLetterProducer creation callback blocking client io thread. (#19930) --- .../java/org/apache/pulsar/client/impl/ConsumerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1feef6ca0a642..74e6bf28baa1f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -641,7 +641,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a if (reconsumeTimes > this.deadLetterPolicy.getMaxRedeliverCount() && StringUtils.isNotBlank(deadLetterPolicy.getDeadLetterTopic())) { initDeadLetterProducerIfNeeded(); - deadLetterProducer.thenAccept(dlqProducer -> { + deadLetterProducer.thenAcceptAsync(dlqProducer -> { TypedMessageBuilder typedMessageBuilderNew = dlqProducer.newMessage(Schema.AUTO_PRODUCE_BYTES(retryMessage.getReaderSchema().get())) .value(retryMessage.getData()) @@ -657,7 +657,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a result.completeExceptionally(ex); return null; }); - }).exceptionally(ex -> { + }, internalPinnedExecutor).exceptionally(ex -> { result.completeExceptionally(ex); deadLetterProducer = null; return null; @@ -2038,7 +2038,7 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) return null; }); } - }).exceptionally(ex -> { + }, internalPinnedExecutor).exceptionally(ex -> { log.error("Dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), ex); deadLetterProducer = null; result.complete(false); From 32e677d5264d9145a5765735ae1e6ba0e88a5ef9 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 5 Apr 2023 05:00:32 -0700 Subject: [PATCH 059/280] [improve][broker] PIP-192 updated metrics and cleanup broker selector (#19945) --- .../pulsar/PulsarClusterMetadataSetup.java | 4 +- .../extensions/ExtensibleLoadManagerImpl.java | 27 +-- .../channel/ServiceUnitStateChannelImpl.java | 197 +++++++++++++----- .../extensions/models/AssignCounter.java | 10 +- ...faultNamespaceBundleSplitStrategyImpl.java | 3 +- .../ExtensibleLoadManagerImplTest.java | 93 +++------ .../channel/ServiceUnitStateChannelTest.java | 35 ++-- ...faultNamespaceBundleSplitStrategyTest.java | 2 +- 8 files changed, 212 insertions(+), 159 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index 1a2ca1ec4fb53..0badbda1afdfd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -392,8 +392,8 @@ static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName nam } } - static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, - String cluster) throws IOException { + public static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, + String cluster) throws IOException { createNamespaceIfAbsent(resources, namespaceName, cluster, DEFAULT_BUNDLE_NUMBER); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 7e84fa5969a6d..c1234b6dab21e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -164,8 +164,6 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private final UnloadCounter unloadCounter = new UnloadCounter(); private final SplitCounter splitCounter = new SplitCounter(); - // record load metrics - private final AtomicReference> brokerLoadMetrics = new AtomicReference<>(); // record unload metrics private final AtomicReference> unloadMetrics = new AtomicReference(); // record split metrics @@ -338,7 +336,6 @@ public CompletableFuture> assign(Optional> assign(Optional lookupRequests.remove(bundle)); + future.whenComplete((r, t) -> { + if (t != null) { + assignCounter.incrementFailure(); + } + lookupRequests.remove(bundle); + } + ); return future; } - private CompletableFuture> selectAsync(ServiceUnitId bundle) { + public CompletableFuture> selectAsync(ServiceUnitId bundle) { BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() .thenCompose(availableBrokers -> { @@ -633,20 +636,12 @@ void playFollower() { } } - void updateBrokerLoadMetrics(BrokerLoadData loadData) { - this.brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress())); - } - - private void updateUnloadMetrics(UnloadDecision decision) { - unloadCounter.update(decision); - this.unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress())); - } - public List getMetrics() { List metricsCollection = new ArrayList<>(); - if (this.brokerLoadMetrics.get() != null) { - metricsCollection.addAll(this.brokerLoadMetrics.get()); + if (this.brokerLoadDataReporter != null) { + metricsCollection.addAll(brokerLoadDataReporter.generateLoadData() + .toMetrics(pulsar.getAdvertisedAddress())); } if (this.unloadMetrics.get() != null) { metricsCollection.addAll(this.unloadMetrics.get()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index e7fb59450b25c..ec79698db1f05 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -41,6 +41,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Stable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; +import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished; import com.google.common.annotations.VisibleForTesting; @@ -65,6 +66,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.pulsar.PulsarClusterMetadataSetup; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -76,8 +78,6 @@ import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; -import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; -import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -104,7 +104,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { public static final String TOPIC = TopicName.get( TopicDomain.persistent.value(), - NamespaceName.SYSTEM_NAMESPACE, + SYSTEM_NAMESPACE, "loadbalancer-service-unit-state").toString(); private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec public static final long VERSION_ID_INIT = 1; // initial versionId @@ -113,6 +113,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private static final long MIN_CLEAN_UP_DELAY_TIME_IN_SECS = 0; // 0 secs to clean immediately private static final long MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS = 10; private static final int MAX_OUTSTANDING_PUB_MESSAGES = 500; + private static final long MAX_OWNED_BUNDLE_COUNT_DELAY_TIME_IN_MILLIS = 10 * 60 * 1000; private final PulsarService pulsar; private final ServiceConfiguration config; private final Schema schema; @@ -120,7 +121,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private final String lookupServiceAddress; private final ConcurrentOpenHashMap> cleanupJobs; private final StateChangeListeners stateChangeListeners; - private BrokerSelectionStrategy brokerSelector; + private ExtensibleLoadManagerImpl loadManager; private BrokerRegistry brokerRegistry; private LeaderElectionService leaderElectionService; private TableView tableview; @@ -144,11 +145,15 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private long totalInactiveBrokerCleanupIgnoredCnt = 0; private long totalInactiveBrokerCleanupCancelledCnt = 0; private volatile ChannelState channelState; + private volatile long lastOwnEventHandledAt = 0; + private long lastOwnedServiceUnitCountAt = 0; + private int totalOwnedServiceUnitCnt = 0; public enum EventType { Assign, Split, - Unload + Unload, + Override } @@ -164,7 +169,7 @@ public Counters(){ } // operation metrics - final Map ownerLookUpCounters; + final Map ownerLookUpCounters; final Map eventCounters; final Map handlerCounters; @@ -207,11 +212,11 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.maxCleanupDelayTimeInSecs = MAX_CLEAN_UP_DELAY_TIME_IN_SECS; this.minCleanupDelayTimeInSecs = MIN_CLEAN_UP_DELAY_TIME_IN_SECS; - Map tmpOwnerLookUpCounters = new HashMap<>(); + Map tmpOwnerLookUpCounters = new HashMap<>(); Map tmpHandlerCounters = new HashMap<>(); Map tmpEventCounters = new HashMap<>(); for (var state : ServiceUnitState.values()) { - tmpOwnerLookUpCounters.put(state, new AtomicLong()); + tmpOwnerLookUpCounters.put(state, new Counters()); tmpHandlerCounters.put(state, new Counters()); } for (var event : EventType.values()) { @@ -267,7 +272,7 @@ public synchronized void start() throws PulsarServerException { log.warn("Failed to find the channel leader."); } this.channelState = LeaderElectionServiceStarted; - brokerSelector = getBrokerSelector(); + loadManager = getLoadManager(); if (producer != null) { producer.close(); @@ -275,6 +280,9 @@ public synchronized void start() throws PulsarServerException { log.info("Closed the channel producer."); } } + PulsarClusterMetadataSetup.createNamespaceIfAbsent + (pulsar.getPulsarResources(), NamespaceName.SYSTEM_NAMESPACE, config.getClusterName()); + producer = pulsar.getClient().newProducer(schema) .enableBatching(true) .maxPendingMessages(MAX_OUTSTANDING_PUB_MESSAGES) @@ -329,9 +337,8 @@ protected LoadManagerContext getContext() { } @VisibleForTesting - protected BrokerSelectionStrategy getBrokerSelector() { - // TODO: make this selector configurable. - return new LeastResourceUsageWithWeight(); + protected ExtensibleLoadManagerImpl getLoadManager() { + return ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get()); } @VisibleForTesting @@ -465,7 +472,7 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { ServiceUnitStateData data = tableview.get(serviceUnit); ServiceUnitState state = state(data); - ownerLookUpCounters.get(state).incrementAndGet(); + ownerLookUpCounters.get(state).getTotal().incrementAndGet(); switch (state) { case Owned -> { return CompletableFuture.completedFuture(Optional.of(data.dstBroker())); @@ -474,16 +481,22 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { return CompletableFuture.completedFuture(Optional.of(data.sourceBroker())); } case Assigning, Releasing -> { - return deferGetOwnerRequest(serviceUnit).thenApply( + return deferGetOwnerRequest(serviceUnit).whenComplete((__, e) -> { + if (e != null) { + ownerLookUpCounters.get(state).getFailure().incrementAndGet(); + } + }).thenApply( broker -> broker == null ? Optional.empty() : Optional.of(broker)); } case Init, Free -> { return CompletableFuture.completedFuture(Optional.empty()); } case Deleted -> { + ownerLookUpCounters.get(state).getFailure().incrementAndGet(); return CompletableFuture.failedFuture(new IllegalArgumentException(serviceUnit + " is deleted.")); } default -> { + ownerLookUpCounters.get(state).getFailure().incrementAndGet(); String errorMsg = String.format("Failed to process service unit state data: %s when get owner.", data); log.error(errorMsg); return CompletableFuture.failedFuture(new IllegalStateException(errorMsg)); @@ -522,6 +535,23 @@ public CompletableFuture publishAssignEventAsync(String serviceUnit, Str return getOwnerRequest; } + private CompletableFuture publishOverrideEventAsync(String serviceUnit, + ServiceUnitStateData orphanData, + ServiceUnitStateData override) { + if (!validateChannelState(Started, true)) { + throw new IllegalStateException("Invalid channel state:" + channelState.name()); + } + EventType eventType = EventType.Override; + eventCounters.get(eventType).getTotal().incrementAndGet(); + return pubAsync(serviceUnit, override).whenComplete((__, e) -> { + if (e != null) { + eventCounters.get(eventType).getFailure().incrementAndGet(); + log.error("Failed to override serviceUnit:{} from orphanData:{} to overrideData:{}", + serviceUnit, orphanData, override, e); + } + }).thenApply(__ -> null); + } + public CompletableFuture publishUnloadEventAsync(Unload unload) { if (!validateChannelState(Started, true)) { return CompletableFuture.failedFuture( @@ -658,6 +688,7 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); + lastOwnEventHandledAt = System.currentTimeMillis(); } } @@ -1022,33 +1053,34 @@ private void scheduleCleanup(String broker, long delayInSecs) { broker, delayInSecs, cleanupJobs.size()); } - private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, Set availableBrokers) { - - Optional selectedBroker = brokerSelector.select(availableBrokers, null, getContext()); + private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData) { + Optional selectedBroker = selectBroker(serviceUnit); if (selectedBroker.isPresent()) { - var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData)); + var override = + new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData)); log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", serviceUnit, orphanData, override); - pubAsync(serviceUnit, override).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed to override serviceUnit:{} from orphanData:{} to overrideData:{}", - serviceUnit, orphanData, override, e); - } - }); + publishOverrideEventAsync(serviceUnit, orphanData, override) + .exceptionally(e -> { + log.error( + "Failed to override the ownership serviceUnit:{} orphanData:{}. " + + "Failed to publish override event. totalCleanupErrorCnt:{}", + serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet()); + return null; + }); } else { - log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker.", - serviceUnit, orphanData); + log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker. " + + "totalCleanupErrorCnt:{}", + serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet()); } } - private void doCleanup(String broker) throws ExecutionException, InterruptedException, TimeoutException { + private void doCleanup(String broker) { long startTime = System.nanoTime(); log.info("Started ownership cleanup for the inactive broker:{}", broker); int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); - var availableBrokers = new HashSet<>(brokerRegistry.getAvailableBrokersAsync() - .get(inFlightStateWaitingTimeInMillis, MILLISECONDS)); for (var etr : tableview.entrySet()) { var stateData = etr.getValue(); @@ -1056,13 +1088,13 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce var state = state(stateData); if (StringUtils.equals(broker, stateData.dstBroker())) { if (isActiveState(state)) { - overrideOwnership(serviceUnit, stateData, availableBrokers); + overrideOwnership(serviceUnit, stateData); orphanServiceUnitCleanupCnt++; } } else if (StringUtils.equals(broker, stateData.sourceBroker())) { if (isInFlightState(state)) { - overrideOwnership(serviceUnit, stateData, availableBrokers); + overrideOwnership(serviceUnit, stateData); orphanServiceUnitCleanupCnt++; } } @@ -1097,31 +1129,40 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce } - private Optional getRollForwardStateData( - Set availableBrokers, LoadManagerContext context, long nextVersionId) { - Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); + private Optional selectBroker(String serviceUnit) { + try { + return loadManager.selectAsync(getNamespaceBundle(serviceUnit)) + .get(inFlightStateWaitingTimeInMillis, MILLISECONDS); + } catch (Throwable e) { + log.error("Failed to select a broker for serviceUnit:{}", serviceUnit); + } + return Optional.empty(); + } + + private Optional getRollForwardStateData(String serviceUnit, long nextVersionId) { + Optional selectedBroker = selectBroker(serviceUnit); if (selectedBroker.isEmpty()) { return Optional.empty(); } return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); } + private Optional getOverrideInFlightStateData( String serviceUnit, ServiceUnitStateData orphanData, - Set availableBrokers, - LoadManagerContext context) { + Set availableBrokers) { long nextVersionId = getNextVersionId(orphanData); var state = orphanData.state(); switch (state) { case Assigning: { - return getRollForwardStateData(availableBrokers, context, nextVersionId); + return getRollForwardStateData(serviceUnit, nextVersionId); } case Splitting, Releasing: { if (availableBrokers.contains(orphanData.sourceBroker())) { // rollback to the src return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); } else { - return getRollForwardStateData(availableBrokers, context, nextVersionId); + return getRollForwardStateData(serviceUnit, nextVersionId); } } default: { @@ -1202,25 +1243,27 @@ protected void monitorOwnerships(List brokers) { handleBrokerDeletionEvent(inactiveBroker); } } else if (!orphanServiceUnits.isEmpty()) { - var context = getContext(); for (var etr : orphanServiceUnits.entrySet()) { var orphanServiceUnit = etr.getKey(); var orphanData = etr.getValue(); var overrideData = getOverrideInFlightStateData( - orphanServiceUnit, orphanData, activeBrokers, context); + orphanServiceUnit, orphanData, activeBrokers); if (overrideData.isPresent()) { - pubAsync(orphanServiceUnit, overrideData.get()).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership orphanServiceUnit:{}, orphanData:{}, " - + "cleanupErrorCnt:{}.", - orphanServiceUnit, orphanData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); - } - }); + publishOverrideEventAsync(orphanServiceUnit, orphanData, overrideData.get()) + .whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the ownership orphanServiceUnit:{}, orphanData:{}, " + + "cleanupErrorCnt:{}.", + orphanServiceUnit, orphanData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); orphanServiceUnitCleanupCnt++; } else { - log.warn("Failed get the overrideStateData from orphanServiceUnit:{}, orphanData:{}. will retry..", - orphanServiceUnit, orphanData); + log.warn("Failed get the overrideStateData from orphanServiceUnit:{}, orphanData:{}," + + " cleanupErrorCnt:{}. will retry..", + orphanServiceUnit, orphanData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart); } } } @@ -1279,6 +1322,25 @@ private String printCleanupMetrics() { ); } + private int getTotalOwnedServiceUnitCnt() { + if (tableview == null) { + return 0; + } + long now = System.currentTimeMillis(); + if (lastOwnEventHandledAt > lastOwnedServiceUnitCountAt + || now - lastOwnedServiceUnitCountAt > MAX_OWNED_BUNDLE_COUNT_DELAY_TIME_IN_MILLIS) { + int cnt = 0; + for (var data : tableview.values()) { + if (data.state() == Owned && isTargetBroker(data.dstBroker())) { + cnt++; + } + } + lastOwnedServiceUnitCountAt = now; + totalOwnedServiceUnitCnt = cnt; + } + return totalOwnedServiceUnitCnt; + } + @Override public List getMetrics() { @@ -1288,11 +1350,25 @@ public List getMetrics() { dimensions.put("broker", pulsar.getAdvertisedAddress()); for (var etr : ownerLookUpCounters.entrySet()) { - var dim = new HashMap<>(dimensions); - dim.put("state", etr.getKey().toString()); - var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_owner_lookup_total", etr.getValue()); - metrics.add(metric); + { + var dim = new HashMap<>(dimensions); + dim.put("state", etr.getKey().toString()); + dim.put("result", "Total"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_owner_lookup_total", + etr.getValue().getTotal().get()); + metrics.add(metric); + } + + { + var dim = new HashMap<>(dimensions); + dim.put("state", etr.getKey().toString()); + dim.put("result", "Failure"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_owner_lookup_total", + etr.getValue().getFailure().get()); + metrics.add(metric); + } } for (var etr : eventCounters.entrySet()) { @@ -1371,10 +1447,19 @@ public List getMetrics() { metrics.add(metric); } + { + var dim = new HashMap<>(dimensions); + dim.put("result", "Success"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCnt); + metrics.add(metric); + } + var metric = Metrics.create(dimensions); - metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCnt); + metric.put("brk_sunit_state_chn_orphan_su_cleanup_ops_total", totalOrphanServiceUnitCleanupCnt); metric.put("brk_sunit_state_chn_su_tombstone_cleanup_ops_total", totalServiceUnitTombstoneCleanupCnt); + metric.put("brk_sunit_state_chn_owned_su_total", getTotalOwnedServiceUnitCnt()); metrics.add(metric); return metrics; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java index 26ff1f5f401d1..8e19b7b45e2d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.models; -import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Empty; +import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Failure; import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Skip; import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Success; import java.util.ArrayList; @@ -35,7 +35,7 @@ public class AssignCounter { enum Label { Success, - Empty, + Failure, Skip, } @@ -44,7 +44,7 @@ enum Label { public AssignCounter() { breakdownCounters = Map.of( Success, new AtomicLong(), - Empty, new AtomicLong(), + Failure, new AtomicLong(), Skip, new AtomicLong() ); } @@ -54,8 +54,8 @@ public void incrementSuccess() { breakdownCounters.get(Success).incrementAndGet(); } - public void incrementEmpty() { - breakdownCounters.get(Empty).incrementAndGet(); + public void incrementFailure() { + breakdownCounters.get(Failure).incrementAndGet(); } public void incrementSkip() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java index 15bfdc747f1fc..bd8b3aa66542a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java @@ -98,9 +98,8 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS if (!channel.isOwner(bundle)) { if (debug) { - log.error(String.format(CANNOT_SPLIT_BUNDLE_MSG + log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG + " This broker is not the owner.", bundle)); - counter.update(Failure, Unknown); } continue; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 502e02e465e64..a9b145e30293c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -37,6 +37,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -59,7 +60,6 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import java.net.URL; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -81,13 +81,11 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; +import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; -import org.apache.pulsar.broker.resources.NamespaceResources; -import org.apache.pulsar.broker.resources.PulsarResources; -import org.apache.pulsar.broker.resources.TenantResources; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.TableViewImpl; @@ -97,11 +95,8 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.stats.Metrics; -import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.testcontainers.shaded.org.awaitility.Awaitility; @@ -122,8 +117,6 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { private PulsarTestContext additionalPulsarTestContext; - private PulsarResources resources; - private ExtensibleLoadManagerImpl primaryLoadManager; private ExtensibleLoadManagerImpl secondaryLoadManager; @@ -164,37 +157,6 @@ public void setup() throws Exception { Sets.newHashSet(this.conf.getClusterName())); } - protected void beforePulsarStart(PulsarService pulsar) throws Exception { - if (resources == null) { - MetadataStoreExtended localStore = pulsar.createLocalMetadataStore(null); - MetadataStoreExtended configStore = (MetadataStoreExtended) pulsar.createConfigurationMetadataStore(null); - resources = new PulsarResources(localStore, configStore); - } - this.createNamespaceIfNotExists(resources, NamespaceName.SYSTEM_NAMESPACE.getTenant(), - NamespaceName.SYSTEM_NAMESPACE); - } - - protected void createNamespaceIfNotExists(PulsarResources resources, - String publicTenant, - NamespaceName ns) throws Exception { - TenantResources tr = resources.getTenantResources(); - NamespaceResources nsr = resources.getNamespaceResources(); - - if (!tr.tenantExists(publicTenant)) { - tr.createTenant(publicTenant, - TenantInfo.builder() - .adminRoles(Sets.newHashSet(conf.getSuperUserRoles())) - .allowedClusters(Sets.newHashSet(conf.getClusterName())) - .build()); - } - - if (!nsr.namespaceExists(ns)) { - Policies nsp = new Policies(); - nsp.replication_clusters = Collections.singleton(conf.getClusterName()); - nsr.createPolicies(ns, nsp); - } - } - @Override @AfterClass protected void cleanup() throws Exception { @@ -635,8 +597,8 @@ public void testRoleChange() @Test public void testGetMetrics() throws Exception { { - var brokerLoadMetrics = (AtomicReference>) - FieldUtils.readDeclaredField(primaryLoadManager, "brokerLoadMetrics", true); + var brokerLoadDataReporter = mock(BrokerLoadDataReporter.class); + FieldUtils.writeDeclaredField(primaryLoadManager, "brokerLoadDataReporter", brokerLoadDataReporter, true); BrokerLoadData loadData = new BrokerLoadData(); SystemResourceUsage usage = new SystemResourceUsage(); var cpu = new ResourceUsage(1.0, 100.0); @@ -650,7 +612,7 @@ public void testGetMetrics() throws Exception { usage.setBandwidthIn(bandwidthIn); usage.setBandwidthOut(bandwidthOut); loadData.update(usage, 1, 2, 3, 4, 5, 6, conf); - brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress())); + doReturn(loadData).when(brokerLoadDataReporter).generateLoadData(); } { var unloadMetrics = (AtomicReference>) @@ -700,8 +662,8 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) { AssignCounter assignCounter = new AssignCounter(); assignCounter.incrementSuccess(); - assignCounter.incrementEmpty(); - assignCounter.incrementEmpty(); + assignCounter.incrementFailure(); + assignCounter.incrementFailure(); assignCounter.incrementSkip(); assignCounter.incrementSkip(); assignCounter.incrementSkip(); @@ -709,7 +671,8 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) } { - + FieldUtils.writeDeclaredField(channel1, "lastOwnedServiceUnitCountAt", System.currentTimeMillis(), true); + FieldUtils.writeDeclaredField(channel1, "totalOwnedServiceUnitCnt", 10, true); FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCnt", 1, true); FieldUtils.writeDeclaredField(channel1, "totalServiceUnitTombstoneCleanupCnt", 2, true); FieldUtils.writeDeclaredField(channel1, "totalOrphanServiceUnitCleanupCnt", 3, true); @@ -718,21 +681,21 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupIgnoredCnt", 6, true); FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCancelledCnt", 7, true); - Map ownerLookUpCounters = new LinkedHashMap<>(); + Map ownerLookUpCounters = new LinkedHashMap<>(); Map handlerCounters = new LinkedHashMap<>(); Map eventCounters = new LinkedHashMap<>(); - int i = 1; int j = 0; for (var state : ServiceUnitState.values()) { - ownerLookUpCounters.put(state, new AtomicLong(i)); + ownerLookUpCounters.put(state, + new ServiceUnitStateChannelImpl.Counters( + new AtomicLong(j + 1), new AtomicLong(j + 2))); handlerCounters.put(state, new ServiceUnitStateChannelImpl.Counters( new AtomicLong(j + 1), new AtomicLong(j + 2))); - i++; j += 2; } - i = 0; + int i = 0; for (var type : ServiceUnitStateChannelImpl.EventType.values()) { eventCounters.put(type, new ServiceUnitStateChannelImpl.Counters( @@ -769,22 +732,31 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Admin, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=5}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=6}] - dimensions=[{broker=localhost, metric=assign, result=Empty}], metrics=[{brk_lb_assign_broker_breakdown_total=2}] + dimensions=[{broker=localhost, metric=assign, result=Failure}], metrics=[{brk_lb_assign_broker_breakdown_total=2}] dimensions=[{broker=localhost, metric=assign, result=Skip}], metrics=[{brk_lb_assign_broker_breakdown_total=3}] dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=6}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=7}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=6}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=7}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=8}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=9}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=10}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=11}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=12}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=13}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=14}] dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=1}] dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=2}] dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=3}] dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=4}] dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=5}] dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=6}] + dimensions=[{broker=localhost, event=Override, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=7}] + dimensions=[{broker=localhost, event=Override, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=8}] dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=1}] dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=2}] dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=3}] @@ -803,7 +775,8 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) dimensions=[{broker=localhost, metric=sunitStateChn, result=Skip}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=6}] dimensions=[{broker=localhost, metric=sunitStateChn, result=Cancel}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=7}] dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=5}] - dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=1, brk_sunit_state_chn_orphan_su_cleanup_ops_total=3, brk_sunit_state_chn_su_tombstone_cleanup_ops_total=2}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Success}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_orphan_su_cleanup_ops_total=3, brk_sunit_state_chn_owned_su_total=10, brk_sunit_state_chn_su_tombstone_cleanup_ops_total=2}] """.split("\n")); var actual = primaryLoadManager.getMetrics().stream().map(m -> m.toString()).collect(Collectors.toSet()); assertEquals(actual, expected); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 9eda98e5d842d..2d98bc5cae6e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -75,11 +75,11 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistryImpl; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; -import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; @@ -116,7 +116,7 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { private BrokerRegistryImpl registry; - private BrokerSelectionStrategy brokerSelector; + private ExtensibleLoadManagerImpl loadManager; @BeforeClass @Override @@ -135,7 +135,7 @@ protected void setup() throws Exception { loadManagerContext = mock(LoadManagerContext.class); doReturn(mock(LoadDataStore.class)).when(loadManagerContext).brokerLoadDataStore(); doReturn(mock(LoadDataStore.class)).when(loadManagerContext).topBundleLoadDataStore(); - brokerSelector = mock(BrokerSelectionStrategy.class); + loadManager = mock(ExtensibleLoadManagerImpl.class); additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf()); pulsar2 = additionalPulsarTestContext.getPulsarService(); @@ -496,7 +496,7 @@ public void transferTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests2.size()); // recovered, check the monitor update state : Assigned -> Owned - doReturn(Optional.of(lookupServiceAddress1)).when(brokerSelector).select(any(), any(), any()); + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))).when(loadManager).selectAsync(any()); FieldUtils.writeDeclaredField(channel2, "producer", producer, true); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 1 , true); @@ -714,7 +714,7 @@ public void handleBrokerDeletionEventTest() var owner1 = channel1.getOwnerAsync(bundle1); var owner2 = channel2.getOwnerAsync(bundle2); - doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any()); + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any()); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); @@ -1076,7 +1076,7 @@ public void assignTestWhenDestBrokerFails() "inFlightStateWaitingTimeInMillis", 3 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any()); + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any()); channel1.publishAssignEventAsync(bundle, lookupServiceAddress2); // channel1 is broken. the assign won't be complete. waitUntilState(channel1, bundle); @@ -1418,11 +1418,12 @@ private static void cleanOpsCounters(ServiceUnitStateChannel channel) } var ownerLookUpCounters = - (Map) + (Map) FieldUtils.readDeclaredField(channel, "ownerLookUpCounters", true); for(var val : ownerLookUpCounters.values()){ - val.set(0); + val.getFailure().set(0); + val.getTotal().set(0); } } @@ -1518,20 +1519,20 @@ private static void validateOwnerLookUpCounters(ServiceUnitStateChannel channel, ) throws IllegalAccessException { var ownerLookUpCounters = - (Map) + (Map) FieldUtils.readDeclaredField(channel, "ownerLookUpCounters", true); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> { // wait until true - assertEquals(assigned, ownerLookUpCounters.get(Assigning).get()); - assertEquals(owned, ownerLookUpCounters.get(Owned).get()); - assertEquals(released, ownerLookUpCounters.get(Releasing).get()); - assertEquals(splitting, ownerLookUpCounters.get(Splitting).get()); - assertEquals(free, ownerLookUpCounters.get(Free).get()); - assertEquals(deleted, ownerLookUpCounters.get(Deleted).get()); - assertEquals(init, ownerLookUpCounters.get(Init).get()); + assertEquals(assigned, ownerLookUpCounters.get(Assigning).getTotal().get()); + assertEquals(owned, ownerLookUpCounters.get(Owned).getTotal().get()); + assertEquals(released, ownerLookUpCounters.get(Releasing).getTotal().get()); + assertEquals(splitting, ownerLookUpCounters.get(Splitting).getTotal().get()); + assertEquals(free, ownerLookUpCounters.get(Free).getTotal().get()); + assertEquals(deleted, ownerLookUpCounters.get(Deleted).getTotal().get()); + assertEquals(init, ownerLookUpCounters.get(Init).getTotal().get()); }); } @@ -1565,7 +1566,7 @@ ServiceUnitStateChannelImpl createChannel(PulsarService pulsar) doReturn(loadManagerContext).when(channel).getContext(); doReturn(registry).when(channel).getBrokerRegistry(); - doReturn(brokerSelector).when(channel).getBrokerSelector(); + doReturn(loadManager).when(channel).getLoadManager(); var leaderElectionService = new LeaderElectionService( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java index 0aa055b58acd3..8c765e7a3df64 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java @@ -169,7 +169,7 @@ public void testNoBundleOwner() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); var expected = Set.of(); assertEquals(actual, expected); - verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); + verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } public void testError() throws Exception { From 0c9a866f948ed6636050fac110563ad4e64bb3b1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 5 Apr 2023 16:21:33 +0300 Subject: [PATCH 060/280] [improve][proxy] Implement graceful shutdown for Pulsar Proxy (#20011) --- .../pulsar/proxy/server/ProxyService.java | 76 +++++++++++++++---- 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index dfe888d06510e..4cca24f5f4892 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.proxy.server; import static java.util.Objects.requireNonNull; @@ -188,7 +189,7 @@ public ProxyService(ProxyConfiguration proxyConfig, statsExecutor = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("proxy-stats-executor")); - statsExecutor.schedule(()->{ + statsExecutor.schedule(() -> { this.clientCnxs.forEach(cnx -> { if (cnx.getDirectProxyHandler() != null && cnx.getDirectProxyHandler().getInboundChannelRequestsRate() != null) { @@ -223,7 +224,7 @@ public void start() throws Exception { pulsarResources = new PulsarResources(localMetadataStore, configMetadataStore); discoveryProvider = new BrokerDiscoveryProvider(this.proxyConfig, pulsarResources); authorizationService = new AuthorizationService(PulsarConfigurationLoader.convertFrom(proxyConfig), - pulsarResources); + pulsarResources); } ServerBootstrap bootstrap = new ServerBootstrap(); @@ -265,7 +266,7 @@ public void start() throws Exception { } final String hostname = - ServiceConfigurationUtils.getDefaultOrConfiguredAddress(proxyConfig.getAdvertisedAddress()); + ServiceConfigurationUtils.getDefaultOrConfiguredAddress(proxyConfig.getAdvertisedAddress()); if (proxyConfig.getServicePort().isPresent()) { this.serviceUrl = String.format("pulsar://%s:%d/", hostname, getListenPort().get()); @@ -352,18 +353,38 @@ public BrokerDiscoveryProvider getDiscoveryProvider() { } public void close() throws IOException { - dnsAddressResolverGroup.close(); + if (listenChannel != null) { + try { + listenChannel.close().sync(); + } catch (InterruptedException e) { + LOG.info("Shutdown of listenChannel interrupted"); + Thread.currentThread().interrupt(); + } + } - if (discoveryProvider != null) { - discoveryProvider.close(); + if (listenChannelTls != null) { + try { + listenChannelTls.close().sync(); + } catch (InterruptedException e) { + LOG.info("Shutdown of listenChannelTls interrupted"); + Thread.currentThread().interrupt(); + } } - if (listenChannel != null) { - listenChannel.close(); + // Don't accept any new connections + try { + acceptorGroup.shutdownGracefully().sync(); + } catch (InterruptedException e) { + LOG.info("Shutdown of acceptorGroup interrupted"); + Thread.currentThread().interrupt(); } - if (listenChannelTls != null) { - listenChannelTls.close(); + closeAllConnections(); + + dnsAddressResolverGroup.close(); + + if (discoveryProvider != null) { + discoveryProvider.close(); } if (statsExecutor != null) { @@ -391,10 +412,39 @@ public void close() throws IOException { throw new IOException(e); } } - acceptorGroup.shutdownGracefully(); - workerGroup.shutdownGracefully(); + try { + workerGroup.shutdownGracefully().sync(); + } catch (InterruptedException e) { + LOG.info("Shutdown of workerGroup interrupted"); + Thread.currentThread().interrupt(); + } for (EventLoopGroup group : extensionsWorkerGroups) { - group.shutdownGracefully(); + try { + group.shutdownGracefully().sync(); + } catch (InterruptedException e) { + LOG.info("Shutdown of {} interrupted", group); + Thread.currentThread().interrupt(); + } + } + LOG.info("ProxyService closed."); + } + + private void closeAllConnections() { + try { + workerGroup.submit(() -> { + // Close all the connections + if (!clientCnxs.isEmpty()) { + LOG.info("Closing {} proxy connections, including connections to brokers", clientCnxs.size()); + for (ProxyConnection clientCnx : clientCnxs) { + clientCnx.ctx().close(); + } + } else { + LOG.info("No proxy connections to close"); + } + }).sync(); + } catch (InterruptedException e) { + LOG.info("Closing of connections interrupted"); + Thread.currentThread().interrupt(); } } From be7e890d91cbb28014280b088de35b3a3bf513b8 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 5 Apr 2023 22:32:48 +0800 Subject: [PATCH 061/280] [improve][broker] PIP-192: Update the lookup data path to support deploy and rollback (#19999) Co-authored-by: Jiwe Guo --- .../extensions/BrokerRegistryImpl.java | 2 +- .../ExtensibleLoadManagerImplTest.java | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java index de0d361316d8d..4c2beba5d30b3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java @@ -51,7 +51,7 @@ @Slf4j public class BrokerRegistryImpl implements BrokerRegistry { - protected static final String LOOKUP_DATA_PATH = "/loadbalance/brokers"; + protected static final String LOOKUP_DATA_PATH = "/loadbalance/extension/brokers"; private final PulsarService pulsar; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index a9b145e30293c..57590d8d7da0e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -84,6 +84,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -166,7 +167,7 @@ protected void cleanup() throws Exception { this.additionalPulsarTestContext.close(); } - @BeforeMethod + @BeforeMethod(alwaysRun = true) protected void initializeState() throws PulsarAdminException { admin.namespaces().unload("public/default"); reset(primaryLoadManager, secondaryLoadManager); @@ -470,6 +471,33 @@ public Map filter(Map broker assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress()); } + @Test + public void testStartOldLoadManager() throws Exception { + ServiceConfiguration defaultConf = getDefaultConf(); + defaultConf.setAllowAutoTopicCreation(true); + defaultConf.setForceDeleteNamespaceAllowed(true); + defaultConf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerSheddingEnabled(false); + try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { + // start pulsar3 with old load manager + var pulsar3 = additionalPulsarTestContext.getPulsarService(); + + var availableBrokers = pulsar3.getLoadManager().get().getAvailableBrokers(); + assertEquals(availableBrokers.size(), 1); + assertEquals(availableBrokers.iterator().next(), pulsar3.getLookupServiceAddress()); + + availableBrokers = pulsar1.getLoadManager().get().getAvailableBrokers(); + assertEquals(availableBrokers.size(), 2); + assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress())); + assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress())); + + availableBrokers = pulsar2.getLoadManager().get().getAvailableBrokers(); + assertEquals(availableBrokers.size(), 2); + assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress())); + assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress())); + } + } + @Test public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception { var topBundlesLoadDataStorePrimary = From 1a6c28dd0072de05a544dbc9243bfbe6bccea5db Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 5 Apr 2023 18:10:55 -0500 Subject: [PATCH 062/280] [fix][broker] Only validate superuser access if authz enabled (#19989) ### Motivation In #19455, I added a requirement that only the proxy role could supply an original principal. That check is only supposed to apply when the broker has authorization enabled. However, in one case, that was not the case. This PR does a check and returns early when authorization is not enabled in the broker. See https://github.com/apache/pulsar/pull/19830#issuecomment-1492262201 for additional motivation. ### Modifications * Update the `PulsarWebResource#validateSuperUserAccessAsync` to only validate when authentication and authorization are enabled in the configuration. ### Verifying this change This is a trivial change. It'd be good to add tests, but I didn't include them here because this is a somewhat urgent fix. There was one test that broke because of this change, so there is at least some existing coverage. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/39 --- .../pulsar/broker/web/PulsarWebResource.java | 29 +++++++------------ .../admin/v3/AdminApiTransactionTest.java | 1 + 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index c5713ebddaa2f..a182e4733fdfe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -185,8 +185,8 @@ protected boolean hasSuperUserAccess() { return true; } - public CompletableFuture validateSuperUserAccessAsync(){ - if (!config().isAuthenticationEnabled()) { + public CompletableFuture validateSuperUserAccessAsync() { + if (!config().isAuthenticationEnabled() || !config().isAuthorizationEnabled()) { return CompletableFuture.completedFuture(null); } String appId = clientAppId(); @@ -221,22 +221,15 @@ public CompletableFuture validateSuperUserAccessAsync(){ } }); } else { - if (config().isAuthorizationEnabled()) { - return pulsar.getBrokerService() - .getAuthorizationService() - .isSuperUser(appId, clientAuthData()) - .thenAccept(proxyAuthorizationSuccess -> { - if (!proxyAuthorizationSuccess) { - throw new RestException(Status.UNAUTHORIZED, - "This operation requires super-user access"); - } - }); - } - if (log.isDebugEnabled()) { - log.debug("Successfully authorized {} as super-user", - appId); - } - return CompletableFuture.completedFuture(null); + return pulsar.getBrokerService() + .getAuthorizationService() + .isSuperUser(appId, clientAuthData()) + .thenAccept(proxyAuthorizationSuccess -> { + if (!proxyAuthorizationSuccess) { + throw new RestException(Status.UNAUTHORIZED, + "This operation requires super-user access"); + } + }); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java index 019b7c11fd579..0e51470da75a5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java @@ -630,6 +630,7 @@ public void testUpdateTransactionCoordinatorNumber() throws Exception { Awaitility.await().until(() -> pulsar.getTransactionMetadataStoreService().getStores().size() == coordinatorSize * 2); pulsar.getConfiguration().setAuthenticationEnabled(true); + pulsar.getConfiguration().setAuthorizationEnabled(true); Set proxyRoles = spy(Set.class); doReturn(true).when(proxyRoles).contains(any()); pulsar.getConfiguration().setProxyRoles(proxyRoles); From 94ae340f94e004fb1de76a2d05e1e7160bb8eff1 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 6 Apr 2023 12:59:34 +0800 Subject: [PATCH 063/280] [fix][client] Release the orphan producers after the primary consumer is closed (#19858) Motivation: The producers ["retryLetterProducer", "deadLetterProducer"] will be auto-created by consumers if enabled `DLQ`, but these producers will not close after consumers are closed. Modifications: Auto close "retryLetterProducer" and "deadLetterProducer" after the primary consumer is closed --- .../pulsar/client/api/RetryTopicTest.java | 48 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 18 ++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java index 7012cb0d698ac..2ccae72143443 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.Data; @@ -650,4 +651,51 @@ public void testRetryTopicException() throws Exception { consumer.close(); } + + @Test(timeOut = 30000L) + public void testRetryProducerWillCloseByConsumer() throws Exception { + final String topicName = "persistent://my-property/my-ns/tp_" + UUID.randomUUID().toString(); + final String subscriptionName = "sub1"; + final String topicRetry = topicName + "-" + subscriptionName + "-RETRY"; + final String topicDLQ = topicName + "-" + subscriptionName + "-DLQ"; + + // Trigger the DLQ and retry topic creation. + final Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .enableRetry(true) + .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic(topicDLQ).maxRedeliverCount(2).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + final Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + // send messages. + for (int i = 0; i < 5; i++) { + producer.newMessage() + .value("msg-" + i) + .sendAsync(); + } + producer.flush(); + for (int i = 0; i < 20; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + if (msg != null) { + consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS); + } else { + break; + } + } + + consumer.close(); + producer.close(); + admin.topics().delete(topicName, false); + + // Verify: "retryLetterProducer" and "deadLetterProducer" will be closed by "consumer.close()", so these two + // topics can be deleted successfully. + admin.topics().delete(topicRetry, false); + admin.topics().delete(topicDLQ, false); + } + } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 74e6bf28baa1f..fb372566426d3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1041,7 +1041,23 @@ public CompletableFuture closeAsync() { }); } - return closeFuture; + ArrayList> closeFutures = new ArrayList<>(4); + closeFutures.add(closeFuture); + if (retryLetterProducer != null) { + closeFutures.add(retryLetterProducer.closeAsync().whenComplete((ignore, ex) -> { + if (ex != null) { + log.warn("Exception ignored in closing retryLetterProducer of consumer", ex); + } + })); + } + if (deadLetterProducer != null) { + closeFutures.add(deadLetterProducer.thenCompose(p -> p.closeAsync()).whenComplete((ignore, ex) -> { + if (ex != null) { + log.warn("Exception ignored in closing deadLetterProducer of consumer", ex); + } + })); + } + return FutureUtil.waitForAll(closeFutures); } private void cleanupAtClose(CompletableFuture closeFuture, Throwable exception) { From f76beda21134aff24cc0384e5447ffbe33c5c039 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 6 Apr 2023 00:59:21 -0500 Subject: [PATCH 064/280] [improve][txn] Cleanup how superusers abort txns (#19976) ### Motivation This PR builds on https://github.com/apache/pulsar/pull/19467. When we modify/abort transactions, we need to make sure that authorization is checked for both the proxy and the client. ### Modifications * Add a second authorization check when `originalPrincipal` is set in the `ServerCnx`. * Fix a bug where we were not doing a deep copy of the `SubscriptionsList` object. (Tests caught this bug!) ### Verifying this change Added a new test to cover some of the changes. ### Does this pull request potentially affect one of the following parts: This is an internal change. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/38 --- .../pulsar/broker/service/ServerCnx.java | 49 ++++++++++++------- .../pulsar/broker/service/ServerCnxTest.java | 43 ++++++++++++++++ 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 4fc79a124acd8..973dec8980ea0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.IdentityHashMap; @@ -2534,32 +2535,39 @@ protected void handleEndTxn(CommandEndTxn command) { }); } - private CompletableFuture verifyTxnOwnershipForTCToBrokerCommands() { + private CompletableFuture isSuperUser() { + assert ctx.executor().inEventLoop(); if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) { - return getBrokerService() - .getAuthorizationService() - .isSuperUser(getPrincipal(), getAuthenticationData()); + CompletableFuture isAuthRoleAuthorized = service.getAuthorizationService().isSuperUser( + authRole, authenticationData); + if (originalPrincipal != null) { + CompletableFuture isOriginalPrincipalAuthorized = service.getAuthorizationService() + .isSuperUser(originalPrincipal, + originalAuthData != null ? originalAuthData : authenticationData); + return isOriginalPrincipalAuthorized.thenCombine(isAuthRoleAuthorized, + (originalPrincipal, authRole) -> originalPrincipal && authRole); + } else { + return isAuthRoleAuthorized; + } } else { return CompletableFuture.completedFuture(true); } } private CompletableFuture verifyTxnOwnership(TxnID txnID) { - final String checkOwner = getPrincipal(); + assert ctx.executor().inEventLoop(); return service.pulsar().getTransactionMetadataStoreService() - .verifyTxnOwnership(txnID, checkOwner) - .thenCompose(isOwner -> { + .verifyTxnOwnership(txnID, getPrincipal()) + .thenComposeAsync(isOwner -> { if (isOwner) { return CompletableFuture.completedFuture(true); } if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) { - return getBrokerService() - .getAuthorizationService() - .isSuperUser(checkOwner, getAuthenticationData()); + return isSuperUser(); } else { return CompletableFuture.completedFuture(false); } - }); + }, ctx.executor()); } @Override @@ -2576,10 +2584,10 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) { txnID, txnAction); } CompletableFuture> topicFuture = service.getTopicIfExists(TopicName.get(topic).toString()); - topicFuture.thenAccept(optionalTopic -> { + topicFuture.thenAcceptAsync(optionalTopic -> { if (optionalTopic.isPresent()) { - // we only accept super user becase this endpoint is reserved for tc to broker communication - verifyTxnOwnershipForTCToBrokerCommands() + // we only accept superuser because this endpoint is reserved for tc to broker communication + isSuperUser() .thenCompose(isOwner -> { if (!isOwner) { return failedFutureTxnTcNotAllowed(txnID); @@ -2629,7 +2637,7 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) { return null; }); } - }).exceptionally(e -> { + }, ctx.executor()).exceptionally(e -> { log.error("handleEndTxnOnPartition fail ! topic {}, " + "txnId: [{}], txnAction: [{}]", topic, txnID, TxnAction.valueOf(txnAction), e.getCause()); @@ -2658,7 +2666,7 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) { } CompletableFuture> topicFuture = service.getTopicIfExists(TopicName.get(topic).toString()); - topicFuture.thenAccept(optionalTopic -> { + topicFuture.thenAcceptAsync(optionalTopic -> { if (optionalTopic.isPresent()) { Subscription subscription = optionalTopic.get().getSubscription(subName); if (subscription == null) { @@ -2670,7 +2678,7 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) { return; } // we only accept super user becase this endpoint is reserved for tc to broker communication - verifyTxnOwnershipForTCToBrokerCommands() + isSuperUser() .thenCompose(isOwner -> { if (!isOwner) { return failedFutureTxnTcNotAllowed(txnID); @@ -2720,7 +2728,7 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) { return null; }); } - }).exceptionally(e -> { + }, ctx.executor()).exceptionally(e -> { log.error("handleEndTxnOnSubscription fail ! topic: {}, subscription: {}" + "txnId: [{}], txnAction: [{}]", topic, subName, txnID, TxnAction.valueOf(txnAction), e.getCause()); @@ -2757,7 +2765,10 @@ protected void handleAddSubscriptionToTxn(CommandAddSubscriptionToTxn command) { checkArgument(state == State.Connected); final TxnID txnID = new TxnID(command.getTxnidMostBits(), command.getTxnidLeastBits()); final long requestId = command.getRequestId(); - final List subscriptionsList = command.getSubscriptionsList(); + final List subscriptionsList = new ArrayList<>(); + for (org.apache.pulsar.common.api.proto.Subscription sub : command.getSubscriptionsList()) { + subscriptionsList.add(new org.apache.pulsar.common.api.proto.Subscription().copyFrom(sub)); + } if (log.isDebugEnabled()) { log.debug("Receive add published partition to txn request {} from {} with txnId {}", requestId, remoteAddress, txnID); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 4580f028de2b0..874d43896223c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -33,6 +33,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.matches; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -56,6 +57,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -75,6 +77,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockAlwaysExpiredAuthenticationProvider; +import org.apache.pulsar.broker.auth.MockAuthorizationProvider; import org.apache.pulsar.broker.auth.MockMutableAuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -3171,6 +3174,46 @@ public void sendAddPartitionToTxnResponseFailed() throws Exception { channel.finish(); } + @Test(timeOut = 30000) + public void sendAddPartitionToTxnResponseFailedAuth() throws Exception { + AuthenticationService authenticationService = mock(AuthenticationService.class); + AuthenticationProvider authenticationProvider = new MockAuthenticationProvider(); + String authMethodName = authenticationProvider.getAuthMethodName(); + when(brokerService.getAuthenticationService()).thenReturn(authenticationService); + when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); + svcConfig.setAuthenticationEnabled(true); + svcConfig.setProxyRoles(Set.of("pass.fail")); + + svcConfig.setAuthorizationProvider(MockAuthorizationProvider.class.getName()); + AuthorizationService authorizationService = + spyWithClassAndConstructorArgsRecordingInvocations(AuthorizationService.class, svcConfig, + pulsarTestContext.getPulsarResources()); + when(brokerService.getAuthorizationService()).thenReturn(authorizationService); + svcConfig.setAuthorizationEnabled(true); + + final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(false)); + when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); + svcConfig.setTransactionCoordinatorEnabled(true); + resetChannel(); + + ByteBuf connect = Commands.newConnect(authMethodName, "pass.fail", "test", "localhost", + "pass.pass", "pass.pass", authMethodName); + channel.writeInbound(connect); + Object connectResponse = getResponse(); + assertTrue(connectResponse instanceof CommandConnected); + + ByteBuf clientCommand = Commands.newAddPartitionToTxn(89L, 1L, 12L, + List.of("tenant/ns/topic1")); + channel.writeInbound(clientCommand); + CommandAddPartitionToTxnResponse response = (CommandAddPartitionToTxnResponse) getResponse(); + + assertEquals(response.getError(), ServerError.TransactionNotFound); + verify(txnStore, never()).addProducedPartitionToTxn(any(TxnID.class), any()); + + channel.finish(); + } + @Test(timeOut = 30000) public void sendAddSubscriptionToTxnResponse() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); From dd054080490f01e9ba9694756b335e3e10a637d5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 6 Apr 2023 13:00:49 +0300 Subject: [PATCH 065/280] [fix][proxy] Fix connection read timeout handling in Pulsar Proxy (#20014) --- .../proxy/server/DirectProxyHandler.java | 6 ++- .../pulsar/proxy/server/ProxyConnection.java | 3 ++ .../proxy/server/ProxyReadTimeoutHandler.java | 46 ------------------- .../server/ServiceChannelInitializer.java | 3 +- 4 files changed, 10 insertions(+), 48 deletions(-) delete mode 100644 pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 23c7faa2d4bb7..cb93f885a1d68 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -43,6 +43,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslProvider; +import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.CharsetUtil; import java.net.InetSocketAddress; import java.util.Arrays; @@ -205,7 +206,7 @@ protected void initChannel(SocketChannel ch) { int brokerProxyReadTimeoutMs = service.getConfiguration().getBrokerProxyReadTimeoutMs(); if (brokerProxyReadTimeoutMs > 0) { ch.pipeline().addLast("readTimeoutHandler", - new ProxyReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS)); + new ReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS)); } ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder( service.getConfiguration().getMaxMessageSize() + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0, @@ -362,6 +363,9 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce if (service.proxyZeroCopyModeEnabled && service.proxyLogLevel == 0) { if (!isTlsOutboundChannel && !DirectProxyHandler.this.proxyConnection.isTlsInboundChannel) { + if (ctx.pipeline().get("readTimeoutHandler") != null) { + ctx.pipeline().remove("readTimeoutHandler"); + } ProxyConnection.spliceNIC2NIC((EpollSocketChannel) ctx.channel(), (EpollSocketChannel) inboundChannel, ProxyConnection.SPLICE_BYTES) .addListener(future -> { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index b7f5534ee81fb..9530389b524b3 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -263,6 +263,9 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce if (service.proxyZeroCopyModeEnabled && service.proxyLogLevel == 0) { if (!directProxyHandler.isTlsOutboundChannel && !isTlsInboundChannel) { + if (ctx.pipeline().get("readTimeoutHandler") != null) { + ctx.pipeline().remove("readTimeoutHandler"); + } spliceNIC2NIC((EpollSocketChannel) ctx.channel(), (EpollSocketChannel) directProxyHandler.outboundChannel, SPLICE_BYTES) .addListener(future -> { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java deleted file mode 100644 index df650a0ca16c1..0000000000000 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.proxy.server; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.timeout.IdleStateHandler; -import io.netty.handler.timeout.ReadTimeoutHandler; -import java.lang.reflect.Field; -import java.util.concurrent.TimeUnit; - -public class ProxyReadTimeoutHandler extends ReadTimeoutHandler { - - private final Field readingField; - - public ProxyReadTimeoutHandler(long timeout, TimeUnit unit) { - super(timeout, unit); - try { - this.readingField = IdleStateHandler.class.getDeclaredField("reading"); - this.readingField.setAccessible(true); - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException("Exception caused while get 'reading' field", e); - } - } - - @Override - public void channelReadComplete(ChannelHandlerContext ctx) throws Exception { - this.readingField.setBoolean(this, true); - super.channelReadComplete(ctx); - } -} diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index 7e7ba31b79fe3..19f4002ad52ce 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -25,6 +25,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslProvider; +import io.netty.handler.timeout.ReadTimeoutHandler; import java.util.concurrent.TimeUnit; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.OptionalProxyProtocolDecoder; @@ -106,7 +107,7 @@ protected void initChannel(SocketChannel ch) throws Exception { } if (brokerProxyReadTimeoutMs > 0) { ch.pipeline().addLast("readTimeoutHandler", - new ProxyReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS)); + new ReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS)); } if (proxyService.getConfiguration().isHaProxyProtocolEnabled()) { ch.pipeline().addLast(OptionalProxyProtocolDecoder.NAME, new OptionalProxyProtocolDecoder()); From 067e3c038edf406b27f8f91847f688f559a2ba25 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 6 Apr 2023 08:45:58 -0500 Subject: [PATCH 066/280] [feat][fn] PIP-257: Support mounting k8s ServiceAccount for OIDC auth (#19888) PIP: #19771 ### Motivation In order to make OIDC work with functions, we must give them a way to authenticate with the broker using tokens that are able to be validated by an using an Authorization Server. This PR introduces the `KubernetesServiceAccountAuthProvider`. ### Modifications * Create an `KubernetesServiceAccountAuthProvider` implementation. It adds a service account token volume projection as defined in the k8s docs [here](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-token-volume-projection). The implementation provides a way to specify the expiration time that the token will receive. * Instead of creating a secret with the broker's trusted `ca.crt` in it, this new `KubernetesServiceAccountAuthProvider` expects a secret to already exist with the `ca.crt`. The major advantage for this implementation is that when the `ca.crt` is rotated, we can refresh it (assuming the client is configured to observe the updated file). * Add support for specifying the token's expiration and audience. * One point of divergence from the `KubernetesSecretsTokenAuthProvider` implementation is that I did not provide a way for functions to authenticate as the anonymous role. It seems like a stretch that functions would use such authentication because it will not be multi-tenant. However, if that is a concern, we can add the support. * The feature will be configured with the following yaml: ```yaml functionRuntimeFactoryConfigs: kubernetesFunctionAuthProviderConfig: brokerClientTrustCertsSecretName: "secret-name" serviceAccountTokenExpirationSeconds: 3600 serviceAccountTokenAudience: "pulsar-cluster-audience" ``` ### Verifying this change I verified the correctness of the code with unit tests. I'll verify the integration with k8s once we've determined this PR's design is correct. ### Does this pull request potentially affect one of the following parts: This adds new configuration options to the function worker. ### Documentation - [x] `doc-required` ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/36 --- .../auth/KubernetesFunctionAuthProvider.java | 12 + ...rnetesServiceAccountTokenAuthProvider.java | 208 ++++++++++++++++++ .../kubernetes/KubernetesRuntimeFactory.java | 2 +- .../KubernetesRuntimeFactoryConfig.java | 6 + ...esServiceAccountTokenAuthProviderTest.java | 128 +++++++++++ 5 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java create mode 100644 pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java index 29365424f547f..3d05d519552c9 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java @@ -20,6 +20,7 @@ import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1StatefulSet; +import java.util.Map; import java.util.Optional; import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.proto.Function; @@ -31,6 +32,11 @@ public interface KubernetesFunctionAuthProvider extends FunctionAuthProvider { void initialize(CoreV1Api coreClient); + /** + * @deprecated use + * {@link #initialize(CoreV1Api, byte[], java.util.function.Function, Map)} + */ + @Deprecated(since = "3.0.0") default void initialize(CoreV1Api coreClient, byte[] caBytes, java.util.function.Function namespaceCustomizerFunc) { setCaBytes(caBytes); @@ -38,6 +44,12 @@ default void initialize(CoreV1Api coreClient, byte[] caBytes, initialize(coreClient); } + default void initialize(CoreV1Api coreClient, byte[] caBytes, + java.util.function.Function namespaceCustomizerFunc, + Map config) { + initialize(coreClient, caBytes, namespaceCustomizerFunc); + } + default void setCaBytes(byte[] caBytes) { } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java new file mode 100644 index 0000000000000..06fadec42d42f --- /dev/null +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.auth; + +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Container; +import io.kubernetes.client.openapi.models.V1KeyToPath; +import io.kubernetes.client.openapi.models.V1PodSpec; +import io.kubernetes.client.openapi.models.V1ProjectedVolumeSource; +import io.kubernetes.client.openapi.models.V1SecretVolumeSource; +import io.kubernetes.client.openapi.models.V1ServiceAccountTokenProjection; +import io.kubernetes.client.openapi.models.V1StatefulSet; +import io.kubernetes.client.openapi.models.V1Volume; +import io.kubernetes.client.openapi.models.V1VolumeMount; +import io.kubernetes.client.openapi.models.V1VolumeProjection; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Optional; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.functions.instance.AuthenticationConfig; +import org.apache.pulsar.functions.proto.Function; +import org.eclipse.jetty.util.StringUtil; + +/** + * Kubernetes Function Authentication Provider that adds Service Account Token Projection to a function pod's container + * definition. This token can be used to authenticate the function instance with the broker and the function worker via + * OpenId Connect when each server is configured to trust the kubernetes issuer. See docs for additional details. + * Relevant settings: + *

+ * brokerClientTrustCertsSecretName: The Kubernetes secret containing the broker's trust certs. If it is not set, + * the function will not use a custom trust store. The secret must already exist in each function's target + * namespace. The secret must contain a key named `ca.crt` with the trust certs. Only the ca.crt will be mounted. + *

+ *

+ * serviceAccountTokenExpirationSeconds: The expiration for the token created by the + * {@link KubernetesServiceAccountTokenAuthProvider}. The default value is 3600 seconds. + *

+ *

+ * serviceAccountTokenAudience: The audience for the token created by the + * {@link KubernetesServiceAccountTokenAuthProvider}. + *

+ * Note: the pod inherits the namespace's default service account. + */ +public class KubernetesServiceAccountTokenAuthProvider implements KubernetesFunctionAuthProvider { + + private static final String BROKER_CLIENT_TRUST_CERTS_SECRET_NAME = "brokerClientTrustCertsSecretName"; + private static final String SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS = "serviceAccountTokenExpirationSeconds"; + private static final String SERVICE_ACCOUNT_TOKEN_AUDIENCE = "serviceAccountTokenAudience"; + + private static final String SERVICE_ACCOUNT_VOLUME_NAME = "service-account-token"; + private static final String TRUST_CERT_VOLUME_NAME = "ca-cert"; + private static final String DEFAULT_MOUNT_DIR = "/etc/auth"; + private static final String FUNCTION_AUTH_TOKEN = "token"; + private static final String FUNCTION_CA_CERT = "ca.crt"; + private static final String DEFAULT_CERT_PATH = DEFAULT_MOUNT_DIR + "/" + FUNCTION_CA_CERT; + private String brokerTrustCertsSecretName; + private long serviceAccountTokenExpirationSeconds; + private String serviceAccountTokenAudience; + + @Override + public void initialize(CoreV1Api coreClient, byte[] caBytes, + java.util.function.Function namespaceCustomizerFunc, + Map config) { + setNamespaceProviderFunc(namespaceCustomizerFunc); + Object certSecretName = config.get(BROKER_CLIENT_TRUST_CERTS_SECRET_NAME); + if (certSecretName instanceof String) { + brokerTrustCertsSecretName = (String) certSecretName; + } else if (certSecretName != null) { + // Throw exception because user set this configuration, but it isn't valid. + throw new IllegalArgumentException("Invalid value for " + BROKER_CLIENT_TRUST_CERTS_SECRET_NAME + + ". Expected a string."); + } + Object tokenExpirationSeconds = config.get(SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS); + if (tokenExpirationSeconds instanceof Long) { + serviceAccountTokenExpirationSeconds = (Long) tokenExpirationSeconds; + } else if (tokenExpirationSeconds instanceof String) { + try { + serviceAccountTokenExpirationSeconds = Long.parseLong((String) tokenExpirationSeconds); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid value for " + SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS + + ". Expected a long."); + } + } else if (tokenExpirationSeconds != null) { + // Throw exception because user set this configuration, but it isn't valid. + throw new IllegalArgumentException("Invalid value for " + SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS + + ". Expected a long."); + } + Object tokenAudience = config.get(SERVICE_ACCOUNT_TOKEN_AUDIENCE); + if (tokenAudience instanceof String) { + serviceAccountTokenAudience = (String) tokenAudience; + } else if (tokenAudience != null) { + throw new IllegalArgumentException("Invalid value for " + SERVICE_ACCOUNT_TOKEN_AUDIENCE + + ". Expected a string."); + } + } + + @Override + public void configureAuthenticationConfig(AuthenticationConfig authConfig, + Optional functionAuthData) { + authConfig.setClientAuthenticationPlugin(AuthenticationToken.class.getName()); + authConfig.setClientAuthenticationParameters(Paths.get(DEFAULT_MOUNT_DIR, FUNCTION_AUTH_TOKEN) + .toUri().toString()); + if (StringUtil.isNotBlank(brokerTrustCertsSecretName)) { + authConfig.setTlsTrustCertsFilePath(DEFAULT_CERT_PATH); + } + } + + /** + * No need to cache anything. Kubernetes generates the token used for authentication. + */ + @Override + public Optional cacheAuthData(Function.FunctionDetails funcDetails, + AuthenticationDataSource authenticationDataSource) + throws Exception { + return Optional.empty(); + } + + /** + * No need to update anything. Kubernetes updates the token used for authentication. + */ + @Override + public Optional updateAuthData(Function.FunctionDetails funcDetails, + Optional existingFunctionAuthData, + AuthenticationDataSource authenticationDataSource) + throws Exception { + return Optional.empty(); + } + + /** + * No need to clean up anything. Kubernetes cleans up the secret when the pod is deleted. + */ + @Override + public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional functionAuthData) + throws Exception { + + } + + @Override + public void initialize(CoreV1Api coreClient) { + } + + @Override + public void configureAuthDataStatefulSet(V1StatefulSet statefulSet, Optional functionAuthData) { + V1PodSpec podSpec = statefulSet.getSpec().getTemplate().getSpec(); + // configure pod mount secret with auth token + if (StringUtil.isNotBlank(brokerTrustCertsSecretName)) { + podSpec.addVolumesItem(createTrustCertVolume()); + } + podSpec.addVolumesItem(createServiceAccountVolume()); + podSpec.getContainers().forEach(this::addVolumeMountsToContainer); + } + + private V1Volume createServiceAccountVolume() { + V1ProjectedVolumeSource projectedVolumeSource = new V1ProjectedVolumeSource(); + V1VolumeProjection volumeProjection = new V1VolumeProjection(); + volumeProjection.serviceAccountToken( + new V1ServiceAccountTokenProjection() + .audience(serviceAccountTokenAudience) + .expirationSeconds(serviceAccountTokenExpirationSeconds) + .path(FUNCTION_AUTH_TOKEN)); + projectedVolumeSource.addSourcesItem(volumeProjection); + return new V1Volume() + .name(SERVICE_ACCOUNT_VOLUME_NAME) + .projected(projectedVolumeSource); + } + + private V1Volume createTrustCertVolume() { + return new V1Volume() + .name(TRUST_CERT_VOLUME_NAME) + .secret(new V1SecretVolumeSource() + .secretName(brokerTrustCertsSecretName) + .addItemsItem(new V1KeyToPath() + .key(FUNCTION_CA_CERT) + .path(FUNCTION_CA_CERT))); + } + + private void addVolumeMountsToContainer(V1Container container) { + container.addVolumeMountsItem( + new V1VolumeMount() + .name(SERVICE_ACCOUNT_VOLUME_NAME) + .mountPath(DEFAULT_MOUNT_DIR) + .readOnly(true)); + if (StringUtil.isNotBlank(brokerTrustCertsSecretName)) { + container.addVolumeMountsItem( + new V1VolumeMount() + .name(TRUST_CERT_VOLUME_NAME) + .mountPath(DEFAULT_MOUNT_DIR) + .readOnly(true)); + } + } +} diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 692884a303d58..895304138a530 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -252,7 +252,7 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic kubernetesFunctionAuthProvider.initialize(coreClient, serverCaBytes, (funcDetails) -> getRuntimeCustomizer() .map((customizer) -> customizer.customizeNamespace(funcDetails, jobNamespace)) - .orElse(jobNamespace)); + .orElse(jobNamespace), factoryConfig.getKubernetesFunctionAuthProviderConfig()); this.authProvider = Optional.of(kubernetesFunctionAuthProvider); } } else { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java index 5cbca7b65bfd7..43cdc035076c3 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.runtime.kubernetes; +import java.util.HashMap; import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; @@ -169,4 +170,9 @@ public class KubernetesRuntimeFactoryConfig { ) protected int gracePeriodSeconds = 5; + @FieldContext( + doc = "A map of custom configurations passed to implementations of the KubernetesFunctionAuthProvider" + + " interface." + ) + private Map kubernetesFunctionAuthProviderConfig = new HashMap<>(); } diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java new file mode 100644 index 0000000000000..2e8cf75151044 --- /dev/null +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.auth; + +import io.kubernetes.client.openapi.models.V1Container; +import io.kubernetes.client.openapi.models.V1PodSpec; +import io.kubernetes.client.openapi.models.V1PodTemplateSpec; +import io.kubernetes.client.openapi.models.V1ServiceAccountTokenProjection; +import io.kubernetes.client.openapi.models.V1StatefulSet; +import io.kubernetes.client.openapi.models.V1StatefulSetSpec; +import io.kubernetes.client.openapi.models.V1Volume; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.functions.instance.AuthenticationConfig; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class KubernetesServiceAccountTokenAuthProviderTest { + @Test + public void testConfigureAuthDataStatefulSet() { + HashMap config = new HashMap<>(); + config.put("brokerClientTrustCertsSecretName", "my-secret"); + config.put("serviceAccountTokenExpirationSeconds", "600"); + config.put("serviceAccountTokenAudience", "my-audience"); + KubernetesServiceAccountTokenAuthProvider provider = new KubernetesServiceAccountTokenAuthProvider(); + provider.initialize(null, null, (fd) -> "default", config); + + // Create a stateful set with a container + V1StatefulSet statefulSet = new V1StatefulSet(); + statefulSet.setSpec( + new V1StatefulSetSpec().template( + new V1PodTemplateSpec().spec( + new V1PodSpec().containers( + Collections.singletonList(new V1Container()))))); + provider.configureAuthDataStatefulSet(statefulSet, Optional.empty()); + + List volumes = statefulSet.getSpec().getTemplate().getSpec().getVolumes(); + Assert.assertEquals(volumes.size(), 2); + + Assert.assertEquals(volumes.get(0).getName(), "ca-cert"); + Assert.assertEquals(volumes.get(0).getSecret().getSecretName(), "my-secret"); + Assert.assertEquals(volumes.get(0).getSecret().getItems().size(), 1); + Assert.assertEquals(volumes.get(0).getSecret().getItems().get(0).getKey(), "ca.crt"); + Assert.assertEquals(volumes.get(0).getSecret().getItems().get(0).getPath(), "ca.crt"); + + + Assert.assertEquals(volumes.get(1).getName(), "service-account-token"); + Assert.assertEquals(volumes.get(1).getProjected().getSources().size(), 1); + V1ServiceAccountTokenProjection tokenProjection = + volumes.get(1).getProjected().getSources().get(0).getServiceAccountToken(); + Assert.assertEquals(tokenProjection.getExpirationSeconds(), 600); + Assert.assertEquals(tokenProjection.getAudience(), "my-audience"); + + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().size(), 1); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().size(), 2); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getName(), "service-account-token"); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getMountPath(), "/etc/auth"); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(1).getName(), "ca-cert"); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(1).getMountPath(), "/etc/auth"); + } + + @Test + public void testConfigureAuthDataStatefulSetNoCa() { + + HashMap config = new HashMap<>(); + config.put("serviceAccountTokenExpirationSeconds", "600"); + config.put("serviceAccountTokenAudience", "pulsar-cluster"); + KubernetesServiceAccountTokenAuthProvider provider = new KubernetesServiceAccountTokenAuthProvider(); + provider.initialize(null, null, (fd) -> "default", config); + + // Create a stateful set with a container + V1StatefulSet statefulSet = new V1StatefulSet(); + statefulSet.setSpec( + new V1StatefulSetSpec().template( + new V1PodTemplateSpec().spec( + new V1PodSpec().containers( + Collections.singletonList(new V1Container()))))); + provider.configureAuthDataStatefulSet(statefulSet, Optional.empty()); + + List volumes = statefulSet.getSpec().getTemplate().getSpec().getVolumes(); + Assert.assertEquals(volumes.size(), 1); + + Assert.assertEquals(volumes.get(0).getName(), "service-account-token"); + Assert.assertEquals(volumes.get(0).getProjected().getSources().size(), 1); + V1ServiceAccountTokenProjection tokenProjection = + volumes.get(0).getProjected().getSources().get(0).getServiceAccountToken(); + Assert.assertEquals(tokenProjection.getExpirationSeconds(), 600); + Assert.assertEquals(tokenProjection.getAudience(), "pulsar-cluster"); + + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().size(), 1); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().size(), 1); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getName(), "service-account-token"); + Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getMountPath(), "/etc/auth"); + } + + @Test + public void configureAuthenticationConfig() { + HashMap config = new HashMap<>(); + config.put("brokerClientTrustCertsSecretName", "my-secret"); + KubernetesServiceAccountTokenAuthProvider provider = new KubernetesServiceAccountTokenAuthProvider(); + provider.initialize(null, null, (fd) -> "default", config); + AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build(); + provider.configureAuthenticationConfig(authenticationConfig, Optional.empty()); + + Assert.assertEquals(authenticationConfig.getClientAuthenticationPlugin(), AuthenticationToken.class.getName()); + Assert.assertEquals(authenticationConfig.getClientAuthenticationParameters(), "file:///etc/auth/token"); + Assert.assertEquals(authenticationConfig.getTlsTrustCertsFilePath(), "/etc/auth/ca.crt"); + } +} From 42a6969efc8fdc16f2d883d3c4f2865bdb460091 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Thu, 6 Apr 2023 21:50:45 +0800 Subject: [PATCH 067/280] [fix][broker] Return if AbstractDispatcherSingleActiveConsumer closed (#19934) --- ...AbstractDispatcherSingleActiveConsumer.java | 1 + ...nPersistentDispatcherMultipleConsumers.java | 2 +- ...ntStickyKeyDispatcherMultipleConsumers.java | 9 +++++++++ ...ntStickyKeyDispatcherMultipleConsumers.java | 11 +++++++++++ ...rsistentDispatcherFailoverConsumerTest.java | 14 ++++++++++++++ ...ickyKeyDispatcherMultipleConsumersTest.java | 18 +++++++++++++++--- ...ickyKeyDispatcherMultipleConsumersTest.java | 11 +++++++++++ 7 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java index 2e22a80250cc3..1b6df5f809327 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java @@ -161,6 +161,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce if (IS_CLOSED_UPDATER.get(this) == TRUE) { log.warn("[{}] Dispatcher is already closed. Closing consumer {}", this.topicName, consumer); consumer.disconnect(); + return; } if (subscriptionType == SubType.Exclusive && !consumers.isEmpty()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java index f76fc09ba2ebe..d679106094178 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java @@ -47,7 +47,7 @@ public class NonPersistentDispatcherMultipleConsumers extends AbstractDispatcher protected final Subscription subscription; private CompletableFuture closeFuture = null; - private final String name; + protected final String name; protected final Rate msgDrop; protected static final AtomicIntegerFieldUpdater TOTAL_AVAILABLE_PERMITS_UPDATER = AtomicIntegerFieldUpdater diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java index f26e1399be71d..23814bfdbe22e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java @@ -39,6 +39,8 @@ import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; import org.apache.pulsar.common.protocol.Commands; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class NonPersistentStickyKeyDispatcherMultipleConsumers extends NonPersistentDispatcherMultipleConsumers { @@ -84,6 +86,11 @@ public NonPersistentStickyKeyDispatcherMultipleConsumers(NonPersistentTopic topi @Override public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + if (IS_CLOSED_UPDATER.get(this) == TRUE) { + log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer); + consumer.disconnect(); + return; + } super.addConsumer(consumer); try { selector.addConsumer(consumer); @@ -168,4 +175,6 @@ public KeySharedMode getKeySharedMode() { public boolean hasSameKeySharedPolicy(KeySharedMeta ksm) { return (ksm.getKeySharedMode() == this.keySharedMode); } + + private static final Logger log = LoggerFactory.getLogger(NonPersistentStickyKeyDispatcherMultipleConsumers.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 6180f29c96d1d..0c29421edd4ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.persistent; import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; +import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; import java.util.Collections; @@ -100,8 +101,18 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi } } + @VisibleForTesting + public StickyKeyConsumerSelector getSelector() { + return selector; + } + @Override public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + if (IS_CLOSED_UPDATER.get(this) == TRUE) { + log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer); + consumer.disconnect(); + return; + } super.addConsumer(consumer); try { selector.addConsumer(consumer); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index 0dafe4bca3e2c..631b702dfbd08 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -233,6 +233,20 @@ private void verifyActiveConsumerChange(CommandActiveConsumerChange change, assertEquals(isActive, change.isIsActive()); } + @Test(timeOut = 10000) + public void testAddConsumerWhenClosed() throws Exception { + PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, pulsarTestContext.getBrokerService()); + PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false); + PersistentDispatcherSingleActiveConsumer pdfc = new PersistentDispatcherSingleActiveConsumer(cursorMock, + SubType.Failover, 0, topic, sub); + pdfc.close().get(); + + Consumer consumer = mock(Consumer.class); + pdfc.addConsumer(consumer); + verify(consumer, times(1)).disconnect(); + assertEquals(0, pdfc.consumers.size()); + } + @Test public void testConsumerGroupChangesWithOldNewConsumers() throws Exception { PersistentTopic topic = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java index 64807c73852e3..b2638d53ab1c3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -47,6 +48,7 @@ import org.apache.pulsar.broker.service.EntryBatchSizes; import org.apache.pulsar.broker.service.HashRangeAutoSplitStickyKeyConsumerSelector; import org.apache.pulsar.broker.service.RedeliveryTracker; +import org.apache.pulsar.broker.service.StickyKeyConsumerSelector; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.protocol.Commands; @@ -62,6 +64,7 @@ public class NonPersistentStickyKeyDispatcherMultipleConsumersTest { private ServiceConfiguration configMock; private NonPersistentStickyKeyDispatcherMultipleConsumers nonpersistentDispatcher; + private StickyKeyConsumerSelector selector; final String topicName = "non-persistent://public/default/testTopic"; @@ -88,10 +91,19 @@ public void setup() throws Exception { doReturn(topicPolicies).when(topicMock).getHierarchyTopicPolicies(); subscriptionMock = mock(NonPersistentSubscription.class); - + selector = new HashRangeAutoSplitStickyKeyConsumerSelector(); nonpersistentDispatcher = new NonPersistentStickyKeyDispatcherMultipleConsumers( - topicMock, subscriptionMock, - new HashRangeAutoSplitStickyKeyConsumerSelector()); + topicMock, subscriptionMock, selector); + } + + @Test(timeOut = 10000) + public void testAddConsumerWhenClosed() throws Exception { + nonpersistentDispatcher.close().get(); + Consumer consumer = mock(Consumer.class); + nonpersistentDispatcher.addConsumer(consumer); + verify(consumer, times(1)).disconnect(); + assertEquals(0, nonpersistentDispatcher.getConsumers().size()); + assertTrue(selector.getConsumerKeyHashRanges().isEmpty()); } @Test(timeOut = 10000) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index 1c23afd957a64..48a4bfc923608 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -35,6 +35,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -159,6 +160,16 @@ public void cleanup() { } } + @Test(timeOut = 10000) + public void testAddConsumerWhenClosed() throws Exception { + persistentDispatcher.close().get(); + Consumer consumer = mock(Consumer.class); + persistentDispatcher.addConsumer(consumer); + verify(consumer, times(1)).disconnect(); + assertEquals(0, persistentDispatcher.getConsumers().size()); + assertTrue(persistentDispatcher.getSelector().getConsumerKeyHashRanges().isEmpty()); + } + @Test public void testSendMarkerMessage() { try { From 8b0a0a388ad9b15defcb487fbd9ac153d7a06888 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Fri, 7 Apr 2023 13:50:44 +0800 Subject: [PATCH 068/280] [improve][ml] Add Read cache misses metric for ledger (#20001) --- .../mledger/ManagedLedgerMXBean.java | 5 + .../mledger/impl/ManagedLedgerMBeanImpl.java | 11 ++ .../impl/cache/EntryCacheDisabled.java | 2 + .../impl/cache/RangeEntryCacheImpl.java | 2 + .../stats/metrics/ManagedLedgerMetrics.java | 2 + .../prometheus/AggregatedBrokerStats.java | 3 + .../prometheus/AggregatedNamespaceStats.java | 1 + .../stats/prometheus/ManagedLedgerStats.java | 2 + .../prometheus/NamespaceStatsAggregator.java | 5 + .../broker/stats/prometheus/TopicStats.java | 3 + .../broker/stats/PrometheusMetricsTest.java | 118 ++++++++++++++++++ 11 files changed, 154 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java index 94c2f61e00afe..50a3ffb157961 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java @@ -100,6 +100,11 @@ public interface ManagedLedgerMXBean { */ long getReadEntriesErrors(); + /** + * @return the number of readEntries requests that cache miss Rate + */ + double getReadEntriesOpsCacheMissesRate(); + // Entry size statistics double getEntrySizeAverage(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java index dad101c9b72d1..cb3d72cc5972f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java @@ -39,6 +39,7 @@ public class ManagedLedgerMBeanImpl implements ManagedLedgerMXBean { private final Rate addEntryOpsFailed = new Rate(); private final Rate readEntriesOps = new Rate(); private final Rate readEntriesOpsFailed = new Rate(); + private final Rate readEntriesOpsCacheMisses = new Rate(); private final Rate markDeleteOps = new Rate(); private final LongAdder dataLedgerOpenOp = new LongAdder(); @@ -72,6 +73,7 @@ public void refreshStats(long period, TimeUnit unit) { addEntryOpsFailed.calculateRate(seconds); readEntriesOps.calculateRate(seconds); readEntriesOpsFailed.calculateRate(seconds); + readEntriesOpsCacheMisses.calculateRate(seconds); markDeleteOps.calculateRate(seconds); addEntryLatencyStatsUsec.refresh(); @@ -98,6 +100,10 @@ public void recordReadEntriesError() { readEntriesOpsFailed.recordEvent(); } + public void recordReadEntriesOpsCacheMisses() { + readEntriesOpsCacheMisses.recordEvent(); + } + public void addAddEntryLatencySample(long latency, TimeUnit unit) { addEntryLatencyStatsUsec.addValue(unit.toMicros(latency)); } @@ -228,6 +234,11 @@ public long getReadEntriesErrors() { return readEntriesOpsFailed.getCount(); } + @Override + public double getReadEntriesOpsCacheMissesRate() { + return readEntriesOpsCacheMisses.getRate(); + } + @Override public double getMarkDeleteRate() { return markDeleteOps.getRate(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java index 1c5563b38b120..d2add99b701ac 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java @@ -93,6 +93,7 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole } finally { ledgerEntries.close(); } + ml.getMbean().recordReadEntriesOpsCacheMisses(); ml.getFactory().getMbean().recordCacheMiss(entries.size(), totalSize); ml.getMbean().addReadEntriesSample(entries.size(), totalSize); @@ -120,6 +121,7 @@ public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks. LedgerEntry ledgerEntry = iterator.next(); EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor); + ml.getMbean().recordReadEntriesOpsCacheMisses(); ml.getFactory().getMbean().recordCacheMiss(1, returnEntry.getLength()); ml.getMbean().addReadEntriesSample(1, returnEntry.getLength()); callback.readEntryComplete(returnEntry, ctx); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java index 28a2f00cf683c..7747f9bcd93b6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java @@ -256,6 +256,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt LedgerEntry ledgerEntry = iterator.next(); EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor); + ml.getMbean().recordReadEntriesOpsCacheMisses(); manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength()); ml.getMbean().addReadEntriesSample(1, returnEntry.getLength()); callback.readEntryComplete(returnEntry, ctx); @@ -449,6 +450,7 @@ CompletableFuture> readFromStorage(ReadHandle lh, } } + ml.getMbean().recordReadEntriesOpsCacheMisses(); manager.mlFactoryMBean.recordCacheMiss(entriesToReturn.size(), totalSize); ml.getMbean().addReadEntriesSample(entriesToReturn.size(), totalSize); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java index 6d82ec682ea1a..36004bc1281bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java @@ -108,6 +108,8 @@ private List aggregate(Map> ledgersByD (double) lStats.getReadEntriesErrors()); populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesRate", lStats.getReadEntriesRate()); + populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesOpsCacheMissesRate", + lStats.getReadEntriesOpsCacheMissesRate()); populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesSucceeded", (double) lStats.getReadEntriesSucceeded()); populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_StoredMessagesSize", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java index 00c6cecdbfca1..715231d3c6ee1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java @@ -31,6 +31,7 @@ public class AggregatedBrokerStats { public long storageLogicalSize; public double storageWriteRate; public double storageReadRate; + public double storageReadCacheMissesRate; public long msgBacklog; void updateStats(TopicStats stats) { @@ -46,6 +47,7 @@ void updateStats(TopicStats stats) { storageLogicalSize += stats.managedLedgerStats.storageLogicalSize; storageWriteRate += stats.managedLedgerStats.storageWriteRate; storageReadRate += stats.managedLedgerStats.storageReadRate; + storageReadCacheMissesRate += stats.managedLedgerStats.storageReadCacheMissesRate; msgBacklog += stats.msgBacklog; } @@ -62,6 +64,7 @@ public void reset() { storageLogicalSize = 0; storageWriteRate = 0; storageReadRate = 0; + storageReadCacheMissesRate = 0; msgBacklog = 0; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index ea77bd69302a0..9fe5588044d2f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -107,6 +107,7 @@ void updateStats(TopicStats stats) { managedLedgerStats.storageWriteRate += stats.managedLedgerStats.storageWriteRate; managedLedgerStats.storageReadRate += stats.managedLedgerStats.storageReadRate; + managedLedgerStats.storageReadCacheMissesRate += stats.managedLedgerStats.storageReadCacheMissesRate; msgBacklog += stats.msgBacklog; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java index db807b51df49f..659f4441adbb7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java @@ -34,11 +34,13 @@ public class ManagedLedgerStats { double storageWriteRate; double storageReadRate; + double storageReadCacheMissesRate; public void reset() { storageSize = 0; storageWriteRate = 0; storageReadRate = 0; + storageReadCacheMissesRate = 0; backlogSize = 0; offloadedStorageUsed = 0; storageLogicalSize = 0; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 32fb06ea3ce8c..4e72fa0d72b16 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -190,6 +190,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include stats.managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate(); stats.managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate(); + stats.managedLedgerStats.storageReadCacheMissesRate = mlStats.getReadEntriesOpsCacheMissesRate(); } TopicStatsImpl tStatus = topic.getStats(getPreciseBacklog, subscriptionBacklogSize, false); stats.msgInCounter = tStatus.msgInCounter; @@ -331,6 +332,8 @@ private static void printBrokerStats(PrometheusMetricStreams stream, String clus writeMetric(stream, "pulsar_broker_storage_logical_size", brokerStats.storageLogicalSize, cluster); writeMetric(stream, "pulsar_broker_storage_write_rate", brokerStats.storageWriteRate, cluster); writeMetric(stream, "pulsar_broker_storage_read_rate", brokerStats.storageReadRate, cluster); + writeMetric(stream, "pulsar_broker_storage_read_cache_misses_rate", + brokerStats.storageReadCacheMissesRate, cluster); writeMetric(stream, "pulsar_broker_msg_backlog", brokerStats.msgBacklog, cluster); } @@ -376,6 +379,8 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat cluster, namespace); writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, cluster, namespace); + writeMetric(stream, "pulsar_storage_read_cache_misses_rate", + stats.managedLedgerStats.storageReadCacheMissesRate, cluster, namespace); writeMetric(stream, "pulsar_subscription_delayed", stats.msgDelayed, cluster, namespace); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index 3a2563a87587b..dda03e3e59dd4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -152,6 +152,9 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st cluster, namespace, topic, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_read_cache_misses_rate", + stats.managedLedgerStats.storageReadCacheMissesRate, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 13e67762ace6f..04b0a5509b68c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -58,6 +58,7 @@ import javax.crypto.SecretKey; import javax.naming.AuthenticationException; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; @@ -84,6 +85,7 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker") @@ -540,6 +542,122 @@ public void testPerTopicStatsReconnect() throws Exception { assertEquals(cm.get(0).tags.get("subscription"), "test"); } + @DataProvider(name = "cacheEnable") + public static Object[][] cacheEnable() { + return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; + } + + @Test(dataProvider = "cacheEnable") + public void testStorageReadCacheMissesRate(boolean cacheEnable) throws Exception { + cleanup(); + conf.setManagedLedgerStatsPeriodSeconds(Integer.MAX_VALUE); + conf.setManagedLedgerCacheEvictionTimeThresholdMillis(Long.MAX_VALUE); + conf.setCacheEvictionByMarkDeletedPosition(true); + if (cacheEnable) { + conf.setManagedLedgerCacheSizeMB(1); + } else { + conf.setManagedLedgerCacheSizeMB(0); + } + setup(); + String ns = "prop/ns-abc1"; + admin.namespaces().createNamespace(ns); + String topic = "persistent://" + ns + "/testStorageReadCacheMissesRate" + UUID.randomUUID(); + + @Cleanup + Producer producer = pulsarClient.newProducer().enableBatching(false).topic(topic).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("test") + .subscribe(); + byte[] msg = new byte[2 * 1024 * 1024]; + new Random().nextBytes(msg); + producer.send(msg); + consumer.receive(); + // when cacheEnable, the second msg will read cache miss + producer.send(msg); + consumer.receive(); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get().get(); + ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) persistentTopic.getManagedLedger()); + managedLedger.getMbean().refreshStats(1, TimeUnit.SECONDS); + + // includeTopicMetric true + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + String metricsStr = statsOut.toString(); + Multimap metrics = parseMetrics(metricsStr); + + metrics.entries().forEach(e -> System.out.println(e.getKey() + ": " + e.getValue())); + + List cm = (List) metrics.get("pulsar_storage_read_cache_misses_rate"); + assertEquals(cm.size(), 1); + if (cacheEnable) { + assertEquals(cm.get(0).value, 1.0); + } else { + assertEquals(cm.get(0).value, 2.0); + } + + assertEquals(cm.get(0).tags.get("topic"), topic); + assertEquals(cm.get(0).tags.get("namespace"), ns); + assertEquals(cm.get(0).tags.get("cluster"), "test"); + + List brokerMetric = (List) metrics.get("pulsar_broker_storage_read_cache_misses_rate"); + assertEquals(brokerMetric.size(), 1); + if (cacheEnable) { + assertEquals(brokerMetric.get(0).value, 1.0); + } else { + assertEquals(brokerMetric.get(0).value, 2.0); + } + + assertEquals(brokerMetric.get(0).tags.get("cluster"), "test"); + assertNull(brokerMetric.get(0).tags.get("namespace")); + assertNull(brokerMetric.get(0).tags.get("topic")); + + // includeTopicMetric false + ByteArrayOutputStream statsOut2 = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut2); + String metricsStr2 = statsOut2.toString(); + Multimap metrics2 = parseMetrics(metricsStr2); + + metrics2.entries().forEach(e -> System.out.println(e.getKey() + ": " + e.getValue())); + + List cm2 = (List) metrics2.get("pulsar_storage_read_cache_misses_rate"); + assertEquals(cm2.size(), 1); + if (cacheEnable) { + assertEquals(cm2.get(0).value, 1.0); + } else { + assertEquals(cm2.get(0).value, 2.0); + } + + assertNull(cm2.get(0).tags.get("topic")); + assertEquals(cm2.get(0).tags.get("namespace"), ns); + assertEquals(cm2.get(0).tags.get("cluster"), "test"); + + List brokerMetric2 = (List) metrics.get("pulsar_broker_storage_read_cache_misses_rate"); + assertEquals(brokerMetric2.size(), 1); + if (cacheEnable) { + assertEquals(brokerMetric2.get(0).value, 1.0); + } else { + assertEquals(brokerMetric2.get(0).value, 2.0); + } + assertEquals(brokerMetric2.get(0).tags.get("cluster"), "test"); + assertNull(brokerMetric2.get(0).tags.get("namespace")); + assertNull(brokerMetric2.get(0).tags.get("topic")); + + // test ManagedLedgerMetrics + List mlMetric = ((List) metrics.get("pulsar_ml_ReadEntriesOpsCacheMissesRate")); + assertEquals(mlMetric.size(), 1); + if (cacheEnable) { + assertEquals(mlMetric.get(0).value, 1.0); + } else { + assertEquals(mlMetric.get(0).value, 2.0); + } + assertEquals(mlMetric.get(0).tags.get("cluster"), "test"); + assertEquals(mlMetric.get(0).tags.get("namespace"), ns + "/persistent"); + } + @Test public void testPerTopicExpiredStat() throws Exception { String ns = "prop/ns-abc1"; From 3b118b6bd54cd8e592b8c2cd329a3fa3dfd18a11 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 7 Apr 2023 08:57:19 +0300 Subject: [PATCH 069/280] [fix][sec] Upgrade Spring dependency to 5.3.26 to fix OWASP Dependency Check (#20029) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f6f2909c1af3..4534036f3fd2c 100644 --- a/pom.xml +++ b/pom.xml @@ -232,7 +232,7 @@ flexible messaging model and an intuitive client API. 1.4.32 1.0 9.1.6 - 5.3.20 + 5.3.26 4.5.13 4.4.15 0.5.11 From 55acbe6cef2a0fbd356ad7d085c87e398b7c69db Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 7 Apr 2023 12:21:12 -0500 Subject: [PATCH 070/280] [refactor][fn] Use AuthorizationServer more in Function Worker API (#19975) ### Motivation The current function worker interfaces introduced by #8560 are hard to extend. It is better to pass an object that we can add fields to as needed. ### Modifications * Introduce a new `Authentication` class to wrap all of the relevant authentication "data". This class is somewhat unfortunate in that we already have many classes that hold authentication information. However, my goal with this class is to wrap all of the relevant auth info. This class is only currently used in the Function Worker API, but I plan to add it to the rest of the Admin HTTP endpoints. * Deprecate all of the methods that are no longer needed. Add a default implementation to call the new methods that supersede those methods. * Add `proxyRoles` setting to function worker. ### Verifying this change There are tests to cover the changes. ### Does this pull request potentially affect one of the following parts: This PR changes several interfaces in completely backwards compatible ways. It's possible we'll want to improve the abstraction for the `Authentication` class by making it an interface. ### Documentation - [x] `doc` This change has Javadoc updates to document the changes. ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/37 --- conf/functions_worker.yml | 2 + .../AuthenticationParameters.java | 67 +++ .../authorization/AuthorizationService.java | 128 +++++- .../apache/pulsar/broker/PulsarService.java | 1 + .../broker/admin/impl/BrokerStatsBase.java | 1 + .../broker/admin/impl/FunctionsBase.java | 49 +-- .../pulsar/broker/admin/impl/SinksBase.java | 28 +- .../pulsar/broker/admin/impl/SourcesBase.java | 29 +- .../pulsar/broker/admin/v2/Functions.java | 31 +- .../apache/pulsar/broker/admin/v2/Worker.java | 20 +- .../pulsar/broker/admin/v2/WorkerStats.java | 4 +- .../pulsar/broker/web/PulsarWebResource.java | 9 + .../pulsar/functions/worker/WorkerConfig.java | 6 + .../src/test/resources/test_worker_config.yml | 3 + .../worker/rest/FunctionApiResource.java | 18 + .../worker/rest/api/ComponentImpl.java | 411 +++++++----------- .../worker/rest/api/FunctionsImpl.java | 76 ++-- .../worker/rest/api/FunctionsImplV2.java | 66 +-- .../functions/worker/rest/api/SinksImpl.java | 92 +--- .../worker/rest/api/SourcesImpl.java | 92 +--- .../functions/worker/rest/api/WorkerImpl.java | 105 +++-- .../rest/api/v2/FunctionsApiV2Resource.java | 31 +- .../rest/api/v2/WorkerApiV2Resource.java | 36 +- .../rest/api/v2/WorkerStatsApiV2Resource.java | 20 +- .../rest/api/v3/FunctionsApiV3Resource.java | 48 +- .../rest/api/v3/SinksApiV3Resource.java | 35 +- .../rest/api/v3/SourcesApiV3Resource.java | 30 +- .../worker/service/api/Component.java | 211 +++------ .../worker/service/api/Functions.java | 86 +--- .../worker/service/api/FunctionsV2.java | 47 +- .../functions/worker/service/api/Sinks.java | 80 +--- .../functions/worker/service/api/Sources.java | 82 +--- .../functions/worker/service/api/Workers.java | 21 +- .../functions/worker/WorkerUtilsTest.java | 14 + .../worker/rest/api/FunctionsImplTest.java | 150 ++++--- .../worker/rest/api/WorkerImplTest.java | 120 +++++ .../api/v2/FunctionApiV2ResourceTest.java | 40 +- .../api/v3/FunctionApiV3ResourceTest.java | 53 +-- .../rest/api/v3/SinkApiV3ResourceTest.java | 43 +- .../rest/api/v3/SourceApiV3ResourceTest.java | 34 +- 40 files changed, 1169 insertions(+), 1250 deletions(-) create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java create mode 100644 pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index b41ac8f37a44f..bb15e0ca416da 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -311,6 +311,8 @@ authenticationProviders: authorizationProvider: org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider # Set of role names that are treated as "super-user", meaning they will be able to access any admin-api superUserRoles: +# Set of role names that are treated as "proxy" roles. These are the roles that can supply the originalPrincipal. +proxyRoles: #### tls configuration for worker service # Enable TLS diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java new file mode 100644 index 0000000000000..638772345bdf3 --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication; + +import lombok.Builder; + +/** + * A class to collect all the common fields used for authentication. Because the authentication data source is + * not always consistent when using the Pulsar Protocol and the Pulsar Proxy + * (see 19332), this class is currently restricted + * to use only in authenticating HTTP requests. + */ +@Builder +@lombok.Value +public class AuthenticationParameters { + + /** + * The original principal (or role) of the client. + *

+ * For HTTP Authentication, there are two possibilities. When the client connects directly to the broker, this + * field is null (assuming the client hasn't supplied the original principal header). When the client connects + * through the proxy, the proxy calculates the original principal based on the client's authentication data and + * supplies it in this field. + *

+ */ + String originalPrincipal; + + /** + * The client role. + *

+ * For HTTP Authentication, there are three possibilities. When the client connects directly to the broker, this + * is the client's role. When the client connects through the proxy using mTLS, this is the role of the proxy. + * In this case, the {@link #originalPrincipal} is also supplied and is the role of the original client as + * determined by the proxy. When the client connects through the proxy using any other form of authentication, + * this is the role of the original client. In this case, the {@link #originalPrincipal} is also the role of + * the original client. + *

+ */ + String clientRole; + + /** + * The authentication data source used to generate the {@link #clientRole}. + *

+ * For HTTP Authentication, there are three possibilities. When the client connects directly to the broker, this + * is the client's {@link AuthenticationDataSource}. When the client connects through the proxy using mTLS, this + * is the proxy's {@link AuthenticationDataSource}. When the client connects through the proxy using any other + * form of authentication, this is the original client's {@link AuthenticationDataSource}. + *

+ */ + AuthenticationDataSource clientAuthenticationDataSource; +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 6fff04b33b618..6c730f20092dd 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -23,10 +23,12 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -49,6 +51,7 @@ public class AuthorizationService { private static final Logger log = LoggerFactory.getLogger(AuthorizationService.class); + private final PulsarResources resources; private final AuthorizationProvider provider; private final ServiceConfiguration conf; @@ -61,6 +64,7 @@ public AuthorizationService(ServiceConfiguration conf, PulsarResources pulsarRes provider = (AuthorizationProvider) Class.forName(providerClassname) .getDeclaredConstructor().newInstance(); provider.initialize(conf, pulsarResources); + this.resources = pulsarResources; log.info("{} has been loaded.", providerClassname); } else { throw new PulsarServerException("No authorization providers are present."); @@ -72,6 +76,23 @@ public AuthorizationService(ServiceConfiguration conf, PulsarResources pulsarRes } } + public CompletableFuture isSuperUser(AuthenticationParameters authParams) { + if (!isValidOriginalPrincipal(authParams)) { + return CompletableFuture.completedFuture(false); + } + if (isProxyRole(authParams.getClientRole())) { + CompletableFuture isRoleAuthorizedFuture = isSuperUser(authParams.getClientRole(), + authParams.getClientAuthenticationDataSource()); + // The current paradigm is to pass the client auth data when we don't have access to the original auth data. + CompletableFuture isOriginalAuthorizedFuture = isSuperUser(authParams.getOriginalPrincipal(), + authParams.getClientAuthenticationDataSource()); + return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture, + (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized); + } else { + return isSuperUser(authParams.getClientRole(), authParams.getClientAuthenticationDataSource()); + } + } + public CompletableFuture isSuperUser(String user, AuthenticationDataSource authenticationData) { return provider.isSuperUser(user, authenticationData, conf); } @@ -279,17 +300,118 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return provider.allowFunctionOpsAsync(namespaceName, role, authenticationData); + return isSuperUserOrAdmin(namespaceName, role, authenticationData) + .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin + ? CompletableFuture.completedFuture(true) + : provider.allowFunctionOpsAsync(namespaceName, role, authenticationData) + ); + } + + public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, + AuthenticationParameters authParams) { + if (!isValidOriginalPrincipal(authParams)) { + return CompletableFuture.completedFuture(false); + } + if (isProxyRole(authParams.getClientRole())) { + CompletableFuture isRoleAuthorizedFuture = allowFunctionOpsAsync(namespaceName, + authParams.getClientRole(), authParams.getClientAuthenticationDataSource()); + // The current paradigm is to pass the client auth data when we don't have access to the original auth data. + CompletableFuture isOriginalAuthorizedFuture = allowFunctionOpsAsync( + namespaceName, authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource()); + return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture, + (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized); + } else { + return allowFunctionOpsAsync(namespaceName, authParams.getClientRole(), + authParams.getClientAuthenticationDataSource()); + } } public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return provider.allowSourceOpsAsync(namespaceName, role, authenticationData); + return isSuperUserOrAdmin(namespaceName, role, authenticationData) + .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin + ? CompletableFuture.completedFuture(true) + : provider.allowSourceOpsAsync(namespaceName, role, authenticationData) + ); + } + + public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, + AuthenticationParameters authParams) { + if (!isValidOriginalPrincipal(authParams)) { + return CompletableFuture.completedFuture(false); + } + if (isProxyRole(authParams.getClientRole())) { + CompletableFuture isRoleAuthorizedFuture = allowSourceOpsAsync(namespaceName, + authParams.getClientRole(), authParams.getClientAuthenticationDataSource()); + // The current paradigm is to pass the client auth data when we don't have access to the original auth data. + CompletableFuture isOriginalAuthorizedFuture = allowSourceOpsAsync( + namespaceName, authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource()); + return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture, + (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized); + } else { + return allowSourceOpsAsync(namespaceName, authParams.getClientRole(), + authParams.getClientAuthenticationDataSource()); + } } public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return provider.allowSinkOpsAsync(namespaceName, role, authenticationData); + return isSuperUserOrAdmin(namespaceName, role, authenticationData) + .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin + ? CompletableFuture.completedFuture(true) + : provider.allowSinkOpsAsync(namespaceName, role, authenticationData) + ); + } + + public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, + AuthenticationParameters authParams) { + if (!isValidOriginalPrincipal(authParams)) { + return CompletableFuture.completedFuture(false); + } + if (isProxyRole(authParams.getClientRole())) { + CompletableFuture isRoleAuthorizedFuture = allowSinkOpsAsync(namespaceName, + authParams.getClientRole(), authParams.getClientAuthenticationDataSource()); + // The current paradigm is to pass the client auth data when we don't have access to the original auth data. + CompletableFuture isOriginalAuthorizedFuture = allowSinkOpsAsync( + namespaceName, authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource()); + return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture, + (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized); + } else { + return allowSinkOpsAsync(namespaceName, authParams.getClientRole(), + authParams.getClientAuthenticationDataSource()); + } + } + + /** + * Functions, sources, and sinks each have their own method in this class. This method first checks for + * tenant admin access, then for namespace level permission. + */ + private CompletableFuture isSuperUserOrAdmin(NamespaceName namespaceName, + String role, + AuthenticationDataSource authenticationData) { + return isSuperUser(role, authenticationData) + .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin + ? CompletableFuture.completedFuture(true) + : isTenantAdmin(namespaceName.getTenant(), role, authenticationData)); + } + + private CompletableFuture isTenantAdmin(String tenant, String role, + AuthenticationDataSource authData) { + return resources.getTenantResources() + .getTenantAsync(tenant) + .thenCompose(op -> { + if (op.isPresent()) { + return isTenantAdmin(tenant, role, op.get(), authData); + } else { + return CompletableFuture.failedFuture(new RestException(Response.Status.NOT_FOUND, + "Tenant does not exist")); + } + }); + } + + private boolean isValidOriginalPrincipal(AuthenticationParameters authParams) { + return isValidOriginalPrincipal(authParams.getClientRole(), + authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource()); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 4af94c339c82f..b2eefe298b53c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1860,6 +1860,7 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu // inherit super users workerConfig.setSuperUserRoles(brokerConfig.getSuperUserRoles()); + workerConfig.setProxyRoles(brokerConfig.getProxyRoles()); workerConfig.setFunctionsWorkerEnablePackageManagement(brokerConfig.isFunctionsWorkerEnablePackageManagement()); // inherit the nar package locations diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java index 670fa10b28b88..6d49dd81da13d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java @@ -159,6 +159,7 @@ public LoadManagerReport getLoadReport() throws Exception { protected Map> internalBrokerResourceAvailability(NamespaceName namespace) { try { + validateSuperUserAccess(); LoadManager lm = pulsar().getLoadManager().get(); if (lm instanceof SimpleLoadManagerImpl) { return ((SimpleLoadManagerImpl) lm).getResourceAvailabilityFor(namespace).asMap(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java index bee81bf5dcdb6..4350316e2f011 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java @@ -194,7 +194,7 @@ public void registerFunction( ) final @FormDataParam("functionConfig") FunctionConfig functionConfig) { functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionConfig, clientAppId(), clientAuthData()); + functionPkgUrl, functionConfig, authParams()); } @PUT @@ -322,7 +322,7 @@ public void updateFunction( final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) throws IOException { functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionConfig, clientAppId(), clientAuthData(), updateOptions); + functionPkgUrl, functionConfig, authParams(), updateOptions); } @@ -343,7 +343,7 @@ public void deregisterFunction( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName) { - functions().deregisterFunction(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().deregisterFunction(tenant, namespace, functionName, authParams()); } @GET @@ -365,7 +365,7 @@ public FunctionConfig getFunctionInfo( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName) throws IOException { - return functions().getFunctionInfo(tenant, namespace, functionName, clientAppId(), clientAuthData()); + return functions().getFunctionInfo(tenant, namespace, functionName, authParams()); } @GET @@ -389,7 +389,7 @@ public FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData getFunct + " the stats of all instances is returned") final @PathParam("instanceId") String instanceId) throws IOException { return functions().getFunctionInstanceStatus(tenant, namespace, functionName, - instanceId, uri.getRequestUri(), clientAppId(), clientAuthData()); + instanceId, uri.getRequestUri(), authParams()); } @GET @@ -413,7 +413,7 @@ public FunctionStatus getFunctionStatus( @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName) throws IOException { return functions().getFunctionStatus(tenant, namespace, functionName, uri.getRequestUri(), - clientAppId(), clientAuthData()); + authParams()); } @GET @@ -437,7 +437,7 @@ public FunctionStatsImpl getFunctionStats( @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName) throws IOException { return functions().getFunctionStats(tenant, namespace, functionName, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @GET @@ -461,7 +461,7 @@ public FunctionInstanceStatsDataImpl getFunctionInstanceStats( + " (if instance-id is not provided, the stats of all instances is returned") final @PathParam( "instanceId") String instanceId) throws IOException { return functions().getFunctionsInstanceStats(tenant, namespace, functionName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @GET @@ -480,7 +480,7 @@ public List listFunctions( final @PathParam("tenant") String tenant, @ApiParam(value = "The namespace of a Pulsar Function") final @PathParam("namespace") String namespace) { - return functions().listFunctions(tenant, namespace, clientAppId(), clientAuthData()); + return functions().listFunctions(tenant, namespace, authParams()); } @POST @@ -509,7 +509,7 @@ public String triggerFunction( + " consumes from which you want to inject the data to") final @FormDataParam("topic") String topic) { return functions().triggerFunction(tenant, namespace, functionName, triggerValue, - triggerStream, topic, clientAppId(), clientAuthData()); + triggerStream, topic, authParams()); } @GET @@ -533,7 +533,7 @@ public FunctionState getFunctionState( final @PathParam("functionName") String functionName, @ApiParam(value = "The stats key") final @PathParam("key") String key) { - return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId(), clientAuthData()); + return functions().getFunctionState(tenant, namespace, functionName, key, authParams()); } @POST @@ -553,7 +553,7 @@ public void putFunctionState(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("key") String key, final @FormDataParam("state") FunctionState stateJson) { - functions().putFunctionState(tenant, namespace, functionName, key, stateJson, clientAppId(), clientAuthData()); + functions().putFunctionState(tenant, namespace, functionName, key, stateJson, authParams()); } @POST @@ -574,7 +574,7 @@ public void restartFunction( "The instanceId of a Pulsar Function (if instance-id is not provided, all instances are restarted") final @PathParam("instanceId") String instanceId) { functions().restartFunctionInstance(tenant, namespace, functionName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -593,7 +593,7 @@ public void restartFunction( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName) { - functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().restartFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -613,7 +613,7 @@ public void stopFunction( "The instanceId of a Pulsar Function (if instance-id is not provided, all instances are stopped. ") final @PathParam("instanceId") String instanceId) { functions().stopFunctionInstance(tenant, namespace, functionName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -632,7 +632,7 @@ public void stopFunction( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName) { - functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().stopFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -652,7 +652,7 @@ public void startFunction( + " (if instance-id is not provided, all instances sre started. ") final @PathParam("instanceId") String instanceId) { functions().startFunctionInstance(tenant, namespace, functionName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -671,7 +671,7 @@ public void startFunction( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName) { - functions().startFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().startFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -683,7 +683,7 @@ public void startFunction( @Consumes(MediaType.MULTIPART_FORM_DATA) public void uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("path") String path) { - functions().uploadFunction(uploadedInputStream, path, clientAppId(), clientAuthData()); + functions().uploadFunction(uploadedInputStream, path, authParams()); } @GET @@ -693,7 +693,7 @@ public void uploadFunction(final @FormDataParam("data") InputStream uploadedInpu ) @Path("/download") public StreamingOutput downloadFunction(final @QueryParam("path") String path) { - return functions().downloadFunction(path, clientAppId(), clientAuthData()); + return functions().downloadFunction(path, authParams()); } @GET @@ -712,8 +712,7 @@ public StreamingOutput downloadFunction( @ApiParam(value = "Whether to download the transform-function") final @QueryParam("transform-function") boolean transformFunction) { - return functions() - .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), transformFunction); + return functions().downloadFunction(tenant, namespace, functionName, authParams(), transformFunction); } @GET @@ -746,7 +745,7 @@ public List getConnectorsList() throws IOException { }) @Path("/builtins/reload") public void reloadBuiltinFunctions() throws IOException { - functions().reloadBuiltinFunctions(clientAppId(), clientAuthData()); + functions().reloadBuiltinFunctions(authParams()); } @GET @@ -763,7 +762,7 @@ public void reloadBuiltinFunctions() throws IOException { @Path("/builtins") @Produces(MediaType.APPLICATION_JSON) public List getBuiltinFunction() { - return functions().getBuiltinFunctions(clientAppId(), clientAuthData()); + return functions().getBuiltinFunctions(authParams()); } @PUT @@ -785,6 +784,6 @@ public void updateFunctionOnWorkerLeader(final @PathParam("tenant") String tenan final @FormDataParam("delete") boolean delete) { functions().updateFunctionOnWorkerLeader(tenant, namespace, functionName, uploadedInputStream, - delete, uri.getRequestUri(), clientAppId(), clientAuthData()); + delete, uri.getRequestUri(), authParams()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java index 7a3a554a3a00c..80ad72d6f9aa9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java @@ -168,7 +168,7 @@ public void registerSink(@ApiParam(value = "The tenant of a Pulsar Sink") final ) final @FormDataParam("sinkConfig") SinkConfig sinkConfig) { sinks().registerSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail, - sinkPkgUrl, sinkConfig, clientAppId(), clientAuthData()); + sinkPkgUrl, sinkConfig, authParams()); } @PUT @@ -271,7 +271,7 @@ public void updateSink(@ApiParam(value = "The tenant of a Pulsar Sink") final @P @ApiParam(value = "Update options for the Pulsar Sink") final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) { sinks().updateSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail, - sinkPkgUrl, sinkConfig, clientAppId(), clientAuthData(), updateOptions); + sinkPkgUrl, sinkConfig, authParams(), updateOptions); } @@ -295,7 +295,7 @@ public void deregisterSink(@ApiParam(value = "The tenant of a Pulsar Sink") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Sink") final @PathParam("sinkName") String sinkName) { - sinks().deregisterFunction(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().deregisterFunction(tenant, namespace, sinkName, authParams()); } @GET @@ -315,7 +315,7 @@ public SinkConfig getSinkInfo(@ApiParam(value = "The tenant of a Pulsar Sink") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Sink") final @PathParam("sinkName") String sinkName) throws IOException { - return sinks().getSinkInfo(tenant, namespace, sinkName); + return sinks().getSinkInfo(tenant, namespace, sinkName, authParams()); } @GET @@ -342,7 +342,7 @@ public SinkStatus.SinkInstanceStatus.SinkInstanceStatusData getSinkInstanceStatu @ApiParam(value = "The instanceId of a Pulsar Sink") final @PathParam("instanceId") String instanceId) throws IOException { return sinks().getSinkInstanceStatus( - tenant, namespace, sinkName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData()); + tenant, namespace, sinkName, instanceId, uri.getRequestUri(), authParams()); } @GET @@ -365,7 +365,7 @@ public SinkStatus getSinkStatus(@ApiParam(value = "The tenant of a Pulsar Sink") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Sink") final @PathParam("sinkName") String sinkName) throws IOException { - return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), clientAppId(), clientAuthData()); + return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), authParams()); } @GET @@ -385,7 +385,7 @@ public List listSinks(@ApiParam(value = "The tenant of a Pulsar Sink") final @PathParam("tenant") String tenant, @ApiParam(value = "The namespace of a Pulsar Sink") final @PathParam("namespace") String namespace) { - return sinks().listFunctions(tenant, namespace, clientAppId(), clientAuthData()); + return sinks().listFunctions(tenant, namespace, authParams()); } @POST @@ -411,7 +411,7 @@ public void restartSink(@ApiParam(value = "The tenant of a Pulsar Sink") @ApiParam(value = "The instanceId of a Pulsar Sink") final @PathParam("instanceId") String instanceId) { sinks().restartFunctionInstance(tenant, namespace, sinkName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -432,7 +432,7 @@ public void restartSink(@ApiParam(value = "The tenant of a Pulsar Sink") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Sink") final @PathParam("sinkName") String sinkName) { - sinks().restartFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().restartFunctionInstances(tenant, namespace, sinkName, authParams()); } @POST @@ -456,7 +456,7 @@ public void stopSink(@ApiParam(value = "The tenant of a Pulsar Sink") @ApiParam(value = "The instanceId of a Pulsar Sink") final @PathParam("instanceId") String instanceId) { sinks().stopFunctionInstance(tenant, namespace, - sinkName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData()); + sinkName, instanceId, uri.getRequestUri(), authParams()); } @POST @@ -477,7 +477,7 @@ public void stopSink(@ApiParam(value = "The tenant of a Pulsar Sink") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Sink") final @PathParam("sinkName") String sinkName) { - sinks().stopFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().stopFunctionInstances(tenant, namespace, sinkName, authParams()); } @POST @@ -501,7 +501,7 @@ public void startSink(@ApiParam(value = "The tenant of a Pulsar Sink") @ApiParam(value = "The instanceId of a Pulsar Sink") final @PathParam("instanceId") String instanceId) { sinks().startFunctionInstance(tenant, namespace, sinkName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -522,7 +522,7 @@ public void startSink(@ApiParam(value = "The tenant of a Pulsar Sink") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Sink") final @PathParam("sinkName") String sinkName) { - sinks().startFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().startFunctionInstances(tenant, namespace, sinkName, authParams()); } @GET @@ -571,6 +571,6 @@ public List getSinkConfigDefinition( }) @Path("/reloadBuiltInSinks") public void reloadSinks() { - sinks().reloadConnectors(clientAppId(), clientAuthData()); + sinks().reloadConnectors(authParams()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java index 1bf092784dd3a..4af0afc0d6ec5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java @@ -142,7 +142,7 @@ public void registerSource( ) final @FormDataParam("sourceConfig") SourceConfig sourceConfig) { sources().registerSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail, - sourcePkgUrl, sourceConfig, clientAppId(), clientAuthData()); + sourcePkgUrl, sourceConfig, authParams()); } @PUT @@ -227,7 +227,7 @@ public void updateSource( @ApiParam(value = "Update options for Pulsar Source") final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) { sources().updateSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail, - sourcePkgUrl, sourceConfig, clientAppId(), clientAuthData(), updateOptions); + sourcePkgUrl, sourceConfig, authParams(), updateOptions); } @@ -250,7 +250,7 @@ public void deregisterSource( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Source") final @PathParam("sourceName") String sourceName) { - sources().deregisterFunction(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().deregisterFunction(tenant, namespace, sourceName, authParams()); } @GET @@ -271,7 +271,7 @@ public SourceConfig getSourceInfo( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Source") final @PathParam("sourceName") String sourceName) throws IOException { - return sources().getSourceInfo(tenant, namespace, sourceName); + return sources().getSourceInfo(tenant, namespace, sourceName, authParams()); } @GET @@ -294,7 +294,7 @@ public SourceStatus.SourceInstanceStatus.SourceInstanceStatusData getSourceInsta + " (if instance-id is not provided, the stats of all instances is returned).") final @PathParam( "instanceId") String instanceId) throws IOException { return sources().getSourceInstanceStatus( - tenant, namespace, sourceName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData()); + tenant, namespace, sourceName, instanceId, uri.getRequestUri(), authParams()); } @GET @@ -316,8 +316,7 @@ public SourceStatus getSourceStatus( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Source") final @PathParam("sourceName") String sourceName) throws IOException { - return sources().getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), clientAppId(), - clientAuthData()); + return sources().getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), authParams()); } @GET @@ -339,7 +338,7 @@ public List listSources( final @PathParam("tenant") String tenant, @ApiParam(value = "The namespace of a Pulsar Source") final @PathParam("namespace") String namespace) { - return sources().listFunctions(tenant, namespace, clientAppId(), clientAuthData()); + return sources().listFunctions(tenant, namespace, authParams()); } @POST @@ -362,7 +361,7 @@ public void restartSource( + " (if instance-id is not provided, the stats of all instances is returned).") final @PathParam( "instanceId") String instanceId) { sources().restartFunctionInstance(tenant, namespace, sourceName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -383,7 +382,7 @@ public void restartSource( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Source") final @PathParam("sourceName") String sourceName) { - sources().restartFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().restartFunctionInstances(tenant, namespace, sourceName, authParams()); } @POST @@ -404,7 +403,7 @@ public void stopSource( @ApiParam(value = "The instanceId of a Pulsar Source (if instance-id is not provided," + " the stats of all instances is returned).") final @PathParam("instanceId") String instanceId) { sources().stopFunctionInstance(tenant, namespace, sourceName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -425,7 +424,7 @@ public void stopSource( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Source") final @PathParam("sourceName") String sourceName) { - sources().stopFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().stopFunctionInstances(tenant, namespace, sourceName, authParams()); } @POST @@ -446,7 +445,7 @@ public void startSource( @ApiParam(value = "The instanceId of a Pulsar Source (if instance-id is not provided," + " the stats of all instances is returned).") final @PathParam("instanceId") String instanceId) { sources().startFunctionInstance(tenant, namespace, sourceName, instanceId, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @POST @@ -467,7 +466,7 @@ public void startSource( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Source") final @PathParam("sourceName") String sourceName) { - sources().startFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().startFunctionInstances(tenant, namespace, sourceName, authParams()); } @GET @@ -520,6 +519,6 @@ public List getSourceConfigDefinition( }) @Path("/reloadBuiltInSources") public void reloadSources() { - sources().reloadConnectors(clientAppId(), clientAuthData()); + sources().reloadConnectors(authParams()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java index 0d302bec065bc..69f99a42d0685 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java @@ -75,7 +75,7 @@ public Response registerFunction(final @PathParam("tenant") String tenant, final @FormDataParam("functionDetails") String functionDetailsJson) { return functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionDetailsJson, clientAppId()); + functionPkgUrl, functionDetailsJson, authParams()); } @PUT @@ -96,7 +96,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant, final @FormDataParam("functionDetails") String functionDetailsJson) { return functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionDetailsJson, clientAppId()); + functionPkgUrl, functionDetailsJson, authParams()); } @@ -113,7 +113,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant, public Response deregisterFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - return functions().deregisterFunction(tenant, namespace, functionName, clientAppId()); + return functions().deregisterFunction(tenant, namespace, functionName, authParams()); } @GET @@ -132,8 +132,7 @@ public Response getFunctionInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws IOException { - return functions().getFunctionInfo( - tenant, namespace, functionName, clientAppId()); + return functions().getFunctionInfo(tenant, namespace, functionName, authParams()); } @GET @@ -154,7 +153,7 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena final @PathParam("instanceId") String instanceId) throws IOException { return functions().getFunctionInstanceStatus(tenant, namespace, functionName, instanceId, uri.getRequestUri(), - clientAppId()); + authParams()); } @GET @@ -172,7 +171,7 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws IOException { return functions().getFunctionStatusV2( - tenant, namespace, functionName, uri.getRequestUri(), clientAppId()); + tenant, namespace, functionName, uri.getRequestUri(), authParams()); } @GET @@ -188,7 +187,7 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant, @Path("/{tenant}/{namespace}") public Response listFunctions(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace) { - return functions().listFunctions(tenant, namespace, clientAppId()); + return functions().listFunctions(tenant, namespace, authParams()); } @POST @@ -211,7 +210,7 @@ public Response triggerFunction(final @PathParam("tenant") String tenant, final @FormDataParam("dataStream") InputStream triggerStream, final @FormDataParam("topic") String topic) { return functions().triggerFunction(tenant, namespace, functionName, - triggerValue, triggerStream, topic, clientAppId()); + triggerValue, triggerStream, topic, authParams()); } @GET @@ -230,7 +229,7 @@ public Response getFunctionState(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName, final @PathParam("key") String key) { - return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId()); + return functions().getFunctionState(tenant, namespace, functionName, key, authParams()); } @POST @@ -247,7 +246,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) { return functions().restartFunctionInstance(tenant, namespace, functionName, - instanceId, uri.getRequestUri(), clientAppId()); + instanceId, uri.getRequestUri(), authParams()); } @POST @@ -260,7 +259,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant, public Response restartFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - return functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId()); + return functions().restartFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -275,7 +274,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) { return functions().stopFunctionInstance(tenant, namespace, functionName, - instanceId, uri.getRequestUri(), clientAppId()); + instanceId, uri.getRequestUri(), authParams()); } @POST @@ -288,7 +287,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant, public Response stopFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - return functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId()); + return functions().stopFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -300,7 +299,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant, @Consumes(MediaType.MULTIPART_FORM_DATA) public Response uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("path") String path) { - return functions().uploadFunction(uploadedInputStream, path, clientAppId()); + return functions().uploadFunction(uploadedInputStream, path, authParams()); } @GET @@ -310,7 +309,7 @@ public Response uploadFunction(final @FormDataParam("data") InputStream uploaded ) @Path("/download") public Response downloadFunction(final @QueryParam("path") String path) { - return functions().downloadFunction(path, clientAppId()); + return functions().downloadFunction(path, authParams()); } @GET diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java index 80c75b04917ce..3813790e4f428 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java @@ -66,7 +66,7 @@ public WorkerService get() { @Path("/cluster") @Produces(MediaType.APPLICATION_JSON) public List getCluster() { - return workers().getCluster(clientAppId()); + return workers().getCluster(authParams()); } @GET @@ -81,7 +81,7 @@ public List getCluster() { @Path("/cluster/leader") @Produces(MediaType.APPLICATION_JSON) public WorkerInfo getClusterLeader() { - return workers().getClusterLeader(clientAppId()); + return workers().getClusterLeader(authParams()); } @GET @@ -96,7 +96,7 @@ public WorkerInfo getClusterLeader() { @Path("/assignments") @Produces(MediaType.APPLICATION_JSON) public Map> getAssignments() { - return workers().getAssignments(clientAppId()); + return workers().getAssignments(authParams()); } @GET @@ -112,7 +112,7 @@ public Map> getAssignments() { @Path("/connectors") @Produces(MediaType.APPLICATION_JSON) public List getConnectorsList() throws IOException { - return workers().getListOfConnectors(clientAppId()); + return workers().getListOfConnectors(authParams()); } @PUT @@ -126,7 +126,7 @@ public List getConnectorsList() throws IOException { }) @Path("/rebalance") public void rebalance() { - workers().rebalance(uri.getRequestUri(), clientAppId()); + workers().rebalance(uri.getRequestUri(), authParams()); } @PUT @@ -142,7 +142,7 @@ public void rebalance() { }) @Path("/leader/drain") public void drainAtLeader(@QueryParam("workerId") String workerId) { - workers().drain(uri.getRequestUri(), workerId, clientAppId(), true); + workers().drain(uri.getRequestUri(), workerId, authParams(), true); } @PUT @@ -158,7 +158,7 @@ public void drainAtLeader(@QueryParam("workerId") String workerId) { }) @Path("/drain") public void drain() { - workers().drain(uri.getRequestUri(), null, clientAppId(), false); + workers().drain(uri.getRequestUri(), null, authParams(), false); } @GET @@ -172,7 +172,7 @@ public void drain() { }) @Path("/leader/drain") public LongRunningProcessStatus getDrainStatusFromLeader(@QueryParam("workerId") String workerId) { - return workers().getDrainStatus(uri.getRequestUri(), workerId, clientAppId(), true); + return workers().getDrainStatus(uri.getRequestUri(), workerId, authParams(), true); } @GET @@ -186,7 +186,7 @@ public LongRunningProcessStatus getDrainStatusFromLeader(@QueryParam("workerId") }) @Path("/drain") public LongRunningProcessStatus getDrainStatus() { - return workers().getDrainStatus(uri.getRequestUri(), null, clientAppId(), false); + return workers().getDrainStatus(uri.getRequestUri(), null, authParams(), false); } @GET @@ -199,6 +199,6 @@ public LongRunningProcessStatus getDrainStatus() { }) @Path("/cluster/leader/ready") public Boolean isLeaderReady() { - return workers().isLeaderReady(clientAppId()); + return workers().isLeaderReady(authParams()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java index 5a77d81b6f1e3..1716b5e588c77 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java @@ -56,7 +56,7 @@ public Workers workers() { }) @Produces(MediaType.APPLICATION_JSON) public Collection getMetrics() throws Exception { - return workers().getWorkerMetrics(clientAppId()); + return workers().getWorkerMetrics(authParams()); } @GET @@ -72,6 +72,6 @@ public Collection getMetrics() throws Exception { }) @Produces(MediaType.APPLICATION_JSON) public List getStats() throws IOException { - return workers().getFunctionsMetrics(clientAppId()); + return workers().getFunctionsMetrics(authParams()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index a182e4733fdfe..3d23d7812543a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -54,6 +54,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.namespace.LookupOptions; @@ -143,6 +144,14 @@ public static String splitPath(String source, int slice) { return PolicyPath.splitPath(source, slice); } + public AuthenticationParameters authParams() { + return AuthenticationParameters.builder() + .originalPrincipal(originalPrincipal()) + .clientRole(clientAppId()) + .clientAuthenticationDataSource(clientAuthData()) + .build(); + } + /** * Gets a caller id (IP + role). * diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index b87477f78c641..3b8ddf774d162 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -563,6 +563,12 @@ public boolean isBrokerClientAuthenticationEnabled() { ) private Set superUserRoles = new TreeSet<>(); + @FieldContext( + category = CATEGORY_WORKER_SECURITY, + doc = "Role names that are treated as `proxy roles`. These are the only roles that can supply the " + + "originalPrincipal.") + private Set proxyRoles = new TreeSet<>(); + @FieldContext( category = CATEGORY_WORKER_SECURITY, doc = "This is a regexp, which limits the range of possible ids which can connect to the Broker using SASL." diff --git a/pulsar-functions/src/test/resources/test_worker_config.yml b/pulsar-functions/src/test/resources/test_worker_config.yml index f0ecf2bd71bc6..a297788037035 100644 --- a/pulsar-functions/src/test/resources/test_worker_config.yml +++ b/pulsar-functions/src/test/resources/test_worker_config.yml @@ -23,6 +23,9 @@ pulsarServiceUrl: pulsar://localhost:6650 functionMetadataTopicName: test-function-metadata-topic numFunctionPackageReplicas: 3 maxPendingAsyncRequests: 200 +proxyRoles: + - "proxyA" + - "proxyB" properties: # Fake Bookkeeper Client config to be applied to the DLog Bookkeeper Client bookkeeper_testKey: "fakeValue" diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java index d37f7f15c606e..cfce3d3e68912 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java @@ -24,12 +24,14 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.UriInfo; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.functions.worker.WorkerService; public class FunctionApiResource implements Supplier { public static final String ATTRIBUTE_FUNCTION_WORKER = "function-worker"; + public static final String ORIGINAL_PRINCIPAL_HEADER = "X-Original-Principal"; private WorkerService workerService; @Context @@ -47,12 +49,28 @@ public synchronized WorkerService get() { return this.workerService; } + public AuthenticationParameters authParams() { + return AuthenticationParameters.builder() + .originalPrincipal(httpRequest.getHeader(ORIGINAL_PRINCIPAL_HEADER)) + .clientRole(clientAppId()) + .clientAuthenticationDataSource(clientAuthData()) + .build(); + } + + /** + * @deprecated use {@link #authParams()} instead. + */ + @Deprecated public String clientAppId() { return httpRequest != null ? (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName) : null; } + /** + * @deprecated use {@link #authParams()} instead. + */ + @Deprecated public AuthenticationDataSource clientAuthData() { return (AuthenticationDataSource) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 0b6c2bbd61f55..7daa9c9aee8ed 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -67,6 +67,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.internal.FunctionsImpl; import org.apache.pulsar.client.api.Message; @@ -84,7 +85,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl; import org.apache.pulsar.common.policies.data.FunctionStatsImpl; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.instance.InstanceUtils; @@ -456,23 +456,14 @@ private void deleteStatestoreTableAsync(String namespace, String table) { public void deregisterFunction(final String tenant, final String namespace, final String componentName, - final String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to deregister {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "deregister", + authParams); // validate parameters try { @@ -540,23 +531,14 @@ private void deleteComponentFromStorage(String tenant, String namespace, String public FunctionConfig getFunctionInfo(final String tenant, final String namespace, final String componentName, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to get {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get", + authParams); // validate parameters try { @@ -591,10 +573,8 @@ public void stopFunctionInstance(final String tenant, final String componentName, final String instanceId, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { - changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, false, uri, clientRole, - clientAuthenticationDataHttps); + final AuthenticationParameters authParams) { + changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, false, uri, authParams); } @Override @@ -603,12 +583,11 @@ public void startFunctionInstance(final String tenant, final String componentName, final String instanceId, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { - changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, true, uri, clientRole, - clientAuthenticationDataHttps); + final AuthenticationParameters authParams) { + changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, true, uri, authParams); } + @Deprecated public void changeFunctionInstanceStatus(final String tenant, final String namespace, final String componentName, @@ -617,21 +596,27 @@ public void changeFunctionInstanceStatus(final String tenant, final URI uri, final String clientRole, final AuthenticationDataSource clientAuthenticationDataHttps) { + AuthenticationParameters authParams = AuthenticationParameters.builder() + .clientRole(clientRole) + .clientAuthenticationDataSource(clientAuthenticationDataHttps) + .build(); + changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, start, uri, authParams); + } + + public void changeFunctionInstanceStatus(final String tenant, + final String namespace, + final String componentName, + final String instanceId, + final boolean start, + final URI uri, + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to start/stop {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "start/stop", + authParams); // validate parameters try { @@ -678,22 +663,13 @@ public void restartFunctionInstance(final String tenant, final String componentName, final String instanceId, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to restart {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "restart", + authParams); // validate parameters try { @@ -738,43 +714,41 @@ public void restartFunctionInstance(final String tenant, public void stopFunctionInstances(final String tenant, final String namespace, final String componentName, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { - changeFunctionStatusAllInstances(tenant, namespace, componentName, false, clientRole, - clientAuthenticationDataHttps); + final AuthenticationParameters authParams) { + changeFunctionStatusAllInstances(tenant, namespace, componentName, false, authParams); } @Override public void startFunctionInstances(final String tenant, final String namespace, final String componentName, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { - changeFunctionStatusAllInstances(tenant, namespace, componentName, true, clientRole, - clientAuthenticationDataHttps); + final AuthenticationParameters authParams) { + changeFunctionStatusAllInstances(tenant, namespace, componentName, true, authParams); } + @Deprecated public void changeFunctionStatusAllInstances(final String tenant, final String namespace, final String componentName, final boolean start, final String clientRole, final AuthenticationDataSource clientAuthenticationDataHttps) { + AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole) + .clientAuthenticationDataSource(clientAuthenticationDataHttps).build(); + changeFunctionStatusAllInstances(tenant, namespace, componentName, start, authParams); + } + public void changeFunctionStatusAllInstances(final String tenant, + final String namespace, + final String componentName, + final boolean start, + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to start/stop {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "start/stop", + authParams); // validate parameters try { @@ -815,25 +789,17 @@ public void changeFunctionStatusAllInstances(final String tenant, namespace, componentName)); } + @Override public void restartFunctionInstances(final String tenant, final String namespace, final String componentName, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to restart {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "restart", + authParams); // validate parameters try { @@ -873,26 +839,18 @@ public void restartFunctionInstances(final String tenant, } } + @Override public FunctionStatsImpl getFunctionStats(final String tenant, final String namespace, final String componentName, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to get stats for {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get stats for", + authParams); // validate parameters try { @@ -941,22 +899,13 @@ public FunctionStatsImpl getFunctionStats(final String tenant, final String componentName, final String instanceId, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to get stats for {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get stats for", + authParams); // validate parameters try { @@ -1012,23 +961,13 @@ public FunctionStatsImpl getFunctionStats(final String tenant, @Override public List listFunctions(final String tenant, final String namespace, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{} Client [{}] is not authorized to list {}", tenant, namespace, clientRole, - ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{} Failed to authorize [{}]", tenant, namespace, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, null, "list", authParams); // validate parameters try { @@ -1070,13 +1009,13 @@ public List getListOfConnectors() { } @Override - public void reloadConnectors(String clientRole, AuthenticationDataSource authenticationData) { + public void reloadConnectors(AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } if (worker().getWorkerConfig().isAuthorizationEnabled()) { // Only superuser has permission to do this operation. - if (!isSuperUser(clientRole, authenticationData)) { + if (!isSuperUser(authParams)) { throw new RestException(Status.UNAUTHORIZED, "This operation requires super-user access"); } } @@ -1094,23 +1033,13 @@ public String triggerFunction(final String tenant, final String input, final InputStream uploadedInputStream, final String topic, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to trigger {}", tenant, namespace, - functionName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "trigger", authParams); // validate parameters try { @@ -1224,23 +1153,14 @@ public FunctionState getFunctionState(final String tenant, final String namespace, final String functionName, final String key, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to get state for {}", tenant, namespace, - functionName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "get state for", + authParams); if (null == worker().getStateStoreAdminClient()) { throwStateStoreUnvailableResponse(); @@ -1310,8 +1230,7 @@ public void putFunctionState(final String tenant, final String functionName, final String key, final FunctionState state, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); @@ -1321,16 +1240,8 @@ public void putFunctionState(final String tenant, throwStateStoreUnvailableResponse(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to put state for {}", tenant, namespace, - functionName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "put state for", + authParams); if (!key.equals(state.getKey())) { log.error("{}/{}/{} Bad putFunction Request, path key doesn't match key in json", tenant, namespace, @@ -1386,14 +1297,14 @@ public void putFunctionState(final String tenant, } @Override - public void uploadFunction(final InputStream uploadedInputStream, final String path, String clientRole, - AuthenticationDataSource authenticationData) { + public void uploadFunction(final InputStream uploadedInputStream, final String path, + AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole, authenticationData)) { + if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(authParams)) { throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } @@ -1428,23 +1339,13 @@ public void uploadFunction(final InputStream uploadedInputStream, final String p @Override public StreamingOutput downloadFunction(String tenant, String namespace, String componentName, - String clientRole, AuthenticationDataSource clientAuthenticationDataHttps, - boolean transformFunction) { + AuthenticationParameters authParams, boolean transformFunction) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not admin and authorized to download package for {} ", tenant, - namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "download package for", + authParams); FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); if (!functionMetaDataManager.containsFunction(tenant, namespace, componentName)) { @@ -1529,8 +1430,7 @@ private Path getBuiltinArchivePath(String pkgPath, FunctionDetails.ComponentType } @Override - public StreamingOutput downloadFunction( - final String path, String clientRole, AuthenticationDataSource clientAuthenticationDataHttps) { + public StreamingOutput downloadFunction(final String path, final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); @@ -1544,19 +1444,10 @@ public StreamingOutput downloadFunction( String namespace = tokens[1]; String componentName = tokens[2]; - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not admin and authorized to download package for {} ", tenant, - namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "download package for", + authParams); } else { - if (!isSuperUser(clientRole, clientAuthenticationDataHttps)) { + if (!isSuperUser(authParams)) { throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } } @@ -1694,57 +1585,61 @@ public static String createPackagePath(String tenant, String namespace, String f getUniquePackageName(Codec.encode(fileName))); } + /** + * @deprecated use {@link #isAuthorizedRole(String, String, AuthenticationParameters)} instead. + */ + @Deprecated public boolean isAuthorizedRole(String tenant, String namespace, String clientRole, AuthenticationDataSource authenticationData) throws PulsarAdminException { - if (worker().getWorkerConfig().isAuthorizationEnabled()) { - // skip authorization if client role is super-user - if (isSuperUser(clientRole, authenticationData)) { - return true; - } - - if (clientRole != null) { - try { - TenantInfoImpl tenantInfo = - (TenantInfoImpl) worker().getBrokerAdmin().tenants().getTenantInfo(tenant); - if (tenantInfo != null && worker().getAuthorizationService() - .isTenantAdmin(tenant, clientRole, tenantInfo, authenticationData).get()) { - return true; - } - } catch (PulsarAdminException.NotFoundException | InterruptedException | ExecutionException e) { + AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole) + .clientAuthenticationDataSource(authenticationData).build(); + return isAuthorizedRole(tenant, namespace, authParams); + } - } - } + public boolean isAuthorizedRole(String tenant, String namespace, + AuthenticationParameters authParams) throws PulsarAdminException { + if (worker().getWorkerConfig().isAuthorizationEnabled()) { + return allowFunctionOps(NamespaceName.get(tenant, namespace), authParams); + } else { + return true; + } + } - // check if role has permissions granted - if (clientRole != null && authenticationData != null) { - return allowFunctionOps(NamespaceName.get(tenant, namespace), clientRole, authenticationData); - } else { - return false; + public void throwRestExceptionIfUnauthorizedForNamespace(String tenant, String namespace, String componentName, + String action, AuthenticationParameters authParams) { + try { + if (!isAuthorizedRole(tenant, namespace, authParams)) { + log.warn("{}/{}/{} Client with role [{}] and originalPrincipal [{}] is not authorized to {} {}", + tenant, namespace, componentName, authParams.getClientRole(), + authParams.getOriginalPrincipal(), action, ComponentTypeUtils.toString(componentType)); + throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } + } catch (PulsarAdminException e) { + log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); + throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); } - return true; } - + @Deprecated protected void componentStatusRequestValidate(final String tenant, final String namespace, final String componentName, final String clientRole, final AuthenticationDataSource clientAuthenticationDataHttps) { + AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole) + .clientAuthenticationDataSource(clientAuthenticationDataHttps).build(); + componentStatusRequestValidate(tenant, namespace, componentName, authParams); + } + + protected void componentStatusRequestValidate(final String tenant, final String namespace, + final String componentName, + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throw new RestException(Status.SERVICE_UNAVAILABLE, "Function worker service is not done initializing. Please try again in a little while."); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized get status for {}", tenant, namespace, - componentName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e); - throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get status for", + authParams); // validate parameters try { @@ -1773,13 +1668,25 @@ protected void componentStatusRequestValidate(final String tenant, final String } } + @Deprecated protected void componentInstanceStatusRequestValidate(final String tenant, final String namespace, final String componentName, final int instanceId, final String clientRole, final AuthenticationDataSource clientAuthenticationDataHttps) { - componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps); + AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole) + .clientAuthenticationDataSource(clientAuthenticationDataHttps).build(); + componentInstanceStatusRequestValidate(tenant, namespace, componentName, instanceId, authParams); + } + + protected void componentInstanceStatusRequestValidate(final String tenant, + final String namespace, + final String componentName, + final int instanceId, + final AuthenticationParameters authParams) { + + componentStatusRequestValidate(tenant, namespace, componentName, authParams); FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); FunctionMetaData functionMetaData = @@ -1794,44 +1701,58 @@ protected void componentInstanceStatusRequestValidate(final String tenant, } } - public boolean isSuperUser(String clientRole, AuthenticationDataSource authenticationData) { - if (clientRole != null) { + public boolean isSuperUser(AuthenticationParameters authParams) { + if (authParams.getClientRole() != null) { try { - if ((worker().getWorkerConfig().getSuperUserRoles() != null - && worker().getWorkerConfig().getSuperUserRoles().contains(clientRole))) { - return true; - } - return worker().getAuthorizationService().isSuperUser(clientRole, authenticationData) - .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + return worker().getAuthorizationService().isSuperUser(authParams) + .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); } catch (InterruptedException e) { - log.warn("Time-out {} sec while checking the role {} is a super user role ", - worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), clientRole); + log.warn("Time-out {} sec while checking the role {} originalPrincipal {} is a super user role ", + worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), + authParams.getClientRole(), authParams.getOriginalPrincipal()); throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); } catch (Exception e) { - log.warn("Admin-client with Role - failed to check the role {} is a super user role {} ", clientRole, - e.getMessage(), e); + log.warn("Failed verifying role {} originalPrincipal {} is a super user role", + authParams.getClientRole(), authParams.getOriginalPrincipal(), e); throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); } } return false; } + /** + * @deprecated use {@link #isSuperUser(AuthenticationParameters)} + */ + @Deprecated + public boolean isSuperUser(String clientRole, AuthenticationDataSource authenticationData) { + AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole) + .clientAuthenticationDataSource(authenticationData).build(); + return isSuperUser(authParams); + } + + /** + * @deprecated use {@link #isSuperUser(AuthenticationParameters)} + */ + @Deprecated public boolean allowFunctionOps(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { + AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(role) + .clientAuthenticationDataSource(authenticationData).build(); + return allowFunctionOps(namespaceName, authParams); + } + + public boolean allowFunctionOps(NamespaceName namespaceName, AuthenticationParameters authParams) { try { switch (componentType) { case SINK: - return worker().getAuthorizationService().allowSinkOpsAsync( - namespaceName, role, authenticationData) + return worker().getAuthorizationService().allowSinkOpsAsync(namespaceName, authParams) .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); case SOURCE: - return worker().getAuthorizationService().allowSourceOpsAsync( - namespaceName, role, authenticationData) + return worker().getAuthorizationService().allowSourceOpsAsync(namespaceName, authParams) .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); case FUNCTION: default: - return worker().getAuthorizationService().allowFunctionOpsAsync( - namespaceName, role, authenticationData) + return worker().getAuthorizationService().allowFunctionOpsAsync(namespaceName, authParams) .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); } } catch (InterruptedException e) { @@ -1839,9 +1760,9 @@ public boolean allowFunctionOps(NamespaceName namespaceName, String role, worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), namespaceName); throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); } catch (Exception e) { - log.warn("Admin-client with Role - {} failed to get function permissions for namespace - {}. {}", role, - namespaceName, - e.getMessage(), e); + log.warn("Admin-client with Role [{}] originalPrincipal [{}] failed to get function permissions for " + + "namespace - {}. {}", authParams.getClientRole(), + authParams.getOriginalPrincipal(), namespaceName, e.getMessage(), e); throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index b7883e14c91e8..c7967600da5f6 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -38,7 +38,7 @@ import javax.ws.rs.core.UriBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.FunctionDefinition; @@ -78,8 +78,7 @@ public void registerFunction(final String tenant, final FormDataContentDisposition fileDetail, final String functionPkgUrl, final FunctionConfig functionConfig, - final String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); @@ -98,16 +97,7 @@ public void registerFunction(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Function config is not provided"); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.error("{}/{}/{} Client [{}] is not authorized to register {}", tenant, namespace, - functionName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e); - throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "register", authParams); try { // Check tenant exists @@ -124,8 +114,8 @@ public void registerFunction(final String tenant, } } } catch (PulsarAdminException.NotAuthorizedException e) { - log.error("{}/{}/{} Client [{}] is not authorized to operate {} on tenant", tenant, namespace, - functionName, clientRole, ComponentTypeUtils.toString(componentType)); + log.error("{}/{}/{} Client is not authorized to operate {} on tenant", tenant, namespace, + functionName, ComponentTypeUtils.toString(componentType)); throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } catch (PulsarAdminException.NotFoundException e) { log.error("{}/{}/{} Tenant {} does not exist", tenant, namespace, functionName, tenant); @@ -193,11 +183,12 @@ public void registerFunction(final String tenant, worker().getFunctionRuntimeManager() .getRuntimeFactory() .getAuthProvider().ifPresent(functionAuthProvider -> { - if (clientAuthenticationDataHttps != null) { + if (authParams.getClientAuthenticationDataSource() != null) { try { Optional functionAuthData = functionAuthProvider - .cacheAuthData(finalFunctionDetails, clientAuthenticationDataHttps); + .cacheAuthData(finalFunctionDetails, + authParams.getClientAuthenticationDataSource()); functionAuthData.ifPresent(authData -> functionMetaDataBuilder.setFunctionAuthSpec( Function.FunctionAuthenticationSpec.newBuilder() @@ -245,8 +236,7 @@ public void updateFunction(final String tenant, final FormDataContentDisposition fileDetail, final String functionPkgUrl, final FunctionConfig functionConfig, - final String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps, + final AuthenticationParameters authParams, UpdateOptionsImpl updateOptions) { if (!isWorkerServiceAvailable()) { @@ -266,17 +256,8 @@ public void updateFunction(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Function config is not provided"); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.error("{}/{}/{} Client [{}] is not authorized to update {}", tenant, namespace, - functionName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e); - throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "update", + authParams); FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); @@ -357,7 +338,7 @@ public void updateFunction(final String tenant, worker().getFunctionRuntimeManager() .getRuntimeFactory() .getAuthProvider().ifPresent(functionAuthProvider -> { - if (clientAuthenticationDataHttps != null && updateOptions + if (authParams.getClientAuthenticationDataSource() != null && updateOptions != null && updateOptions.isUpdateAuthData()) { // get existing auth data if it exists Optional existingFunctionAuthData = Optional.empty(); @@ -369,7 +350,7 @@ public void updateFunction(final String tenant, try { Optional newFunctionAuthData = functionAuthProvider .updateAuthData(finalFunctionDetails, existingFunctionAuthData, - clientAuthenticationDataHttps); + authParams.getClientAuthenticationDataSource()); if (newFunctionAuthData.isPresent()) { functionMetaDataBuilder.setFunctionAuthSpec( @@ -597,12 +578,11 @@ public FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData getFunct final String componentName, final String instanceId, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { // validate parameters componentInstanceStatusRequestValidate(tenant, namespace, componentName, Integer.parseInt(instanceId), - clientRole, clientAuthenticationDataHttps); + authParams); FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData functionInstanceStatusData; try { @@ -632,11 +612,10 @@ public FunctionStatus getFunctionStatus(final String tenant, final String namespace, final String componentName, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { // validate parameters - componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps); + componentStatusRequestValidate(tenant, namespace, componentName, authParams); FunctionStatus functionStatus; try { @@ -658,17 +637,18 @@ public void updateFunctionOnWorkerLeader(final String tenant, final InputStream uploadedInputStream, final boolean delete, URI uri, - final String clientRole, - AuthenticationDataSource authenticationData) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } if (worker().getWorkerConfig().isAuthorizationEnabled()) { - if (!isSuperUser(clientRole, authenticationData)) { - log.error("{}/{}/{} Client [{}] is not superuser to update on worker leader {}", tenant, namespace, - functionName, clientRole, ComponentTypeUtils.toString(componentType)); + if (!isSuperUser(authParams)) { + log.error("{}/{}/{} Client with role [{}] and originalPrincipal [{}] is not superuser to update on" + + " worker leader {}", tenant, namespace, functionName, authParams.getClientRole(), + authParams.getClientAuthenticationDataSource(), + ComponentTypeUtils.toString(componentType)); throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } } @@ -713,26 +693,26 @@ public void updateFunctionOnWorkerLeader(final String tenant, } @Override - public void reloadBuiltinFunctions(String clientRole, AuthenticationDataSource authenticationData) + public void reloadBuiltinFunctions(AuthenticationParameters authParams) throws IOException { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole, authenticationData)) { + if (worker().getWorkerConfig().isAuthorizationEnabled() + && !isSuperUser(authParams)) { throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } worker().getFunctionsManager().reloadFunctions(worker().getWorkerConfig()); } @Override - public List getBuiltinFunctions(String clientRole, - AuthenticationDataSource authenticationData) { + public List getBuiltinFunctions(AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole, authenticationData)) { + if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(authParams)) { throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } return this.worker().getFunctionsManager().getFunctionDefinitions(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java index 7b7f84aa82823..059db75542d59 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.FunctionState; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -59,11 +60,11 @@ public FunctionsImplV2(FunctionsImpl delegate) { @Override public Response getFunctionInfo(final String tenant, final String namespace, - final String functionName, String clientRole) + final String functionName, AuthenticationParameters authParams) throws IOException { // run just for parameter checks - delegate.getFunctionInfo(tenant, namespace, functionName, clientRole, null); + delegate.getFunctionInfo(tenant, namespace, functionName, authParams); FunctionMetaDataManager functionMetaDataManager = delegate.worker().getFunctionMetaDataManager(); @@ -75,11 +76,12 @@ public Response getFunctionInfo(final String tenant, final String namespace, @Override public Response getFunctionInstanceStatus(final String tenant, final String namespace, final String functionName, - final String instanceId, URI uri, String clientRole) throws IOException { + final String instanceId, URI uri, + AuthenticationParameters authParams) throws IOException { org.apache.pulsar.common.policies.data.FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData functionInstanceStatus = delegate.getFunctionInstanceStatus(tenant, namespace, - functionName, instanceId, uri, clientRole, null); + functionName, instanceId, uri, authParams); String jsonResponse = FunctionCommon.printJson(toProto(functionInstanceStatus, instanceId)); return Response.status(Response.Status.OK).entity(jsonResponse).build(); @@ -87,10 +89,10 @@ public Response getFunctionInstanceStatus(final String tenant, final String name @Override public Response getFunctionStatusV2(String tenant, String namespace, String functionName, - URI requestUri, String clientRole) throws + URI requestUri, AuthenticationParameters authParams) throws IOException { FunctionStatus functionStatus = delegate.getFunctionStatus(tenant, namespace, - functionName, requestUri, clientRole, null); + functionName, requestUri, authParams); InstanceCommunication.FunctionStatusList.Builder functionStatusList = InstanceCommunication.FunctionStatusList.newBuilder(); functionStatus.instances.forEach(functionInstanceStatus -> functionStatusList.addFunctionStatusList( @@ -103,7 +105,7 @@ public Response getFunctionStatusV2(String tenant, String namespace, String func @Override public Response registerFunction(String tenant, String namespace, String functionName, InputStream uploadedInputStream, FormDataContentDisposition fileDetail, String functionPkgUrl, String - functionDetailsJson, String clientRole) { + functionDetailsJson, AuthenticationParameters authParams) { Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder(); try { @@ -114,14 +116,15 @@ public Response registerFunction(String tenant, String namespace, String functio FunctionConfig functionConfig = FunctionConfigUtils.convertFromDetails(functionDetailsBuilder.build()); delegate.registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionConfig, clientRole, null); + functionPkgUrl, functionConfig, authParams); return Response.ok().build(); } @Override public Response updateFunction(String tenant, String namespace, String functionName, InputStream uploadedInputStream, FormDataContentDisposition fileDetail, - String functionPkgUrl, String functionDetailsJson, String clientRole) { + String functionPkgUrl, String functionDetailsJson, + AuthenticationParameters authParams) { Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder(); try { @@ -132,35 +135,36 @@ public Response updateFunction(String tenant, String namespace, String functionN FunctionConfig functionConfig = FunctionConfigUtils.convertFromDetails(functionDetailsBuilder.build()); delegate.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionConfig, clientRole, null, null); + functionPkgUrl, functionConfig, authParams, null); return Response.ok().build(); } @Override - public Response deregisterFunction(String tenant, String namespace, String functionName, String clientAppId) { - delegate.deregisterFunction(tenant, namespace, functionName, clientAppId, null); + public Response deregisterFunction(String tenant, String namespace, String functionName, + AuthenticationParameters authParams) { + delegate.deregisterFunction(tenant, namespace, functionName, authParams); return Response.ok().build(); } @Override - public Response listFunctions(String tenant, String namespace, String clientRole) { - Collection functionStateList = delegate.listFunctions(tenant, namespace, clientRole, null); + public Response listFunctions(String tenant, String namespace, AuthenticationParameters authParams) { + Collection functionStateList = delegate.listFunctions(tenant, namespace, authParams); return Response.status(Response.Status.OK).entity(new Gson().toJson(functionStateList.toArray())).build(); } @Override public Response triggerFunction(String tenant, String namespace, String functionName, String triggerValue, - InputStream triggerStream, String topic, String clientRole) { + InputStream triggerStream, String topic, AuthenticationParameters authParams) { String result = delegate.triggerFunction(tenant, namespace, functionName, - triggerValue, triggerStream, topic, clientRole, null); + triggerValue, triggerStream, topic, authParams); return Response.status(Response.Status.OK).entity(result).build(); } @Override public Response getFunctionState(String tenant, String namespace, String functionName, - String key, String clientRole) { + String key, AuthenticationParameters authParams) { FunctionState functionState = delegate.getFunctionState( - tenant, namespace, functionName, key, clientRole, null); + tenant, namespace, functionName, key, authParams); String value; if (functionState.getNumberValue() != null) { @@ -175,39 +179,41 @@ public Response getFunctionState(String tenant, String namespace, String functio @Override public Response restartFunctionInstance(String tenant, String namespace, String functionName, String instanceId, URI - uri, String clientRole) { - delegate.restartFunctionInstance(tenant, namespace, functionName, instanceId, uri, clientRole, null); + uri, AuthenticationParameters authParams) { + delegate.restartFunctionInstance(tenant, namespace, functionName, instanceId, uri, authParams); return Response.ok().build(); } @Override - public Response restartFunctionInstances(String tenant, String namespace, String functionName, String clientRole) { - delegate.restartFunctionInstances(tenant, namespace, functionName, clientRole, null); + public Response restartFunctionInstances(String tenant, String namespace, String functionName, + AuthenticationParameters authParams) { + delegate.restartFunctionInstances(tenant, namespace, functionName, authParams); return Response.ok().build(); } @Override public Response stopFunctionInstance(String tenant, String namespace, String functionName, String instanceId, URI - uri, String clientRole) { - delegate.stopFunctionInstance(tenant, namespace, functionName, instanceId, uri, clientRole, null); + uri, AuthenticationParameters authParams) { + delegate.stopFunctionInstance(tenant, namespace, functionName, instanceId, uri, authParams); return Response.ok().build(); } @Override - public Response stopFunctionInstances(String tenant, String namespace, String functionName, String clientRole) { - delegate.stopFunctionInstances(tenant, namespace, functionName, clientRole, null); + public Response stopFunctionInstances(String tenant, String namespace, String functionName, + AuthenticationParameters authParams) { + delegate.stopFunctionInstances(tenant, namespace, functionName, authParams); return Response.ok().build(); } @Override - public Response uploadFunction(InputStream uploadedInputStream, String path, String clientRole) { - delegate.uploadFunction(uploadedInputStream, path, clientRole, null); + public Response uploadFunction(InputStream uploadedInputStream, String path, AuthenticationParameters authParams) { + delegate.uploadFunction(uploadedInputStream, path, authParams); return Response.ok().build(); } @Override - public Response downloadFunction(String path, String clientRole) { - return Response.status(Response.Status.OK).entity(delegate.downloadFunction(path, clientRole, null)).build(); + public Response downloadFunction(String path, AuthenticationParameters authParams) { + return Response.status(Response.Status.OK).entity(delegate.downloadFunction(path, authParams)).build(); } @Override diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index fff376c5dcca1..98450c4a0b5d0 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -38,7 +38,7 @@ import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; @@ -77,8 +77,7 @@ public void registerSink(final String tenant, final FormDataContentDisposition fileDetail, final String sinkPkgUrl, final SinkConfig sinkConfig, - final String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); @@ -97,16 +96,7 @@ public void registerSink(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Sink config is not provided"); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to register {}", tenant, namespace, - sinkName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sinkName, e); - throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sinkName, "register", authParams); try { // Check tenant exists @@ -123,8 +113,8 @@ public void registerSink(final String tenant, } } } catch (PulsarAdminException.NotAuthorizedException e) { - log.error("{}/{}/{} Client [{}] is not authorized to operate {} on tenant", tenant, namespace, - sinkName, clientRole, ComponentTypeUtils.toString(componentType)); + log.error("{}/{}/{} Client is not authorized to operate {} on tenant", tenant, namespace, + sinkName, ComponentTypeUtils.toString(componentType)); throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } catch (PulsarAdminException.NotFoundException e) { log.error("{}/{}/{} Tenant {} does not exist", tenant, namespace, sinkName, tenant); @@ -193,11 +183,12 @@ public void registerSink(final String tenant, worker().getFunctionRuntimeManager() .getRuntimeFactory() .getAuthProvider().ifPresent(functionAuthProvider -> { - if (clientAuthenticationDataHttps != null) { + if (authParams.getClientAuthenticationDataSource() != null) { try { Optional functionAuthData = functionAuthProvider - .cacheAuthData(finalFunctionDetails, clientAuthenticationDataHttps); + .cacheAuthData(finalFunctionDetails, + authParams.getClientAuthenticationDataSource()); functionAuthData.ifPresent(authData -> functionMetaDataBuilder.setFunctionAuthSpec( Function.FunctionAuthenticationSpec.newBuilder() @@ -251,8 +242,7 @@ public void updateSink(final String tenant, final FormDataContentDisposition fileDetail, final String sinkPkgUrl, final SinkConfig sinkConfig, - final String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps, + final AuthenticationParameters authParams, UpdateOptionsImpl updateOptions) { if (!isWorkerServiceAvailable()) { @@ -272,17 +262,7 @@ public void updateSink(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Sink config is not provided"); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to update {}", tenant, namespace, - sinkName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sinkName, e); - throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sinkName, "update", authParams); FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); @@ -363,7 +343,7 @@ public void updateSink(final String tenant, worker().getFunctionRuntimeManager() .getRuntimeFactory() .getAuthProvider().ifPresent(functionAuthProvider -> { - if (clientAuthenticationDataHttps != null && updateOptions != null + if (authParams.getClientAuthenticationDataSource() != null && updateOptions != null && updateOptions.isUpdateAuthData()) { // get existing auth data if it exists Optional existingFunctionAuthData = Optional.empty(); @@ -375,7 +355,7 @@ public void updateSink(final String tenant, try { Optional newFunctionAuthData = functionAuthProvider .updateAuthData(finalFunctionDetails, existingFunctionAuthData, - clientAuthenticationDataHttps); + authParams.getClientAuthenticationDataSource()); if (newFunctionAuthData.isPresent()) { functionMetaDataBuilder.setFunctionAuthSpec( @@ -629,12 +609,11 @@ private ExceptionInformation getExceptionInformation(InstanceCommunication.Funct final String sinkName, final String instanceId, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { // validate parameters - componentInstanceStatusRequestValidate(tenant, namespace, sinkName, Integer.parseInt(instanceId), clientRole, - clientAuthenticationDataHttps); + componentInstanceStatusRequestValidate(tenant, namespace, sinkName, Integer.parseInt(instanceId), + authParams); SinkStatus.SinkInstanceStatus.SinkInstanceStatusData sinkInstanceStatusData; @@ -655,11 +634,10 @@ public SinkStatus getSinkStatus(final String tenant, final String namespace, final String componentName, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { // validate parameters - componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps); + componentStatusRequestValidate(tenant, namespace, componentName, authParams); SinkStatus sinkStatus; try { @@ -677,38 +655,12 @@ public SinkStatus getSinkStatus(final String tenant, @Override public SinkConfig getSinkInfo(final String tenant, final String namespace, - final String componentName) { - - if (!isWorkerServiceAvailable()) { - throwUnavailableException(); - } - - // validate parameters - try { - validateGetFunctionRequestParams(tenant, namespace, componentName, componentType); - } catch (IllegalArgumentException e) { - log.error("Invalid get {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, - namespace, componentName, e); - throw new RestException(Response.Status.BAD_REQUEST, e.getMessage()); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - if (!functionMetaDataManager.containsFunction(tenant, namespace, componentName)) { - log.error("{} does not exist @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, namespace, - componentName); - throw new RestException(Response.Status.NOT_FOUND, - String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName)); - } + final String componentName, + final AuthenticationParameters authParams) { + componentStatusRequestValidate(tenant, namespace, componentName, authParams); Function.FunctionMetaData functionMetaData = - functionMetaDataManager.getFunctionMetaData(tenant, namespace, componentName); - if (!InstanceUtils.calculateSubjectType(functionMetaData.getFunctionDetails()).equals(componentType)) { - log.error("{}/{}/{} is not a {}", tenant, namespace, componentName, - ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.NOT_FOUND, - String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName)); - } - SinkConfig config = SinkConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails()); - return config; + worker().getFunctionMetaDataManager().getFunctionMetaData(tenant, namespace, componentName); + return SinkConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails()); } @Override diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index 2c5921bf7ea9d..876c7e7572e78 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -37,7 +37,7 @@ import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; @@ -76,8 +76,7 @@ public void registerSource(final String tenant, final FormDataContentDisposition fileDetail, final String sourcePkgUrl, final SourceConfig sourceConfig, - final String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); @@ -96,16 +95,7 @@ public void registerSource(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Source config is not provided"); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to register {}", tenant, namespace, - sourceName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sourceName, e); - throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sourceName, "register", authParams); try { // Check tenant exists @@ -122,8 +112,8 @@ public void registerSource(final String tenant, } } } catch (PulsarAdminException.NotAuthorizedException e) { - log.error("{}/{}/{} Client [{}] is not authorized to operate {} on tenant", tenant, namespace, - sourceName, clientRole, ComponentTypeUtils.toString(componentType)); + log.error("{}/{}/{} Client is not authorized to operate {} on tenant", tenant, namespace, + sourceName, ComponentTypeUtils.toString(componentType)); throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); } catch (PulsarAdminException.NotFoundException e) { log.error("{}/{}/{} Tenant {} does not exist", tenant, namespace, sourceName, tenant); @@ -193,11 +183,12 @@ public void registerSource(final String tenant, worker().getFunctionRuntimeManager() .getRuntimeFactory() .getAuthProvider().ifPresent(functionAuthProvider -> { - if (clientAuthenticationDataHttps != null) { + if (authParams.getClientAuthenticationDataSource() != null) { try { Optional functionAuthData = functionAuthProvider - .cacheAuthData(finalFunctionDetails, clientAuthenticationDataHttps); + .cacheAuthData(finalFunctionDetails, + authParams.getClientAuthenticationDataSource()); functionAuthData.ifPresent(authData -> functionMetaDataBuilder.setFunctionAuthSpec( Function.FunctionAuthenticationSpec.newBuilder() @@ -245,8 +236,7 @@ public void updateSource(final String tenant, final FormDataContentDisposition fileDetail, final String sourcePkgUrl, final SourceConfig sourceConfig, - final String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps, + final AuthenticationParameters authParams, UpdateOptionsImpl updateOptions) { if (!isWorkerServiceAvailable()) { @@ -266,17 +256,7 @@ public void updateSource(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Source config is not provided"); } - try { - if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) { - log.warn("{}/{}/{} Client [{}] is not authorized to update {}", tenant, namespace, - sourceName, clientRole, ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - - } - } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sourceName, e); - throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } + throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sourceName, "update", authParams); FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); @@ -357,7 +337,7 @@ public void updateSource(final String tenant, worker().getFunctionRuntimeManager() .getRuntimeFactory() .getAuthProvider().ifPresent(functionAuthProvider -> { - if (clientAuthenticationDataHttps != null && updateOptions != null + if (authParams.getClientAuthenticationDataSource() != null && updateOptions != null && updateOptions.isUpdateAuthData()) { // get existing auth data if it exists Optional existingFunctionAuthData = Optional.empty(); @@ -369,7 +349,7 @@ public void updateSource(final String tenant, try { Optional newFunctionAuthData = functionAuthProvider .updateAuthData(finalFunctionDetails, existingFunctionAuthData, - clientAuthenticationDataHttps); + authParams.getClientAuthenticationDataSource()); if (newFunctionAuthData.isPresent()) { functionMetaDataBuilder.setFunctionAuthSpec( @@ -591,10 +571,10 @@ public SourceStatus emptyStatus(final int parallelism) { public SourceStatus getSourceStatus(final String tenant, final String namespace, final String componentName, - final URI uri, final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final URI uri, + final AuthenticationParameters authParams) { // validate parameters - componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps); + componentStatusRequestValidate(tenant, namespace, componentName, authParams); SourceStatus sourceStatus; try { @@ -616,11 +596,9 @@ public SourceStatus getSourceStatus(final String tenant, final String sourceName, final String instanceId, final URI uri, - final String clientRole, - final AuthenticationDataSource clientAuthenticationDataHttps) { + final AuthenticationParameters authParams) { // validate parameters - componentInstanceStatusRequestValidate(tenant, namespace, sourceName, Integer.parseInt(instanceId), clientRole, - clientAuthenticationDataHttps); + componentInstanceStatusRequestValidate(tenant, namespace, sourceName, Integer.parseInt(instanceId), authParams); SourceStatus.SourceInstanceStatus.SourceInstanceStatusData sourceInstanceStatusData; try { @@ -638,38 +616,12 @@ public SourceStatus getSourceStatus(final String tenant, @Override public SourceConfig getSourceInfo(final String tenant, final String namespace, - final String componentName) { - - if (!isWorkerServiceAvailable()) { - throwUnavailableException(); - } - - // validate parameters - try { - validateGetFunctionRequestParams(tenant, namespace, componentName, componentType); - } catch (IllegalArgumentException e) { - log.error("Invalid get {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, - namespace, componentName, e); - throw new RestException(Response.Status.BAD_REQUEST, e.getMessage()); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - if (!functionMetaDataManager.containsFunction(tenant, namespace, componentName)) { - log.error("{} does not exist @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, namespace, - componentName); - throw new RestException(Response.Status.NOT_FOUND, - String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName)); - } + final String componentName, + final AuthenticationParameters authParams) { + componentStatusRequestValidate(tenant, namespace, componentName, authParams); Function.FunctionMetaData functionMetaData = - functionMetaDataManager.getFunctionMetaData(tenant, namespace, componentName); - if (!InstanceUtils.calculateSubjectType(functionMetaData.getFunctionDetails()).equals(componentType)) { - log.error("{}/{}/{} is not a {}", tenant, namespace, componentName, - ComponentTypeUtils.toString(componentType)); - throw new RestException(Response.Status.NOT_FOUND, - String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName)); - } - SourceConfig config = SourceConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails()); - return config; + worker().getFunctionMetaDataManager().getFunctionMetaData(tenant, namespace, componentName); + return SourceConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails()); } @Override diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java index 364c3ec1ab104..0b77be4ab0212 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.worker.rest.api; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException; import java.io.IOException; import java.net.URI; @@ -27,12 +28,15 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.UriBuilder; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -77,29 +81,24 @@ private boolean isWorkerServiceAvailable() { } @Override - public List getCluster(String clientRole) { + public List getCluster(AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } + throwIfNotSuperUser(authParams, "get cluster"); List workers = worker().getMembershipManager().getCurrentMembership(); return workers; } @Override - public WorkerInfo getClusterLeader(String clientRole) { + public WorkerInfo getClusterLeader(AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - log.error("Client [{}] is not authorized to get cluster leader", clientRole); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } + throwIfNotSuperUser(authParams, "get cluster leader"); MembershipManager membershipManager = worker().getMembershipManager(); WorkerInfo leader = membershipManager.getLeader(); @@ -112,15 +111,12 @@ public WorkerInfo getClusterLeader(String clientRole) { } @Override - public Map> getAssignments(String clientRole) { + public Map> getAssignments(AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - log.error("Client [{}] is not authorized to get cluster assignments", clientRole); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } + throwIfNotSuperUser(authParams, "get cluster assignments"); FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager(); Map> assignments = functionRuntimeManager.getCurrentAssignments(); @@ -131,33 +127,41 @@ public Map> getAssignments(String clientRole) { return ret; } - private boolean isSuperUser(final String clientRole) { - return clientRole != null && worker().getWorkerConfig().getSuperUserRoles().contains(clientRole); + private void throwIfNotSuperUser(AuthenticationParameters authParams, String action) { + if (worker().getWorkerConfig().isAuthorizationEnabled()) { + try { + if (authParams.getClientRole() == null || !worker().getAuthorizationService().isSuperUser(authParams) + .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS)) { + log.error("Client with role [{}] and originalPrincipal [{}] is not authorized to {}", + authParams.getClientRole(), authParams.getOriginalPrincipal(), action); + throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); + } + } catch (ExecutionException | TimeoutException | InterruptedException e) { + log.warn("Time-out {} sec while checking the role {} originalPrincipal {} is a super user role ", + worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), + authParams.getClientRole(), authParams.getOriginalPrincipal()); + throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } } @Override - public List getWorkerMetrics(final String clientRole) { + public List getWorkerMetrics(final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable() || worker().getMetricsGenerator() == null) { throwUnavailableException(); } - - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - log.error("Client [{}] is not authorized to get worker stats", clientRole); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } + throwIfNotSuperUser(authParams, "get worker stats"); return worker().getMetricsGenerator().generate(); } @Override - public List getFunctionsMetrics(String clientRole) throws IOException { + public List getFunctionsMetrics(AuthenticationParameters authParams) + throws IOException { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - log.error("Client [{}] is not authorized to get function stats", clientRole); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } + throwIfNotSuperUser(authParams, "get function stats"); Map functionRuntimes = worker().getFunctionRuntimeManager() .getFunctionRuntimeInfos(); @@ -196,28 +200,20 @@ public List getFunctionsMetrics(String clientRole) } @Override - public List getListOfConnectors(String clientRole) { + public List getListOfConnectors(AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } - + throwIfNotSuperUser(authParams, "get list of connectors"); return this.worker().getConnectorsManager().getConnectorDefinitions(); } @Override - public void rebalance(final URI uri, final String clientRole) { + public void rebalance(final URI uri, final AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } - - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - log.error("Client [{}] is not authorized to rebalance cluster", clientRole); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation"); - } + throwIfNotSuperUser(authParams, "rebalance cluster"); if (worker().getLeaderService().isLeader()) { try { @@ -239,7 +235,8 @@ public void rebalance(final URI uri, final String clientRole) { } @Override - public void drain(final URI uri, final String inWorkerId, final String clientRole, boolean calledOnLeaderUri) { + public void drain(final URI uri, final String inWorkerId, final AuthenticationParameters authParams, + boolean calledOnLeaderUri) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } @@ -248,15 +245,13 @@ public void drain(final URI uri, final String inWorkerId, final String clientRol final String workerId = (inWorkerId == null || inWorkerId.isEmpty()) ? actualWorkerId : inWorkerId; if (log.isDebugEnabled()) { - log.debug("drain called with URI={}, inWorkerId={}, workerId={}, clientRole={}, calledOnLeaderUri={}, " - + "on actual worker-id={}", - uri, inWorkerId, workerId, clientRole, calledOnLeaderUri, actualWorkerId); + log.debug("drain called with URI={}, inWorkerId={}, workerId={}, clientRole={}, originalPrincipal={}, " + + "calledOnLeaderUri={}, on actual worker-id={}", + uri, inWorkerId, workerId, authParams.getClientRole(), authParams.getOriginalPrincipal(), + calledOnLeaderUri, actualWorkerId); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - log.error("Client [{}] is not authorized to drain worker {}", clientRole, workerId); - throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform drain operation"); - } + throwIfNotSuperUser(authParams, "drain worker"); // Depending on which operations we decide to allow, we may add checks here to error/exception if // calledOnLeaderUri is true on a non-leader @@ -285,7 +280,8 @@ public void drain(final URI uri, final String inWorkerId, final String clientRol } @Override - public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWorkerId, final String clientRole, + public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWorkerId, + final AuthenticationParameters authParams, boolean calledOnLeaderUri) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); @@ -296,15 +292,12 @@ public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWor if (log.isDebugEnabled()) { log.debug("getDrainStatus called with uri={}, inWorkerId={}, workerId={}, clientRole={}, " - + " calledOnLeaderUri={}, on actual workerId={}", - uri, inWorkerId, workerId, clientRole, calledOnLeaderUri, actualWorkerId); + + "originalPrincipal={}, calledOnLeaderUri={}, on actual workerId={}", + uri, inWorkerId, workerId, authParams.getClientRole(), authParams.getOriginalPrincipal(), + calledOnLeaderUri, actualWorkerId); } - if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) { - log.error("Client [{}] is not authorized to get drain status of worker {}", clientRole, workerId); - throw new RestException(Status.UNAUTHORIZED, - "Client is not authorized to get the status of a drain operation"); - } + throwIfNotSuperUser(authParams, "get drain status of worker"); // Depending on which operations we decide to allow, we may add checks here to error/exception if // calledOnLeaderUri is true on a non-leader @@ -321,7 +314,7 @@ public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWor } @Override - public Boolean isLeaderReady(final String clientRole) { + public boolean isLeaderReady(AuthenticationParameters authParams) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java index 9176013b783f6..0b125756b30a1 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java @@ -72,7 +72,7 @@ public Response registerFunction(final @PathParam("tenant") String tenant, final @FormDataParam("functionDetails") String functionDetailsJson) { return functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionDetailsJson, clientAppId()); + functionPkgUrl, functionDetailsJson, authParams()); } @PUT @@ -93,7 +93,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant, final @FormDataParam("functionDetails") String functionDetailsJson) { return functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionDetailsJson, clientAppId()); + functionPkgUrl, functionDetailsJson, authParams()); } @@ -110,7 +110,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant, public Response deregisterFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - return functions().deregisterFunction(tenant, namespace, functionName, clientAppId()); + return functions().deregisterFunction(tenant, namespace, functionName, authParams()); } @GET @@ -129,8 +129,7 @@ public Response getFunctionInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws IOException { - return functions().getFunctionInfo( - tenant, namespace, functionName, clientAppId()); + return functions().getFunctionInfo(tenant, namespace, functionName, authParams()); } @GET @@ -151,7 +150,7 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena final @PathParam("instanceId") String instanceId) throws IOException { return functions().getFunctionInstanceStatus(tenant, namespace, functionName, instanceId, uri.getRequestUri(), - clientAppId()); + authParams()); } @GET @@ -168,7 +167,7 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena public Response getFunctionStatus(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws IOException { - return functions().getFunctionStatusV2(tenant, namespace, functionName, uri.getRequestUri(), clientAppId()); + return functions().getFunctionStatusV2(tenant, namespace, functionName, uri.getRequestUri(), authParams()); } @GET @@ -184,7 +183,7 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant, @Path("/{tenant}/{namespace}") public Response listFunctions(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace) { - return functions().listFunctions(tenant, namespace, clientAppId()); + return functions().listFunctions(tenant, namespace, authParams()); } @POST @@ -207,7 +206,7 @@ public Response triggerFunction(final @PathParam("tenant") String tenant, final @FormDataParam("dataStream") InputStream triggerStream, final @FormDataParam("topic") String topic) { return functions().triggerFunction(tenant, namespace, functionName, triggerValue, triggerStream, topic, - clientAppId()); + authParams()); } @GET @@ -226,7 +225,7 @@ public Response getFunctionState(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName, final @PathParam("key") String key) { - return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId()); + return functions().getFunctionState(tenant, namespace, functionName, key, authParams()); } @POST @@ -243,7 +242,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) { return functions().restartFunctionInstance(tenant, namespace, functionName, instanceId, uri.getRequestUri(), - clientAppId()); + authParams()); } @POST @@ -256,7 +255,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant, public Response restartFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - return functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId()); + return functions().restartFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -271,7 +270,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) { return functions().stopFunctionInstance(tenant, namespace, functionName, instanceId, uri.getRequestUri(), - clientAppId()); + authParams()); } @POST @@ -284,7 +283,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant, public Response stopFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - return functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId()); + return functions().stopFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -296,7 +295,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant, @Consumes(MediaType.MULTIPART_FORM_DATA) public Response uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("path") String path) { - return functions().uploadFunction(uploadedInputStream, path, clientAppId()); + return functions().uploadFunction(uploadedInputStream, path, authParams()); } @GET @@ -306,7 +305,7 @@ public Response uploadFunction(final @FormDataParam("data") InputStream uploaded ) @Path("/download") public Response downloadFunction(final @QueryParam("path") String path) { - return functions().downloadFunction(path, clientAppId()); + return functions().downloadFunction(path, authParams()); } @GET diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java index 496f6f43028fe..276d0ae86d876 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java @@ -39,11 +39,14 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.functions.worker.WorkerService; +import org.apache.pulsar.functions.worker.rest.FunctionApiResource; import org.apache.pulsar.functions.worker.service.api.Workers; @Slf4j @@ -75,12 +78,25 @@ Workers workers() { return get().getWorkers(); } + /** + * @deprecated use {@link #authParams()} instead + */ + @Deprecated public String clientAppId() { return httpRequest != null ? (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName) : null; } + public AuthenticationParameters authParams() { + return AuthenticationParameters.builder() + .clientRole(clientAppId()) + .originalPrincipal(httpRequest.getHeader(FunctionApiResource.ORIGINAL_PRINCIPAL_HEADER)) + .clientAuthenticationDataSource((AuthenticationDataSource) + httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName)) + .build(); + } + @GET @ApiOperation( value = "Fetches information about the Pulsar cluster running Pulsar Functions", @@ -94,7 +110,7 @@ public String clientAppId() { @Path("/cluster") @Produces(MediaType.APPLICATION_JSON) public List getCluster() { - return workers().getCluster(clientAppId()); + return workers().getCluster(authParams()); } @GET @@ -109,7 +125,7 @@ public List getCluster() { @Path("/cluster/leader") @Produces(MediaType.APPLICATION_JSON) public WorkerInfo getClusterLeader() { - return workers().getClusterLeader(clientAppId()); + return workers().getClusterLeader(authParams()); } @GET @@ -124,7 +140,7 @@ public WorkerInfo getClusterLeader() { @Path("/assignments") @Produces(MediaType.APPLICATION_JSON) public Map> getAssignments() { - return workers().getAssignments(clientAppId()); + return workers().getAssignments(authParams()); } @GET @@ -139,7 +155,7 @@ public Map> getAssignments() { }) @Path("/connectors") public List getConnectorsList() throws IOException { - return workers().getListOfConnectors(clientAppId()); + return workers().getListOfConnectors(authParams()); } @PUT @@ -153,7 +169,7 @@ public List getConnectorsList() throws IOException { }) @Path("/rebalance") public void rebalance() { - workers().rebalance(uri.getRequestUri(), clientAppId()); + workers().rebalance(uri.getRequestUri(), authParams()); } @PUT @@ -169,7 +185,7 @@ public void rebalance() { }) @Path("/leader/drain") public void drainAtLeader(@QueryParam("workerId") String workerId) { - workers().drain(uri.getRequestUri(), workerId, clientAppId(), true); + workers().drain(uri.getRequestUri(), workerId, authParams(), true); } @PUT @@ -185,7 +201,7 @@ public void drainAtLeader(@QueryParam("workerId") String workerId) { }) @Path("/drain") public void drain() { - workers().drain(uri.getRequestUri(), null, clientAppId(), false); + workers().drain(uri.getRequestUri(), null, authParams(), false); } @GET @@ -199,7 +215,7 @@ public void drain() { }) @Path("/leader/drain") public LongRunningProcessStatus getDrainStatus(@QueryParam("workerId") String workerId) { - return workers().getDrainStatus(uri.getRequestUri(), workerId, clientAppId(), true); + return workers().getDrainStatus(uri.getRequestUri(), workerId, authParams(), true); } @GET @@ -213,7 +229,7 @@ public LongRunningProcessStatus getDrainStatus(@QueryParam("workerId") String wo }) @Path("/drain") public LongRunningProcessStatus getDrainStatus() { - return workers().getDrainStatus(uri.getRequestUri(), null, clientAppId(), false); + return workers().getDrainStatus(uri.getRequestUri(), null, authParams(), false); } @GET @@ -226,6 +242,6 @@ public LongRunningProcessStatus getDrainStatus() { }) @Path("/cluster/leader/ready") public Boolean isLeaderReady() { - return workers().isLeaderReady(clientAppId()); + return workers().isLeaderReady(authParams()); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java index b6a7914f0fdfd..712505a4ba08a 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java @@ -34,9 +34,12 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.common.policies.data.WorkerFunctionInstanceStats; import org.apache.pulsar.functions.worker.WorkerService; +import org.apache.pulsar.functions.worker.rest.FunctionApiResource; import org.apache.pulsar.functions.worker.service.api.Workers; @Slf4j @@ -66,6 +69,19 @@ Workers workers() { return get().getWorkers(); } + AuthenticationParameters authParams() { + return AuthenticationParameters.builder() + .clientRole(clientAppId()) + .originalPrincipal(httpRequest.getHeader(FunctionApiResource.ORIGINAL_PRINCIPAL_HEADER)) + .clientAuthenticationDataSource((AuthenticationDataSource) + httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName)) + .build(); + } + + /** + * @deprecated use {@link AuthenticationParameters} instead + */ + @Deprecated public String clientAppId() { return httpRequest != null ? (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName) @@ -85,7 +101,7 @@ public String clientAppId() { }) @Produces(MediaType.APPLICATION_JSON) public List getMetrics() throws Exception { - return workers().getWorkerMetrics(clientAppId()); + return workers().getWorkerMetrics(authParams()); } @GET @@ -101,6 +117,6 @@ public List getMetrics() throws Exceptio }) @Produces(MediaType.APPLICATION_JSON) public List getStats() throws IOException { - return workers().getFunctionsMetrics(clientAppId()); + return workers().getFunctionsMetrics(authParams()); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java index bca0cf4bf3d75..7bdc86d5fae3e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java @@ -73,7 +73,7 @@ public void registerFunction(final @PathParam("tenant") String tenant, final @FormDataParam("functionConfig") FunctionConfig functionConfig) { functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionConfig, clientAppId(), clientAuthData()); + functionPkgUrl, functionConfig, authParams()); } @@ -90,7 +90,7 @@ public void updateFunction(final @PathParam("tenant") String tenant, final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) { functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, - functionPkgUrl, functionConfig, clientAppId(), clientAuthData(), updateOptions); + functionPkgUrl, functionConfig, authParams(), updateOptions); } @@ -99,7 +99,7 @@ public void updateFunction(final @PathParam("tenant") String tenant, public void deregisterFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - functions().deregisterFunction(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().deregisterFunction(tenant, namespace, functionName, authParams()); } @GET @@ -107,7 +107,7 @@ public void deregisterFunction(final @PathParam("tenant") String tenant, public FunctionConfig getFunctionInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - return functions().getFunctionInfo(tenant, namespace, functionName, clientAppId(), clientAuthData()); + return functions().getFunctionInfo(tenant, namespace, functionName, authParams()); } @GET @@ -115,7 +115,7 @@ public FunctionConfig getFunctionInfo(final @PathParam("tenant") String tenant, @Path("/{tenant}/{namespace}") public List listFunctions(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace) { - return functions().listFunctions(tenant, namespace, clientAppId(), clientAuthData()); + return functions().listFunctions(tenant, namespace, authParams()); } @GET @@ -137,7 +137,7 @@ public FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData getFunct final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) throws IOException { return functions().getFunctionInstanceStatus( - tenant, namespace, functionName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData()); + tenant, namespace, functionName, instanceId, uri.getRequestUri(), authParams()); } @GET @@ -158,7 +158,7 @@ public FunctionStatus getFunctionStatus( final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws IOException { return functions().getFunctionStatus( - tenant, namespace, functionName, uri.getRequestUri(), clientAppId(), clientAuthData()); + tenant, namespace, functionName, uri.getRequestUri(), authParams()); } @GET @@ -178,7 +178,7 @@ public FunctionStatsImpl getFunctionStats(final @PathParam("tenant") String tena final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws IOException { return functions().getFunctionStats(tenant, namespace, functionName, - uri.getRequestUri(), clientAppId(), clientAuthData()); + uri.getRequestUri(), authParams()); } @GET @@ -200,7 +200,7 @@ public FunctionInstanceStatsDataImpl getFunctionInstanceStats( final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) throws IOException { return functions().getFunctionsInstanceStats( - tenant, namespace, functionName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData()); + tenant, namespace, functionName, instanceId, uri.getRequestUri(), authParams()); } @POST @@ -213,7 +213,7 @@ public String triggerFunction(final @PathParam("tenant") String tenant, final @FormDataParam("dataStream") InputStream uploadedInputStream, final @FormDataParam("topic") String topic) { return functions().triggerFunction(tenant, namespace, functionName, input, - uploadedInputStream, topic, clientAppId(), clientAuthData()); + uploadedInputStream, topic, authParams()); } @POST @@ -231,7 +231,7 @@ public void restartFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) { functions().restartFunctionInstance(tenant, namespace, functionName, instanceId, - this.uri.getRequestUri(), clientAppId(), clientAuthData()); + this.uri.getRequestUri(), authParams()); } @POST @@ -246,7 +246,7 @@ public void restartFunction(final @PathParam("tenant") String tenant, public void restartFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().restartFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -263,7 +263,7 @@ public void stopFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) { functions().stopFunctionInstance(tenant, namespace, functionName, instanceId, - this.uri.getRequestUri(), clientAppId(), clientAuthData()); + this.uri.getRequestUri(), authParams()); } @POST @@ -278,7 +278,7 @@ public void stopFunction(final @PathParam("tenant") String tenant, public void stopFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().stopFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -295,7 +295,7 @@ public void startFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) { functions().startFunctionInstance(tenant, namespace, functionName, instanceId, - this.uri.getRequestUri(), clientAppId(), clientAuthData()); + this.uri.getRequestUri(), authParams()); } @POST @@ -310,7 +310,7 @@ public void startFunction(final @PathParam("tenant") String tenant, public void startFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - functions().startFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData()); + functions().startFunctionInstances(tenant, namespace, functionName, authParams()); } @POST @@ -318,13 +318,13 @@ public void startFunction(final @PathParam("tenant") String tenant, @Consumes(MediaType.MULTIPART_FORM_DATA) public void uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("path") String path) { - functions().uploadFunction(uploadedInputStream, path, clientAppId(), clientAuthData()); + functions().uploadFunction(uploadedInputStream, path, authParams()); } @GET @Path("/download") public StreamingOutput downloadFunction(final @QueryParam("path") String path) { - return functions().downloadFunction(path, clientAppId(), clientAuthData()); + return functions().downloadFunction(path, authParams()); } @GET @@ -344,7 +344,7 @@ public StreamingOutput downloadFunction( final @QueryParam("transform-function") boolean transformFunction) { return functions() - .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), transformFunction); + .downloadFunction(tenant, namespace, functionName, authParams(), transformFunction); } @GET @@ -368,7 +368,7 @@ public List getConnectorsList() throws IOException { }) @Path("/builtins/reload") public void reloadBuiltinFunctions() throws IOException { - functions().reloadBuiltinFunctions(clientAppId(), clientAuthData()); + functions().reloadBuiltinFunctions(authParams()); } @GET @@ -385,7 +385,7 @@ public void reloadBuiltinFunctions() throws IOException { @Path("/builtins") @Produces(MediaType.APPLICATION_JSON) public List getBuiltinFunctions() { - return functions().getBuiltinFunctions(clientAppId(), clientAuthData()); + return functions().getBuiltinFunctions(authParams()); } @GET @@ -394,7 +394,7 @@ public FunctionState getFunctionState(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName, final @PathParam("key") String key) throws IOException { - return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId(), clientAuthData()); + return functions().getFunctionState(tenant, namespace, functionName, key, authParams()); } @POST @@ -405,7 +405,7 @@ public void putFunctionState(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @PathParam("key") String key, final @FormDataParam("state") FunctionState stateJson) throws IOException { - functions().putFunctionState(tenant, namespace, functionName, key, stateJson, clientAppId(), clientAuthData()); + functions().putFunctionState(tenant, namespace, functionName, key, stateJson, authParams()); } @PUT @@ -427,6 +427,6 @@ public void updateFunctionOnWorkerLeader(final @PathParam("tenant") String tenan final @FormDataParam("delete") boolean delete) { functions().updateFunctionOnWorkerLeader(tenant, namespace, functionName, uploadedInputStream, - delete, uri.getRequestUri(), clientAppId(), clientAuthData()); + delete, uri.getRequestUri(), authParams()); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java index b110c8f72c677..8081f76cfd8ec 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java @@ -70,7 +70,7 @@ public void registerSink(final @PathParam("tenant") String tenant, final @FormDataParam("sinkConfig") SinkConfig sinkConfig) { sinks().registerSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail, - functionPkgUrl, sinkConfig, clientAppId(), clientAuthData()); + functionPkgUrl, sinkConfig, authParams()); } @PUT @@ -86,7 +86,7 @@ public void updateSink(final @PathParam("tenant") String tenant, final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) { sinks().updateSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail, - functionPkgUrl, sinkConfig, clientAppId(), clientAuthData(), updateOptions); + functionPkgUrl, sinkConfig, authParams(), updateOptions); } @DELETE @@ -94,7 +94,7 @@ public void updateSink(final @PathParam("tenant") String tenant, public void deregisterSink(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName) { - sinks().deregisterFunction(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().deregisterFunction(tenant, namespace, sinkName, authParams()); } @GET @@ -103,7 +103,7 @@ public SinkConfig getSinkInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName) throws IOException { - return sinks().getSinkInfo(tenant, namespace, sinkName); + return sinks().getSinkInfo(tenant, namespace, sinkName, authParams()); } @GET @@ -124,9 +124,8 @@ public SinkStatus.SinkInstanceStatus.SinkInstanceStatusData getSinkInstanceStatu final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName, final @PathParam("instanceId") String instanceId) throws IOException { - return sinks() - .getSinkInstanceStatus(tenant, namespace, sinkName, instanceId, uri.getRequestUri(), clientAppId(), - clientAuthData()); + return sinks().getSinkInstanceStatus(tenant, namespace, sinkName, instanceId, uri.getRequestUri(), + authParams()); } @GET @@ -145,14 +144,14 @@ public SinkStatus.SinkInstanceStatus.SinkInstanceStatusData getSinkInstanceStatu public SinkStatus getSinkStatus(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName) throws IOException { - return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), clientAppId(), clientAuthData()); + return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), authParams()); } @GET @Path("/{tenant}/{namespace}") public List listSink(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace) { - return sinks().listFunctions(tenant, namespace, clientAppId(), clientAuthData()); + return sinks().listFunctions(tenant, namespace, authParams()); } @POST @@ -169,7 +168,7 @@ public void restartSink(final @PathParam("tenant") String tenant, final @PathParam("sinkName") String sinkName, final @PathParam("instanceId") String instanceId) { sinks().restartFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(), - clientAppId(), clientAuthData()); + authParams()); } @POST @@ -182,7 +181,7 @@ public void restartSink(final @PathParam("tenant") String tenant, public void restartSink(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName) { - sinks().restartFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().restartFunctionInstances(tenant, namespace, sinkName, authParams()); } @POST @@ -196,8 +195,8 @@ public void stopSink(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName, final @PathParam("instanceId") String instanceId) { - sinks().stopFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(), clientAppId(), - clientAuthData()); + sinks().stopFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(), + authParams()); } @POST @@ -210,7 +209,7 @@ public void stopSink(final @PathParam("tenant") String tenant, public void stopSink(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName) { - sinks().stopFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().stopFunctionInstances(tenant, namespace, sinkName, authParams()); } @POST @@ -224,8 +223,8 @@ public void startSink(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName, final @PathParam("instanceId") String instanceId) { - sinks().startFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(), clientAppId(), - clientAuthData()); + sinks().startFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(), + authParams()); } @POST @@ -238,7 +237,7 @@ public void startSink(final @PathParam("tenant") String tenant, public void startSink(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sinkName") String sinkName) { - sinks().startFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData()); + sinks().startFunctionInstances(tenant, namespace, sinkName, authParams()); } @GET @@ -278,6 +277,6 @@ public List getSinkConfigDefinition( }) @Path("/reloadBuiltInSinks") public void reloadSinks() { - sinks().reloadConnectors(clientAppId(), clientAuthData()); + sinks().reloadConnectors(authParams()); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java index a669d4f089dd0..23df423045090 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java @@ -70,7 +70,7 @@ public void registerSource(final @PathParam("tenant") String tenant, final @FormDataParam("sourceConfig") SourceConfig sourceConfig) { sources().registerSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail, - functionPkgUrl, sourceConfig, clientAppId(), clientAuthData()); + functionPkgUrl, sourceConfig, authParams()); } @@ -87,7 +87,7 @@ public void updateSource(final @PathParam("tenant") String tenant, final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) { sources().updateSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail, - functionPkgUrl, sourceConfig, clientAppId(), clientAuthData(), updateOptions); + functionPkgUrl, sourceConfig, authParams(), updateOptions); } @@ -96,7 +96,7 @@ public void updateSource(final @PathParam("tenant") String tenant, public void deregisterSource(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sourceName") String sourceName) { - sources().deregisterFunction(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().deregisterFunction(tenant, namespace, sourceName, authParams()); } @GET @@ -106,7 +106,7 @@ public SourceConfig getSourceInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sourceName") String sourceName) throws IOException { - return sources().getSourceInfo(tenant, namespace, sourceName); + return sources().getSourceInfo(tenant, namespace, sourceName, authParams()); } @GET @@ -127,8 +127,8 @@ public SourceStatus.SourceInstanceStatus.SourceInstanceStatusData getSourceInsta final @PathParam("namespace") String namespace, final @PathParam("sourceName") String sourceName, final @PathParam("instanceId") String instanceId) throws IOException { - return sources().getSourceInstanceStatus( - tenant, namespace, sourceName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData()); + return sources().getSourceInstanceStatus(tenant, namespace, sourceName, instanceId, uri.getRequestUri(), + authParams()); } @GET @@ -148,7 +148,7 @@ public SourceStatus getSourceStatus(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sourceName") String sourceName) throws IOException { return sources() - .getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), clientAppId(), clientAuthData()); + .getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), authParams()); } @GET @@ -156,7 +156,7 @@ public SourceStatus getSourceStatus(final @PathParam("tenant") String tenant, @Path("/{tenant}/{namespace}") public List listSources(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace) { - return sources().listFunctions(tenant, namespace, clientAppId(), clientAuthData()); + return sources().listFunctions(tenant, namespace, authParams()); } @POST @@ -173,7 +173,7 @@ public void restartSource(final @PathParam("tenant") String tenant, final @PathParam("sourceName") String sourceName, final @PathParam("instanceId") String instanceId) { sources().restartFunctionInstance(tenant, namespace, sourceName, instanceId, this.uri.getRequestUri(), - clientAppId(), clientAuthData()); + authParams()); } @POST @@ -186,7 +186,7 @@ public void restartSource(final @PathParam("tenant") String tenant, public void restartSource(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sourceName") String sourceName) { - sources().restartFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().restartFunctionInstances(tenant, namespace, sourceName, authParams()); } @POST @@ -201,7 +201,7 @@ public void stopSource(final @PathParam("tenant") String tenant, final @PathParam("sourceName") String sourceName, final @PathParam("instanceId") String instanceId) { sources().stopFunctionInstance(tenant, namespace, sourceName, instanceId, this.uri.getRequestUri(), - clientAppId(), clientAuthData()); + authParams()); } @POST @@ -214,7 +214,7 @@ public void stopSource(final @PathParam("tenant") String tenant, public void stopSource(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sourceName") String sourceName) { - sources().stopFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().stopFunctionInstances(tenant, namespace, sourceName, authParams()); } @POST @@ -229,7 +229,7 @@ public void startSource(final @PathParam("tenant") String tenant, final @PathParam("sourceName") String sourceName, final @PathParam("instanceId") String instanceId) { sources().startFunctionInstance(tenant, namespace, sourceName, instanceId, this.uri.getRequestUri(), - clientAppId(), clientAuthData()); + authParams()); } @POST @@ -242,7 +242,7 @@ public void startSource(final @PathParam("tenant") String tenant, public void startSource(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("sourceName") String sourceName) { - sources().startFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData()); + sources().startFunctionInstances(tenant, namespace, sourceName, authParams()); } @GET @@ -293,6 +293,6 @@ public List getSourceConfigDefinition( }) @Path("/reloadBuiltInSources") public void reloadSources() { - sources().reloadConnectors(clientAppId(), clientAuthData()); + sources().reloadConnectors(authParams()); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java index fc01ec55b6c4c..770cced1731da 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java @@ -22,8 +22,7 @@ import java.net.URI; import java.util.List; import javax.ws.rs.core.StreamingOutput; -import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.FunctionState; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -40,167 +39,55 @@ public interface Component { W worker(); - void deregisterFunction(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - @Deprecated - default void deregisterFunction(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps) { - deregisterFunction( - tenant, - namespace, - componentName, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps); - } - - FunctionConfig getFunctionInfo(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void stopFunctionInstance(String tenant, - String namespace, - String componentName, - String instanceId, - URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void startFunctionInstance(String tenant, - String namespace, - String componentName, - String instanceId, - URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void restartFunctionInstance(String tenant, - String namespace, - String componentName, - String instanceId, - URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void startFunctionInstances(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void stopFunctionInstances(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void restartFunctionInstances(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - FunctionStatsImpl getFunctionStats(String tenant, - String namespace, - String componentName, - URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - FunctionInstanceStatsDataImpl getFunctionsInstanceStats(String tenant, - String namespace, - String componentName, - String instanceId, - URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - String triggerFunction(String tenant, - String namespace, - String functionName, - String input, - InputStream uploadedInputStream, - String topic, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - List listFunctions(String tenant, - String namespace, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - FunctionState getFunctionState(String tenant, - String namespace, - String functionName, - String key, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void putFunctionState(String tenant, - String namespace, - String functionName, - String key, - FunctionState state, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - void uploadFunction(InputStream uploadedInputStream, - String path, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - StreamingOutput downloadFunction(String path, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - @Deprecated - default StreamingOutput downloadFunction(String path, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps) { - return downloadFunction(path, clientRole, (AuthenticationDataSource) clientAuthenticationDataHttps); - } - - StreamingOutput downloadFunction(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps, - boolean transformFunction); - - @Deprecated - default StreamingOutput downloadFunction(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps) { - return downloadFunction(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps, false); - } - - @Deprecated - default StreamingOutput downloadFunction(String tenant, - String namespace, - String componentName, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps) { - return downloadFunction( - tenant, - namespace, - componentName, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps, - false); - } + void deregisterFunction(String tenant, String namespace, String componentName, AuthenticationParameters authParams); - List getListOfConnectors(); + FunctionConfig getFunctionInfo(String tenant, String namespace, String componentName, + AuthenticationParameters authParams); + + void stopFunctionInstance(String tenant, String namespace, String componentName, String instanceId, URI uri, + AuthenticationParameters authParams); + + void startFunctionInstance(String tenant, String namespace, String componentName, String instanceId, URI uri, + AuthenticationParameters authParams); + + void restartFunctionInstance(String tenant, String namespace, String componentName, String instanceId, URI uri, + AuthenticationParameters authParams); + + void startFunctionInstances(String tenant, String namespace, String componentName, + AuthenticationParameters authParams); + + void stopFunctionInstances(String tenant, String namespace, String componentName, + AuthenticationParameters authParams); + + void restartFunctionInstances(String tenant, String namespace, String componentName, + AuthenticationParameters authParams); + + FunctionStatsImpl getFunctionStats(String tenant, String namespace, String componentName, URI uri, + AuthenticationParameters authParams); + + FunctionInstanceStatsDataImpl getFunctionsInstanceStats(String tenant, String namespace, String componentName, + String instanceId, URI uri, + AuthenticationParameters authParams); + String triggerFunction(String tenant, String namespace, String functionName, String input, + InputStream uploadedInputStream, String topic, AuthenticationParameters authParams); + + List listFunctions(String tenant, String namespace, AuthenticationParameters authParams); + + FunctionState getFunctionState(String tenant, String namespace, String functionName, String key, + AuthenticationParameters authParams); + + void putFunctionState(String tenant, String namespace, String functionName, String key, FunctionState state, + AuthenticationParameters authParams); + + void uploadFunction(InputStream uploadedInputStream, String path, AuthenticationParameters authParams); + + StreamingOutput downloadFunction(String path, AuthenticationParameters authParams); + + StreamingOutput downloadFunction(String tenant, String namespace, String componentName, + AuthenticationParameters authParams, boolean transformFunction); + + List getListOfConnectors(); - void reloadConnectors(String clientRole, AuthenticationDataSource clientAuthenticationDataHttps); + void reloadConnectors(AuthenticationParameters authParams); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java index 954ab1d26182f..28bc73fd0a831 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java @@ -22,8 +22,7 @@ import java.io.InputStream; import java.net.URI; import java.util.List; -import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.UpdateOptionsImpl; @@ -46,8 +45,7 @@ public interface Functions extends Component { * @param fileDetail A form-data content disposition header * @param functionPkgUrl URL path of the Pulsar Function package * @param functionConfig Configuration of Pulsar Function - * @param clientRole Client role for running the pulsar function - * @param clientAuthenticationDataHttps Authentication status of the http client + * @param authParams the authentication parameters associated with the request */ void registerFunction(String tenant, String namespace, @@ -56,35 +54,7 @@ void registerFunction(String tenant, FormDataContentDisposition fileDetail, String functionPkgUrl, FunctionConfig functionConfig, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - /** - * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status, - * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts - * the signature of the AuthenticationDataSource. - */ - @Deprecated - default void registerFunction(String tenant, - String namespace, - String functionName, - InputStream uploadedInputStream, - FormDataContentDisposition fileDetail, - String functionPkgUrl, - FunctionConfig functionConfig, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps) { - registerFunction( - tenant, - namespace, - functionName, - uploadedInputStream, - fileDetail, - functionPkgUrl, - functionConfig, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps); - } + AuthenticationParameters authParams); /** * Update a function. @@ -95,8 +65,7 @@ default void registerFunction(String tenant, * @param fileDetail A form-data content disposition header * @param functionPkgUrl URL path of the Pulsar Function package * @param functionConfig Configuration of Pulsar Function - * @param clientRole Client role for running the Pulsar Function - * @param clientAuthenticationDataHttps Authentication status of the http client + * @param authParams the authentication parameters associated with the request * @param updateOptions Options while updating the function */ void updateFunction(String tenant, @@ -106,66 +75,31 @@ void updateFunction(String tenant, FormDataContentDisposition fileDetail, String functionPkgUrl, FunctionConfig functionConfig, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps, + AuthenticationParameters authParams, UpdateOptionsImpl updateOptions); - /** - * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status, - * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts - * the signature of the AuthenticationDataSource. - */ - @Deprecated - default void updateFunction(String tenant, - String namespace, - String functionName, - InputStream uploadedInputStream, - FormDataContentDisposition fileDetail, - String functionPkgUrl, - FunctionConfig functionConfig, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps, - UpdateOptionsImpl updateOptions) { - updateFunction( - tenant, - namespace, - functionName, - uploadedInputStream, - fileDetail, - functionPkgUrl, - functionConfig, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps, - updateOptions); - } - void updateFunctionOnWorkerLeader(String tenant, String namespace, String functionName, InputStream uploadedInputStream, boolean delete, URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + AuthenticationParameters authParams); FunctionStatus getFunctionStatus(String tenant, String namespace, String componentName, URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + AuthenticationParameters authParams); FunctionInstanceStatusData getFunctionInstanceStatus(String tenant, String namespace, String componentName, String instanceId, URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + AuthenticationParameters authParams); - void reloadBuiltinFunctions(String clientRole, AuthenticationDataSource clientAuthenticationDataHttps) - throws IOException; + void reloadBuiltinFunctions(AuthenticationParameters authParams) throws IOException; - List getBuiltinFunctions(String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + List getBuiltinFunctions(AuthenticationParameters authParams); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java index 6a4be7cdf6f0e..d78d241b3e6a0 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java @@ -23,6 +23,7 @@ import java.net.URI; import java.util.List; import javax.ws.rs.core.Response; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.functions.worker.WorkerService; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -35,20 +36,20 @@ public interface FunctionsV2 { Response getFunctionInfo(String tenant, String namespace, String functionName, - String clientRole) throws IOException; + AuthenticationParameters authParams) throws IOException; Response getFunctionInstanceStatus(String tenant, - String namespace, - String functionName, - String instanceId, - URI uri, - String clientRole) throws IOException; + String namespace, + String functionName, + String instanceId, + URI uri, + AuthenticationParameters authParams) throws IOException; Response getFunctionStatusV2(String tenant, String namespace, String functionName, URI requestUri, - String clientRole) throws IOException; + AuthenticationParameters authParams) throws IOException; Response registerFunction(String tenant, String namespace, @@ -57,7 +58,8 @@ Response registerFunction(String tenant, FormDataContentDisposition fileDetail, String functionPkgUrl, String functionDetailsJson, - String clientRole); + AuthenticationParameters authParams); + Response updateFunction(String tenant, String namespace, @@ -66,14 +68,12 @@ Response updateFunction(String tenant, FormDataContentDisposition fileDetail, String functionPkgUrl, String functionDetailsJson, - String clientRole); + AuthenticationParameters authParams); - Response deregisterFunction(String tenant, - String namespace, - String functionName, - String clientAppId); + Response deregisterFunction(String tenant, String namespace, String functionName, + AuthenticationParameters authParams); - Response listFunctions(String tenant, String namespace, String clientRole); + Response listFunctions(String tenant, String namespace, AuthenticationParameters authParams); Response triggerFunction(String tenant, String namespace, @@ -81,45 +81,44 @@ Response triggerFunction(String tenant, String triggerValue, InputStream triggerStream, String topic, - String clientRole); + AuthenticationParameters authParams); Response getFunctionState(String tenant, String namespace, String functionName, String key, - String clientRole); - + AuthenticationParameters authParams); Response restartFunctionInstance(String tenant, String namespace, String functionName, String instanceId, URI uri, - String clientRole); + AuthenticationParameters authParams); + Response restartFunctionInstances(String tenant, String namespace, String functionName, - String clientRole); + AuthenticationParameters authParams); Response stopFunctionInstance(String tenant, String namespace, String functionName, String instanceId, URI uri, - String clientRole); + AuthenticationParameters authParams); Response stopFunctionInstances(String tenant, String namespace, String functionName, - String clientRole); + AuthenticationParameters authParams); Response uploadFunction(InputStream uploadedInputStream, String path, - String clientRole); - - Response downloadFunction(String path, String clientRole); + AuthenticationParameters authParams); + Response downloadFunction(String path, AuthenticationParameters authParams); List getListOfConnectors(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java index 2c0dcb82e0dbc..c977f55df4bf3 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java @@ -21,8 +21,7 @@ import java.io.InputStream; import java.net.URI; import java.util.List; -import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -46,8 +45,7 @@ public interface Sinks extends Component { * @param fileDetail A form-data content disposition header * @param sinkPkgUrl URL path of the Pulsar Sink package * @param sinkConfig Configuration of Pulsar Sink - * @param clientRole Client role for running the Pulsar Sink - * @param clientAuthenticationDataHttps Authentication status of the http client + * @param authParams the authentication parameters associated with the request */ void registerSink(String tenant, String namespace, @@ -56,35 +54,7 @@ void registerSink(String tenant, FormDataContentDisposition fileDetail, String sinkPkgUrl, SinkConfig sinkConfig, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - /** - * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status, - * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts - * the signature of the AuthenticationDataSource. - */ - @Deprecated - default void registerSink(String tenant, - String namespace, - String sinkName, - InputStream uploadedInputStream, - FormDataContentDisposition fileDetail, - String sinkPkgUrl, - SinkConfig sinkConfig, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps) { - registerSink( - tenant, - namespace, - sinkName, - uploadedInputStream, - fileDetail, - sinkPkgUrl, - sinkConfig, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps); - } + AuthenticationParameters authParams); /** * Update a function. @@ -95,8 +65,7 @@ default void registerSink(String tenant, * @param fileDetail A form-data content disposition header * @param sinkPkgUrl URL path of the Pulsar Sink package * @param sinkConfig Configuration of Pulsar Sink - * @param clientRole Client role for running the Pulsar Sink - * @param clientAuthenticationDataHttps Authentication status of the http client + * @param authParams the authentication parameters associated with the request * @param updateOptions Options while updating the sink */ void updateSink(String tenant, @@ -106,57 +75,26 @@ void updateSink(String tenant, FormDataContentDisposition fileDetail, String sinkPkgUrl, SinkConfig sinkConfig, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps, + AuthenticationParameters authParams, UpdateOptionsImpl updateOptions); - /** - * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status, - * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts - * the signature of the AuthenticationDataSource. - */ - @Deprecated - default void updateSink(String tenant, - String namespace, - String sinkName, - InputStream uploadedInputStream, - FormDataContentDisposition fileDetail, - String sinkPkgUrl, - SinkConfig sinkConfig, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps, - UpdateOptionsImpl updateOptions) { - updateSink( - tenant, - namespace, - sinkName, - uploadedInputStream, - fileDetail, - sinkPkgUrl, - sinkConfig, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps, - updateOptions); - } - SinkInstanceStatusData getSinkInstanceStatus(String tenant, String namespace, String sinkName, String instanceId, URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + AuthenticationParameters authParams); SinkStatus getSinkStatus(String tenant, String namespace, String componentName, URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + AuthenticationParameters authParams); SinkConfig getSinkInfo(String tenant, String namespace, - String componentName); + String componentName, + AuthenticationParameters authParams); List getSinkList(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java index 781a25e92cc87..7fe1e56bf9a13 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java @@ -21,8 +21,7 @@ import java.io.InputStream; import java.net.URI; import java.util.List; -import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -46,8 +45,7 @@ public interface Sources extends Component { * @param fileDetail A form-data content disposition header * @param sourcePkgUrl URL path of the Pulsar Source package * @param sourceConfig Configuration of Pulsar Source - * @param clientRole Client role for running the Pulsar Source - * @param clientAuthenticationDataHttps Authentication status of the http client + * @param authParams the authentication parameters associated with the request */ void registerSource(String tenant, String namespace, @@ -56,35 +54,7 @@ void registerSource(String tenant, FormDataContentDisposition fileDetail, String sourcePkgUrl, SourceConfig sourceConfig, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - - /** - * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status, - * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts - * the signature of the AuthenticationDataSource. - */ - @Deprecated - default void registerSource(String tenant, - String namespace, - String sourceName, - InputStream uploadedInputStream, - FormDataContentDisposition fileDetail, - String sourcePkgUrl, - SourceConfig sourceConfig, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps) { - registerSource( - tenant, - namespace, - sourceName, - uploadedInputStream, - fileDetail, - sourcePkgUrl, - sourceConfig, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps); - } + AuthenticationParameters authParams); /** * Update a function. @@ -95,8 +65,7 @@ default void registerSource(String tenant, * @param fileDetail A form-data content disposition header * @param sourcePkgUrl URL path of the Pulsar Source package * @param sourceConfig Configuration of Pulsar Source - * @param clientRole Client role for running the Pulsar Source - * @param clientAuthenticationDataHttps Authentication status of the http client + * @param authParams the authentication parameters associated with the request * @param updateOptions Options while updating the source */ void updateSource(String tenant, @@ -106,59 +75,26 @@ void updateSource(String tenant, FormDataContentDisposition fileDetail, String sourcePkgUrl, SourceConfig sourceConfig, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps, + AuthenticationParameters authParams, UpdateOptionsImpl updateOptions); - /** - * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status, - * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts - * the signature of the AuthenticationDataSource. - */ - @Deprecated - default void updateSource(String tenant, - String namespace, - String sourceName, - InputStream uploadedInputStream, - FormDataContentDisposition fileDetail, - String sourcePkgUrl, - SourceConfig sourceConfig, - String clientRole, - AuthenticationDataHttps clientAuthenticationDataHttps, - UpdateOptionsImpl updateOptions) { - updateSource( - tenant, - namespace, - sourceName, - uploadedInputStream, - fileDetail, - sourcePkgUrl, - sourceConfig, - clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps, - updateOptions); - } - - SourceStatus getSourceStatus(String tenant, String namespace, String componentName, URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); - + AuthenticationParameters authParams); SourceInstanceStatusData getSourceInstanceStatus(String tenant, String namespace, String sourceName, String instanceId, URI uri, - String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + AuthenticationParameters authParams); SourceConfig getSourceInfo(String tenant, String namespace, - String componentName); + String componentName, + AuthenticationParameters authParams); List getSourceList(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java index a13157ea6476c..420fdcefe8b57 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -35,25 +36,25 @@ */ public interface Workers { - List getCluster(String clientRole); + List getCluster(AuthenticationParameters authParams); - WorkerInfo getClusterLeader(String clientRole); + WorkerInfo getClusterLeader(AuthenticationParameters authParams); - Map> getAssignments(String clientRole); + Map> getAssignments(AuthenticationParameters authParams); - List getWorkerMetrics(String clientRole); + List getWorkerMetrics(AuthenticationParameters authParams); - List getFunctionsMetrics(String clientRole) throws IOException; + List getFunctionsMetrics(AuthenticationParameters authParams) throws IOException; - List getListOfConnectors(String clientRole); + List getListOfConnectors(AuthenticationParameters authParams); - void rebalance(URI uri, String clientRole); + void rebalance(URI uri, AuthenticationParameters authParams); - void drain(URI uri, String workerId, String clientRole, boolean leaderUri); + void drain(URI uri, String workerId, AuthenticationParameters authParams, boolean leaderUri); - LongRunningProcessStatus getDrainStatus(URI uri, String workerId, String clientRole, + LongRunningProcessStatus getDrainStatus(URI uri, String workerId, AuthenticationParameters authParams, boolean leaderUri); - Boolean isLeaderReady(String clientRole); + boolean isLeaderReady(AuthenticationParameters authParams); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java index c5e6b4aaded74..0f5fca4a8a5a3 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java @@ -33,17 +33,20 @@ import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; +import java.util.HashSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import org.apache.distributedlog.DistributedLogConfiguration; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerAccessMode; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.testng.annotations.Test; public class WorkerUtilsTest { @@ -118,4 +121,15 @@ public void testDLogConfiguration() throws URISyntaxException, IOException { assertEquals(dlogConf.getString("bkc.testKey"), "fakeValue", "The bookkeeper client config mapping should apply."); } + + @Test + public void testProxyRolesInWorkerConfigMapToServiceConfiguration() throws Exception { + URL yamlUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); + WorkerConfig wc = WorkerConfig.load(yamlUrl.toURI().getPath()); + ServiceConfiguration conf = PulsarConfigurationLoader.convertFrom(wc); + HashSet proxyRoles = new HashSet<>(); + proxyRoles.add("proxyA"); + proxyRoles.add("proxyB"); + assertEquals(conf.getProxyRoles(), proxyRoles); + } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java index 998d85683e204..cfe087c78406a 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.functions.worker.rest.api; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.doReturn; @@ -28,27 +27,39 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import java.io.InputStream; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.broker.resources.TenantResources; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl; import org.apache.pulsar.common.policies.data.FunctionStatsImpl; +import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.instance.InstanceConfig; import org.apache.pulsar.functions.instance.JavaInstanceRunnable; @@ -94,6 +105,7 @@ public String process(String input, Context context) { private static final int parallelism = 1; private static final String workerId = "worker-0"; private static final String superUser = "superUser"; + private static final String proxyUser = "proxyUser"; private PulsarWorkerService mockedWorkerService; private PulsarAdmin mockedPulsarAdmin; @@ -196,7 +208,7 @@ public void cleanup() { @Test public void testStatusEmpty() { - assertNotNull(this.resource.getFunctionInstanceStatus(tenant, namespace, function, "0", null, null, null)); + assertNotNull(this.resource.getFunctionInstanceStatus(tenant, namespace, function, "0", null, null)); } @Test @@ -231,83 +243,80 @@ public void testMetricsEmpty() throws PulsarClientException { assertNotNull(functionStats.calculateOverall()); } + // Suppress the deprecation warnings until we actually remove the deprecated method + @SuppressWarnings("deprecation") @Test - public void testIsAuthorizedRole() throws PulsarAdminException, InterruptedException, ExecutionException { + public void testIsAuthorizedRole() throws Exception { - TenantInfo tenantInfo = TenantInfo.builder().build(); AuthenticationDataSource authenticationDataSource = mock(AuthenticationDataSource.class); FunctionsImpl functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService)); - AuthorizationService authorizationService = mock(AuthorizationService.class); - doReturn(authorizationService).when(mockedWorkerService).getAuthorizationService(); WorkerConfig workerConfig = new WorkerConfig(); workerConfig.setAuthorizationEnabled(true); - workerConfig.setSuperUserRoles(Collections.singleton(superUser)); + HashSet superUsers = new HashSet<>(); + superUsers.add(superUser); + superUsers.add(proxyUser); + workerConfig.setSuperUserRoles(superUsers); + workerConfig.setProxyRoles(Collections.singleton(proxyUser)); + // TODO remove mocking by relying on TestPulsarResources. Can't do now because this commit needs to be + // cherry picked back. + PulsarResources pulsarResources = mock(PulsarResources.class); + TenantResources tenantResources = mock(TenantResources.class); + when(pulsarResources.getTenantResources()).thenReturn(tenantResources); + TenantInfo tenantInfo = TenantInfo.builder().adminRoles(Collections.singleton("tenant-admin")).build(); + when(tenantResources.getTenantAsync("test-tenant")) + .thenReturn(CompletableFuture.completedFuture(Optional.of(tenantInfo))); + NamespaceResources namespaceResources = mock(NamespaceResources.class); + when(pulsarResources.getNamespaceResources()).thenReturn(namespaceResources); + Policies p = new Policies(); + p.auth_policies.getNamespaceAuthentication().put("test-function-user", Set.of(AuthAction.functions)); + when(namespaceResources.getPoliciesAsync(NamespaceName.get("test-tenant/test-ns"))) + .thenReturn(CompletableFuture.completedFuture(Optional.of(p))); + + AuthorizationService authorizationService = new AuthorizationService( + PulsarConfigurationLoader.convertFrom(workerConfig), pulsarResources); doReturn(workerConfig).when(mockedWorkerService).getWorkerConfig(); + doReturn(authorizationService).when(mockedWorkerService).getAuthorizationService(); // test super user - assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", superUser, authenticationDataSource)); - - // test pulsar super user - final String pulsarSuperUser = "pulsarSuperUser"; - when(authorizationService.isSuperUser(eq(pulsarSuperUser), any())) - .thenReturn(CompletableFuture.completedFuture(true)); - assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", pulsarSuperUser, authenticationDataSource)); - assertTrue(functionImpl.isSuperUser(pulsarSuperUser, null)); + assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", superUser, + authenticationDataSource)); + assertTrue(functionImpl.isSuperUser(superUser, null)); - // test normal user - functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService)); - doReturn(false).when(functionImpl).allowFunctionOps(any(), any(), any()); - Tenants tenants = mock(Tenants.class); - when(tenants.getTenantInfo(any())).thenReturn(tenantInfo); - PulsarAdmin admin = mock(PulsarAdmin.class); - when(admin.tenants()).thenReturn(tenants); - when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin); - when(authorizationService.isTenantAdmin("test-tenant", "test-user", tenantInfo, authenticationDataSource)) - .thenReturn(CompletableFuture.completedFuture(false)); - when(authorizationService.isSuperUser(eq("test-user"), any())) - .thenReturn(CompletableFuture.completedFuture(false)); - assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-user", authenticationDataSource)); + // test normal user with no permissions + assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-non-admin-user", + authenticationDataSource)); // if user is tenant admin - functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService)); - doReturn(false).when(functionImpl).allowFunctionOps(any(), any(), any()); - tenants = mock(Tenants.class); - tenantInfo = TenantInfo.builder().adminRoles(Collections.singleton("test-user")).build(); - when(tenants.getTenantInfo(any())).thenReturn(tenantInfo); - - admin = mock(PulsarAdmin.class); - when(admin.tenants()).thenReturn(tenants); - when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin); - when(authorizationService.isTenantAdmin("test-tenant", "test-user", tenantInfo, authenticationDataSource)) - .thenReturn(CompletableFuture.completedFuture(true)); - when(authorizationService.isSuperUser("test-user", authenticationDataSource)) - .thenReturn(CompletableFuture.completedFuture(false)); - assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-user", authenticationDataSource)); + assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "tenant-admin", + authenticationDataSource)); // test user allow function action - functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService)); - doReturn(true).when(functionImpl).allowFunctionOps(any(), any(), any()); - tenants = mock(Tenants.class); - tenantInfo = TenantInfo.builder().build(); - when(tenants.getTenantInfo(any())).thenReturn(tenantInfo); - - admin = mock(PulsarAdmin.class); - when(admin.tenants()).thenReturn(tenants); - when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin); - when(authorizationService.isTenantAdmin("test-tenant", "test-user", tenantInfo, authenticationDataSource)) - .thenReturn(CompletableFuture.completedFuture(true)); - assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-user", authenticationDataSource)); + assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-function-user", + authenticationDataSource)); // test role is null - functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService)); - doReturn(true).when(functionImpl).allowFunctionOps(any(), any(), any()); - tenants = mock(Tenants.class); - when(tenants.getTenantInfo(any())).thenReturn(TenantInfo.builder().build()); - - admin = mock(PulsarAdmin.class); - when(admin.tenants()).thenReturn(tenants); - when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin); - assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", null, authenticationDataSource)); + assertThrows(RestException.class, () -> functionImpl.isAuthorizedRole("test-tenant", + "test-ns", null, authenticationDataSource)); + + // test proxy user with no original principal + assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", + AuthenticationParameters.builder().clientRole(proxyUser).build())); + + // test proxy user with tenant admin original principal + assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", + AuthenticationParameters.builder().clientRole(proxyUser).originalPrincipal("tenant-admin").build())); + + // test proxy user with non admin user + assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", + AuthenticationParameters.builder().clientRole(proxyUser).originalPrincipal("test-non-admin-user").build())); + + // test proxy user with allow function action + assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", + AuthenticationParameters.builder().clientRole(proxyUser).originalPrincipal("test-function-user").build())); + + // test non-proxy user passing original principal + assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", + AuthenticationParameters.builder().clientRole("nobody").originalPrincipal("test-non-admin-user").build())); } @Test @@ -320,20 +329,19 @@ public void testIsSuperUser() throws PulsarAdminException { workerConfig.setAuthorizationEnabled(true); workerConfig.setSuperUserRoles(Collections.singleton(superUser)); doReturn(workerConfig).when(mockedWorkerService).getWorkerConfig(); - when(authorizationService.isSuperUser(anyString(), any())) + when(authorizationService.isSuperUser(any(AuthenticationParameters.class))) .thenAnswer((invocationOnMock) -> { - String role = invocationOnMock.getArgument(0, String.class); + String role = invocationOnMock.getArgument(0, AuthenticationParameters.class).getClientRole(); return CompletableFuture.completedFuture(superUser.equals(role)); }); - AuthenticationDataSource authenticationDataSource = mock(AuthenticationDataSource.class); assertTrue(functionImpl.isSuperUser(superUser, null)); assertFalse(functionImpl.isSuperUser("normal-user", null)); assertFalse(functionImpl.isSuperUser(null, null)); // test super roles is null and it's not a pulsar super user - when(authorizationService.isSuperUser(superUser, null)) + when(authorizationService.isSuperUser(AuthenticationParameters.builder().clientRole(superUser).build())) .thenReturn(CompletableFuture.completedFuture(false)); functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService)); workerConfig = new WorkerConfig(); @@ -342,10 +350,10 @@ public void testIsSuperUser() throws PulsarAdminException { assertFalse(functionImpl.isSuperUser(superUser, null)); // test super role is null but the auth datasource contains superuser - when(authorizationService.isSuperUser(anyString(), any(AuthenticationDataSource.class))) + when(authorizationService.isSuperUser(any(AuthenticationParameters.class))) .thenAnswer((invocationOnMock -> { - AuthenticationDataSource authData = invocationOnMock.getArgument(1, AuthenticationDataSource.class); - String user = authData.getHttpHeader("mockedUser"); + AuthenticationParameters authData = invocationOnMock.getArgument(0, AuthenticationParameters.class); + String user = authData.getClientAuthenticationDataSource().getHttpHeader("mockedUser"); return CompletableFuture.completedFuture(superUser.equals(user)); })); AuthenticationDataSource authData = mock(AuthenticationDataSource.class); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java new file mode 100644 index 0000000000000..eb7228383552c --- /dev/null +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker.rest.api; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.util.HashSet; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; +import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.util.RestException; +import org.apache.pulsar.functions.worker.LeaderService; +import org.apache.pulsar.functions.worker.MetricsGenerator; +import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Unit test of {@link WorkerImpl} to ensure that all methods properly fail if the {@link AuthenticationParameters} + * do not represent a superuser. + */ +public class WorkerImplTest { + WorkerImpl worker; + + @DataProvider(name = "authParamsForNonSuperusers") + public Object[][] partitionedTopicProvider() { + return new Object[][] { + { AuthenticationParameters.builder().build() }, // all fields null should fail + { AuthenticationParameters.builder().clientRole("user").build() }, // Not superuser, no proxy + { AuthenticationParameters.builder() + .clientRole("proxySuperuser").build() }, // Proxy role needs original principal + { AuthenticationParameters.builder() + .clientRole("proxyNotSuperuser").build() }, // Proxy role needs original principal + { AuthenticationParameters.builder().clientRole("proxyNotSuperuser") + .originalPrincipal("superuser").build() }, // Proxy is not superuser + { AuthenticationParameters.builder().clientRole("proxyNotSuperuser") + .originalPrincipal("user").build() }, // Neither proxy nor original principal is superuser + { AuthenticationParameters.builder().clientRole("proxySuperuser") + .originalPrincipal("user").build() }, // Original principal is not superuser + { AuthenticationParameters.builder().clientRole("user") + .originalPrincipal("superuser").build() }, // User is not a proxyRole + { AuthenticationParameters.builder().clientRole("superuser2") + .originalPrincipal("superuser").build() }, // Both are superusers, but client is not a proxyRole + }; + } + + @BeforeClass + void setup() throws Exception { + WorkerConfig config = new WorkerConfig(); + config.setPulsarFunctionsCluster("testThrowIfNotSuperUserFailures"); + config.setAuthorizationEnabled(true); + HashSet proxyRoles = new HashSet<>(); + proxyRoles.add("proxySuperuser"); + proxyRoles.add("proxyNotSuperuser"); + HashSet superUserRoles = new HashSet<>(); + superUserRoles.add("superuser"); + superUserRoles.add("superuser2"); + superUserRoles.add("proxySuperuser"); + config.setSuperUserRoles(superUserRoles); + config.setProxyRoles(proxyRoles); + AuthorizationService authorizationService = new AuthorizationService( + PulsarConfigurationLoader.convertFrom(config), mock(PulsarResources.class)); + PulsarWorkerService pulsarWorkerService = mock(PulsarWorkerService.class); + when(pulsarWorkerService.getWorkerConfig()).thenReturn(config); + when(pulsarWorkerService.getAuthorizationService()).thenReturn(authorizationService); + when(pulsarWorkerService.isInitialized()).thenReturn(true); + when(pulsarWorkerService.getMetricsGenerator()).thenReturn(mock(MetricsGenerator.class)); + LeaderService leaderService = mock(LeaderService.class); + when(leaderService.isLeader()).thenReturn(true); + when(pulsarWorkerService.getLeaderService()).thenReturn(leaderService); + worker = new WorkerImpl(() -> pulsarWorkerService); + } + + @Test(dataProvider = "authParamsForNonSuperusers") + public void ensureNonSuperuserCombinationsFailAuthorization(AuthenticationParameters authParams) throws Throwable { + assertThrowsRestException401(() -> worker.getCluster(authParams)); + assertThrowsRestException401(() -> worker.getClusterLeader(authParams)); + assertThrowsRestException401(() -> worker.getAssignments(authParams)); + assertThrowsRestException401(() -> worker.getWorkerMetrics(authParams)); + assertThrowsRestException401(() -> worker.getFunctionsMetrics(authParams)); + assertThrowsRestException401(() -> worker.getListOfConnectors(authParams)); + assertThrowsRestException401(() -> worker.rebalance(null, authParams)); + assertThrowsRestException401(() -> worker.drain(null, "test", authParams, false)); + assertThrowsRestException401(() -> worker.getDrainStatus(null, "test", authParams, false)); + // This endpoint is not protected + assertTrue(worker.isLeaderReady(authParams)); + } + + private void assertThrowsRestException401(Assert.ThrowingRunnable runnable) throws Throwable { + try { + runnable.run(); + fail("Should have thrown RestException"); + } catch (RestException e) { + assertEquals(e.getResponse().getStatus(), 401); + } + } +} diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index 80b626165a5d8..32a104c576993 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -53,6 +53,7 @@ import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.Functions; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -542,7 +543,7 @@ private void testRegisterFunctionMissingArguments( details, functionPkgUrl, JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - null); + AuthenticationParameters.builder().build()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } @@ -560,7 +561,7 @@ private void registerDefaultFunction() { mockedFormData, null, JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - null); + AuthenticationParameters.builder().build()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } @@ -943,7 +944,7 @@ private void testUpdateFunctionMissingArguments( details, null, JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - null); + AuthenticationParameters.builder().build()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } @@ -971,7 +972,7 @@ private void updateDefaultFunction() { mockedFormData, null, JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - null); + AuthenticationParameters.builder().build()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } @@ -1048,7 +1049,7 @@ public void testUpdateFunctionWithUrl() throws Exception { null, filePackageUrl, JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - null); + AuthenticationParameters.builder().build()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } @@ -1144,7 +1145,7 @@ private void testDeregisterFunctionMissingArguments( tenant, namespace, function, - null); + AuthenticationParameters.builder().build()); } private void deregisterDefaultFunction() { @@ -1152,7 +1153,7 @@ private void deregisterDefaultFunction() { tenant, namespace, function, - null); + AuthenticationParameters.builder().build()); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") @@ -1257,7 +1258,8 @@ private void testGetFunctionMissingArguments( resource.getFunctionInfo( tenant, namespace, - function, null + function, + AuthenticationParameters.builder().build() ); } @@ -1266,7 +1268,8 @@ private FunctionDetails getDefaultFunctionInfo() throws IOException { String json = (String) resource.getFunctionInfo( tenant, namespace, - function, null + function, + AuthenticationParameters.builder().build() ).getEntity(); FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); mergeJson(json, functionDetailsBuilder); @@ -1353,7 +1356,8 @@ private void testListFunctionsMissingArguments( ) { resource.listFunctions( tenant, - namespace, null + namespace, + AuthenticationParameters.builder().build() ); } @@ -1361,7 +1365,8 @@ private void testListFunctionsMissingArguments( private List listDefaultFunctions() { return new Gson().fromJson((String) resource.listFunctions( tenant, - namespace, null + namespace, + AuthenticationParameters.builder().build() ).getEntity(), List.class); } @@ -1390,7 +1395,8 @@ public void testDownloadFunctionHttpUrl() throws Exception { "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl, null).getEntity(); + StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl, + AuthenticationParameters.builder().build()).getEntity(); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1407,8 +1413,8 @@ public void testDownloadFunctionFile() throws Exception { String fileLocation = file.getAbsolutePath().replace('\\', '/'); String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = - (StreamingOutput) function.downloadFunction("file:///" + fileLocation, null).getEntity(); + StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation, + AuthenticationParameters.builder().build()).getEntity(); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1440,7 +1446,8 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { functionConfig.setOutputSerdeClassName(outputSerdeClassName); try { resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), null); + JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), + AuthenticationParameters.builder().build()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } @@ -1474,7 +1481,8 @@ public void testRegisterFunctionWithConflictingFields() throws Exception { functionConfig.setOutputSerdeClassName(outputSerdeClassName); try { resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), null); + JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), + AuthenticationParameters.builder().build()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index c34b26180f876..337999bf2ad5e 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -49,6 +49,7 @@ import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.Functions; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.Packages; @@ -571,7 +572,7 @@ private void testRegisterFunctionMissingArguments( details, functionPkgUrl, functionConfig, - null, null); + null); } @@ -585,7 +586,7 @@ public void testMissingFunctionConfig() { mockedFormData, null, null, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided") @@ -600,7 +601,7 @@ public void testUpdateMissingFunctionConfig() { mockedFormData, null, null, - null, null, null); + null, null); } private void registerDefaultFunction() { @@ -617,7 +618,7 @@ private void registerDefaultFunctionWithPackageUrl(String packageUrl) { mockedFormData, packageUrl, functionConfig, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists") @@ -795,7 +796,7 @@ public void testRegisterFunctionSuccessK8sNoUpload() throws Exception { mockedFormData, null, functionConfig, - null, null); + null); } } @@ -852,7 +853,7 @@ public void testRegisterFunctionSuccessK8sWithUpload() throws Exception { mockedFormData, null, functionConfig, - null, null); + null); Assert.fail(); } catch (RuntimeException e) { Assert.assertEquals(e.getMessage(), injectedErrMsg); @@ -1115,7 +1116,7 @@ private void testUpdateFunctionMissingArguments( details, null, functionConfig, - null, null, null); + null, null); } @@ -1143,7 +1144,7 @@ private void updateDefaultFunctionWithPackageUrl(String packageUrl) { mockedFormData, packageUrl, functionConfig, - null, null, null); + null, null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") @@ -1217,7 +1218,7 @@ public void testUpdateFunctionWithUrl() { null, filePackageUrl, functionConfig, - null, null, null); + null, null); } @@ -1331,7 +1332,7 @@ private void testDeregisterFunctionMissingArguments( tenant, namespace, function, - null, null); + null); } private void deregisterDefaultFunction() { @@ -1339,7 +1340,7 @@ private void deregisterDefaultFunction() { tenant, namespace, function, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") @@ -1444,7 +1445,7 @@ private void testGetFunctionMissingArguments( resource.getFunctionInfo( tenant, namespace, - function,null,null + function,null ); } @@ -1454,7 +1455,6 @@ private FunctionConfig getDefaultFunctionInfo() { tenant, namespace, function, - null, null ); } @@ -1539,7 +1539,7 @@ private void testListFunctionsMissingArguments( ) { resource.listFunctions( tenant, - namespace,null,null + namespace,null ); } @@ -1547,7 +1547,7 @@ private void testListFunctionsMissingArguments( private List listDefaultFunctions() { return resource.listFunctions( tenant, - namespace,null,null + namespace,null ); } @@ -1604,7 +1604,7 @@ public void testDownloadFunctionHttpUrl() throws Exception { "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - StreamingOutput streamOutput = resource.downloadFunction(jarHttpUrl, null, null); + StreamingOutput streamOutput = resource.downloadFunction(jarHttpUrl, null); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1619,7 +1619,7 @@ public void testDownloadFunctionFile() throws Exception { String fileLocation = file.getAbsolutePath().replace('\\', '/'); String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - StreamingOutput streamOutput = resource.downloadFunction("file:///" + fileLocation, null, null); + StreamingOutput streamOutput = resource.downloadFunction("file:///" + fileLocation, null); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1643,7 +1643,7 @@ public void testDownloadFunctionBuiltinConnector() throws Exception { when(connectorsManager.getConnector("cassandra")).thenReturn(connector); when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); - StreamingOutput streamOutput = resource.downloadFunction("builtin://cassandra", null, null); + StreamingOutput streamOutput = resource.downloadFunction("builtin://cassandra", null); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); @@ -1672,7 +1672,7 @@ public void testDownloadFunctionBuiltinFunction() throws Exception { when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); - StreamingOutput streamOutput = resource.downloadFunction("builtin://exclamation", null, null); + StreamingOutput streamOutput = resource.downloadFunction("builtin://exclamation", null); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); @@ -1707,7 +1707,8 @@ public void testDownloadFunctionBuiltinConnectorByName() throws Exception { when(connectorsManager.getConnector("cassandra")).thenReturn(connector); when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, false); + StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + AuthenticationParameters.builder().build(), false); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1740,7 +1741,8 @@ public void testDownloadFunctionBuiltinFunctionByName() throws Exception { when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, false); + StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + AuthenticationParameters.builder().build(), false); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1774,7 +1776,8 @@ public void testDownloadTransformFunctionByName() throws Exception { when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, true); + StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + AuthenticationParameters.builder().build(), true); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1804,7 +1807,7 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); functionConfig.setOutput(outputTopic); functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null, null); + resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null); } @@ -1834,7 +1837,7 @@ public void testRegisterFunctionWithConflictingFields() throws Exception { functionConfig.setOutput(outputTopic); functionConfig.setOutputSerdeClassName(outputSerdeClassName); resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is either not set or cannot be determined") @@ -1856,7 +1859,7 @@ public void testCreateFunctionWithoutSettingRuntime() throws Exception { functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); functionConfig.setOutput(outputTopic); functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null, null); + resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 9f0de2a12e091..9f786c5b73aac 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -52,6 +52,7 @@ import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.Functions; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.Packages; @@ -534,7 +535,7 @@ private void testRegisterSinkMissingArguments( details, pkgUrl, sinkConfig, - null, null); + null); } @@ -548,7 +549,7 @@ public void testMissingSinkConfig() { mockedFormData, null, null, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink config is not provided") @@ -562,7 +563,7 @@ public void testUpdateMissingSinkConfig() { mockedFormData, null, null, - null, null, null); + null, null); } private void registerDefaultSink() throws IOException { @@ -580,7 +581,7 @@ private void registerDefaultSinkWithPackageUrl(String packageUrl) throws IOExcep mockedFormData, packageUrl, sinkConfig, - null, null); + null); } } @@ -652,7 +653,7 @@ public void testRegisterSinkConflictingFields() throws Exception { mockedFormData, null, sinkConfig, - null, null); + null); } } @@ -754,7 +755,7 @@ public void testRegisterSinkSuccessWithTransformFunction() throws Exception { mockedFormData, null, sinkConfig, - null, null); + null); } } @@ -803,7 +804,7 @@ public void testRegisterSinkFailureWithInvalidTransformFunction() throws Excepti mockedFormData, null, sinkConfig, - null, null); + null); } } catch (RestException e) { // expected exception @@ -1021,7 +1022,7 @@ private void testUpdateSinkMissingArguments( details, null, sinkConfig, - null, null, null); + null, null); } @@ -1064,7 +1065,7 @@ private void updateDefaultSinkWithPackageUrl(String packageUrl) throws Exception mockedFormData, packageUrl, sinkConfig, - null, null, null); + null, null); } } @@ -1149,7 +1150,7 @@ public void testUpdateSinkWithUrl() throws Exception { null, filePackageUrl, sinkConfig, - null, null, null); + null, null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "sink failed to register") @@ -1249,7 +1250,7 @@ public void testUpdateSinkDifferentTransformFunction() throws Exception { mockedFormData, null, sinkConfig, - null, null, null); + null, null); } } @@ -1308,7 +1309,7 @@ private void testDeregisterSinkMissingArguments( tenant, namespace, sink, - null, null); + null); } @@ -1317,7 +1318,7 @@ private void deregisterDefaultSink() { tenant, namespace, sink, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink test-sink doesn't exist") @@ -1532,7 +1533,8 @@ private void testGetSinkMissingArguments( resource.getFunctionInfo( tenant, namespace, - sink, null, null + sink, + AuthenticationParameters.builder().build() ); } @@ -1541,7 +1543,8 @@ private SinkConfig getDefaultSinkInfo() { return resource.getSinkInfo( tenant, namespace, - sink + sink, + AuthenticationParameters.builder().build() ); } @@ -1634,7 +1637,8 @@ private void testListSinksMissingArguments( ) { resource.listFunctions( tenant, - namespace, null, null + namespace, + AuthenticationParameters.builder().build() ); } @@ -1642,7 +1646,8 @@ private void testListSinksMissingArguments( private List listDefaultSinks() { return resource.listFunctions( tenant, - namespace, null, null + namespace, + AuthenticationParameters.builder().build() ); } @@ -1779,7 +1784,7 @@ public void testRegisterSinkSuccessK8sNoUpload() throws Exception { mockedFormData, null, sinkConfig, - null, null); + null); } } @@ -1836,7 +1841,7 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception { mockedFormData, null, sinkConfig, - null, null); + null); Assert.fail(); } catch (RuntimeException e) { Assert.assertEquals(e.getMessage(), injectedErrMsg); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java index 303fa559b1d8e..36be2de716661 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java @@ -51,6 +51,7 @@ import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.client.admin.Functions; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.Packages; @@ -488,7 +489,7 @@ private void testRegisterSourceMissingArguments( details, pkgUrl, sourceConfig, - null, null); + null); } @@ -502,7 +503,7 @@ public void testMissingSinkConfig() { mockedFormData, null, null, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source config is not provided") @@ -516,7 +517,7 @@ public void testUpdateMissingSinkConfig() { mockedFormData, null, null, - null, null, null); + null, null); } private void registerDefaultSource() throws IOException { @@ -533,7 +534,7 @@ private void registerDefaultSourceWithPackageUrl(String packageUrl) throws IOExc null, packageUrl, sourceConfig, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source test-source already " @@ -634,7 +635,7 @@ public void testRegisterSourceConflictingFields() throws Exception { mockedFormData, null, sourceConfig, - null, null); + null); } } @@ -937,7 +938,7 @@ private void testUpdateSourceMissingArguments( details, null, sourceConfig, - null, null, null); + null, null); } @@ -983,7 +984,7 @@ private void updateDefaultSourceWithPackageUrl(String packageUrl) throws Excepti mockedFormData, packageUrl, sourceConfig, - null, null, null); + null, null); } } @@ -1069,7 +1070,7 @@ public void testUpdateSourceWithUrl() throws Exception { null, filePackageUrl, sourceConfig, - null, null, null); + null, null); } @@ -1182,7 +1183,7 @@ private void testDeregisterSourceMissingArguments( tenant, namespace, function, - null, null); + null); } @@ -1191,7 +1192,7 @@ private void deregisterDefaultSource() { tenant, namespace, source, - null, null); + null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source test-source doesn't " + @@ -1385,7 +1386,8 @@ private void testGetSourceMissingArguments( resource.getFunctionInfo( tenant, namespace, - source, null, null + source, + AuthenticationParameters.builder().build() ); } @@ -1393,7 +1395,8 @@ private SourceConfig getDefaultSourceInfo() { return resource.getSourceInfo( tenant, namespace, - source + source, + AuthenticationParameters.builder().build() ); } @@ -1476,14 +1479,17 @@ private void testListSourcesMissingArguments( ) { resource.listFunctions( tenant, - namespace, null, null + namespace, + AuthenticationParameters.builder().build() ); } private List listDefaultSources() { return resource.listFunctions( tenant, - namespace, null, null); + namespace, + AuthenticationParameters.builder().build() + ); } @Test From 421d707419ee42378b9c5b37812bd5c617ef2c9a Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Sat, 8 Apr 2023 03:49:16 -0700 Subject: [PATCH 071/280] [improve][broker] PIP-192 made split handler idempotent (#19988) Master Issue: https://github.com/apache/pulsar/issues/16691 ### Motivation Raising a PR to implement: https://github.com/apache/pulsar/issues/16691 ### Modifications This PR - Makes split handler idempotent . - Makes Leader's orphan monitor keep trying to send split msg until finished. - Select bundle boundaries at the SplitScheduler to have the same split boundaries for each Split handler retry. - Adds a split condition to check if the parent's Splitting state has moved. - Made Admin Unload command forceful to unload any bundles in invalid states. --- conf/broker.conf | 2 +- .../pulsar/broker/ServiceConfiguration.java | 2 +- .../extensions/ExtensibleLoadManagerImpl.java | 2 +- .../channel/ServiceUnitStateChannelImpl.java | 263 +++++++----- .../channel/ServiceUnitStateData.java | 15 + .../loadbalance/extensions/models/Split.java | 6 +- .../loadbalance/extensions/models/Unload.java | 10 +- ...faultNamespaceBundleSplitStrategyImpl.java | 76 +++- .../broker/namespace/NamespaceService.java | 10 +- .../common/naming/NamespaceBundleFactory.java | 8 +- .../naming/NamespaceBundleSplitAlgorithm.java | 3 + ...pecifiedPositionsBundleSplitAlgorithm.java | 47 ++- .../ExtensibleLoadManagerImplTest.java | 1 + .../channel/ServiceUnitStateChannelTest.java | 399 +++++++++++++++--- .../extensions/channel/models/SplitTest.java | 8 +- .../scheduler/SplitSchedulerTest.java | 19 +- ...faultNamespaceBundleSplitStrategyTest.java | 81 +++- ...fiedPositionsBundleSplitAlgorithmTest.java | 15 + 18 files changed, 767 insertions(+), 200 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 1bbb12313f9a6..414756abcb7cb 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1455,7 +1455,7 @@ loadBalancerNamespaceBundleSplitConditionHitCountThreshold=3 # the service-unit state channel when there are a large number of bundles. # minimum value = 30 secs # (only used in load balancer extension logics) -loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds=604800 +loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds=3600 ### --- Replication --- ### diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 52cebe15f6a5c..c86358b9b5b9e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2609,7 +2609,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "minimum value = 30 secs" + "(only used in load balancer extension logics)" ) - private long loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds = 604800; + private long loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds = 3600; @FieldContext( category = CATEGORY_LOAD_BALANCER, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index c1234b6dab21e..e054a1bc662c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -449,7 +449,7 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, log.warn(msg); throw new IllegalArgumentException(msg); } - Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker); + Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker, true); UnloadDecision unloadDecision = new UnloadDecision(unload, UnloadDecision.Label.Success, UnloadDecision.Reason.Admin); return unloadAsync(unloadDecision, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index ec79698db1f05..bd62c53be6083 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -46,7 +46,6 @@ import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished; import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -61,6 +60,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -200,7 +200,7 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { CompletableFuture>newBuilder().build(); this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build(); this.stateChangeListeners = new StateChangeListeners(); - this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateCleanUpDelayTimeInSeconds() + this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateTombstoneDelayTimeInSeconds() * 1000; this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS; this.ownershipMonitorDelayTimeInSecs = OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS; @@ -563,10 +563,11 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) { ServiceUnitStateData next; if (isTransferCommand(unload)) { next = new ServiceUnitStateData( - Releasing, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit)); + Releasing, unload.destBroker().get(), unload.sourceBroker(), + unload.force(), getNextVersionId(serviceUnit)); } else { next = new ServiceUnitStateData( - Releasing, null, unload.sourceBroker(), getNextVersionId(serviceUnit)); + Releasing, null, unload.sourceBroker(), unload.force(), getNextVersionId(serviceUnit)); } return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { @@ -660,7 +661,7 @@ private void log(Throwable e, String serviceUnit, ServiceUnitStateData data, Ser long handlerTotalCount = getHandlerTotalCounter(data).get(); long handlerFailureCount = getHandlerFailureCounter(data).get(); log.info("{} handled {} event for serviceUnit:{}, cur:{}, next:{}, " - + "totalHandledRequests{}, totalFailedRequests:{}", + + "totalHandledRequests:{}, totalFailedRequests:{}", lookupServiceAddress, getLogEventTag(data), serviceUnit, data == null ? "" : data, next == null ? "" : next, @@ -671,7 +672,7 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit, long handlerTotalCount = getHandlerTotalCounter(data).get(); long handlerFailureCount = getHandlerFailureCounter(data).incrementAndGet(); log.error("{} failed to handle {} event for serviceUnit:{}, cur:{}, next:{}, " - + "totalHandledRequests{}, totalFailedRequests:{}", + + "totalHandledRequests:{}, totalFailedRequests:{}", lookupServiceAddress, getLogEventTag(data), serviceUnit, data == null ? "" : data, next == null ? "" : next, @@ -857,110 +858,158 @@ private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnit boundariesSet.add(subBundle.getKeyRange().upperEndpoint()); }); boundaries = new ArrayList<>(boundariesSet); - nsBundleSplitAlgorithm = NamespaceBundleSplitAlgorithm.SPECIFIED_POSITIONS_DIVIDE_ALGO; + nsBundleSplitAlgorithm = NamespaceBundleSplitAlgorithm.SPECIFIED_POSITIONS_DIVIDE_FORCE_ALGO; } final AtomicInteger counter = new AtomicInteger(0); + var childBundles = data.splitServiceUnitToDestBroker().keySet().stream() + .map(child -> bundleFactory.getBundle( + bundle.getNamespaceObject().toString(), child)) + .collect(Collectors.toList()); this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, nsBundleSplitAlgorithm, - bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture); + bundle, childBundles, boundaries, data, counter, startTime, completionFuture); return completionFuture; } + + @VisibleForTesting protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, NamespaceBundleFactory bundleFactory, NamespaceBundleSplitAlgorithm algorithm, - NamespaceBundle bundle, + NamespaceBundle parentBundle, + List childBundles, List boundaries, - String serviceUnit, - ServiceUnitStateData data, + ServiceUnitStateData parentData, AtomicInteger counter, long startTime, CompletableFuture completionFuture) { - CompletableFuture> updateFuture = new CompletableFuture<>(); - - namespaceService.getSplitBoundary(bundle, algorithm, boundaries) - .thenAccept(splitBundlesPair -> { - // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper. - if (splitBundlesPair == null) { - String msg = format("Bundle %s not found under namespace", serviceUnit); - updateFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); - return; - } - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.sourceBroker(), VERSION_ID_INIT); - NamespaceBundles targetNsBundle = splitBundlesPair.getLeft(); - List splitBundles = Collections.unmodifiableList(splitBundlesPair.getRight()); - List successPublishedBundles = - Collections.synchronizedList(new ArrayList<>(splitBundles.size())); - List> futures = new ArrayList<>(splitBundles.size()); - for (NamespaceBundle sBundle : splitBundles) { - futures.add(pubAsync(sBundle.toString(), next).thenAccept(__ -> successPublishedBundles.add(sBundle))); - } - NamespaceName nsname = bundle.getNamespaceObject(); - FutureUtil.waitForAll(futures) - .thenCompose(__ -> namespaceService.updateNamespaceBundles(nsname, targetNsBundle)) - .thenCompose(__ -> namespaceService.updateNamespaceBundlesForPolicies(nsname, targetNsBundle)) - .thenRun(() -> { - bundleFactory.invalidateBundleCache(bundle.getNamespaceObject()); - updateFuture.complete(splitBundles); - }).exceptionally(e -> { - // Clean the new bundle when has exception. - List> futureList = new ArrayList<>(); - for (NamespaceBundle sBundle : successPublishedBundles) { - futureList.add(tombstoneAsync(sBundle.toString()).thenAccept(__ -> {})); - } - FutureUtil.waitForAll(futureList) - .whenComplete((__, ex) -> { - if (ex != null) { - log.warn("Clean new bundles failed,", ex); - } - updateFuture.completeExceptionally(e); - }); - return null; - }); - }).exceptionally(e -> { - updateFuture.completeExceptionally(e); - return null; - }); + ownChildBundles(childBundles, parentData) + .thenCompose(__ -> getSplitNamespaceBundles( + namespaceService, bundleFactory, algorithm, parentBundle, childBundles, boundaries)) + .thenCompose(namespaceBundles -> updateSplitNamespaceBundlesAsync( + namespaceService, bundleFactory, parentBundle, namespaceBundles)) + .thenAccept(__ -> // Update bundled_topic cache for load-report-generation + pulsar.getBrokerService().refreshTopicToStatsMaps(parentBundle)) + .thenAccept(__ -> pubAsync(parentBundle.toString(), new ServiceUnitStateData( + Deleted, null, parentData.sourceBroker(), getNextVersionId(parentData)))) + .thenAccept(__ -> { + double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); + log.info("Successfully split {} parent namespace-bundle to {} in {} ms", + parentBundle, childBundles, splitBundleTime); + completionFuture.complete(null); + }) + .exceptionally(ex -> { + // Retry several times on BadVersion + Throwable throwable = FutureUtil.unwrapCompletionException(ex); + if ((throwable instanceof MetadataStoreException.BadVersionException) + && (counter.incrementAndGet() < NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT)) { + log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit", + counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex); + pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry( + namespaceService, bundleFactory, algorithm, parentBundle, childBundles, + boundaries, parentData, counter, startTime, completionFuture), + 100, MILLISECONDS); + } else { + // Retry enough, or meet other exception + String msg = format("Failed to split bundle %s, Retried %d th / %d limit, reason %s", + parentBundle.toString(), counter.get(), + NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, throwable.getMessage()); + log.warn(msg, throwable); + completionFuture.completeExceptionally( + new BrokerServiceException.ServiceUnitNotReadyException(msg)); + } + return null; + }); + } - updateFuture.thenAccept(r -> { - // Delete the old bundle - pubAsync(serviceUnit, new ServiceUnitStateData( - Deleted, null, data.sourceBroker(), getNextVersionId(data))) - .thenRun(() -> { - // Update bundled_topic cache for load-report-generation - pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); - // TODO: Update the load data immediately if needed. - completionFuture.complete(null); - double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); - log.info("Successfully split {} parent namespace-bundle to {} in {} ms", serviceUnit, r, - splitBundleTime); - }).exceptionally(e -> { - double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); - String msg = format("Failed to free bundle %s in %s ms, under namespace [%s] with error %s", - bundle.getNamespaceObject().toString(), splitBundleTime, bundle, e.getMessage()); - completionFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); - return null; - }); - }).exceptionally(ex -> { - // Retry several times on BadVersion - Throwable throwable = FutureUtil.unwrapCompletionException(ex); - if ((throwable instanceof MetadataStoreException.BadVersionException) - && (counter.incrementAndGet() < NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT)) { - log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit", - counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex); - pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, - algorithm, bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture), - 100, MILLISECONDS); - } else if (throwable instanceof IllegalArgumentException) { - completionFuture.completeExceptionally(throwable); + private CompletableFuture ownChildBundles(List childBundles, + ServiceUnitStateData parentData) { + List> futures = new ArrayList<>(childBundles.size()); + var debug = debug(); + for (var childBundle : childBundles) { + var childBundleStr = childBundle.toString(); + var childData = tableview.get(childBundleStr); + if (childData != null) { + if (debug) { + log.info("Already owned child bundle:{}", childBundleStr); + } } else { - // Retry enough, or meet other exception - String msg = format("Bundle: %s not success update nsBundles, counter %d, reason %s", - bundle.toString(), counter.get(), throwable.getMessage()); - completionFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); + childData = new ServiceUnitStateData(Owned, parentData.sourceBroker(), + VERSION_ID_INIT); + futures.add(pubAsync(childBundleStr, childData).thenApply(__ -> null)); } - return null; - }); + } + + if (!futures.isEmpty()) { + return FutureUtil.waitForAll(futures); + } else { + return CompletableFuture.completedFuture(null); + } + } + + private CompletableFuture getSplitNamespaceBundles(NamespaceService namespaceService, + NamespaceBundleFactory bundleFactory, + NamespaceBundleSplitAlgorithm algorithm, + NamespaceBundle parentBundle, + List childBundles, + List boundaries) { + CompletableFuture future = new CompletableFuture(); + final var debug = debug(); + var targetNsBundle = bundleFactory.getBundles(parentBundle.getNamespaceObject()); + boolean found = false; + try { + targetNsBundle.validateBundle(parentBundle); + } catch (IllegalArgumentException e) { + if (debug) { + log.info("Namespace bundles do not contain the parent bundle:{}", + parentBundle); + } + for (var childBundle : childBundles) { + try { + targetNsBundle.validateBundle(childBundle); + if (debug) { + log.info("Namespace bundles contain the child bundle:{}", + childBundle); + } + } catch (Exception ex) { + future.completeExceptionally( + new BrokerServiceException.ServiceUnitNotReadyException( + "Namespace bundles do not contain the child bundle:" + childBundle, e)); + return future; + } + } + found = true; + } catch (Exception e) { + future.completeExceptionally( + new BrokerServiceException.ServiceUnitNotReadyException( + "Failed to validate the parent bundle in the namespace bundles.", e)); + return future; + } + if (found) { + future.complete(targetNsBundle); + return future; + } else { + return namespaceService.getSplitBoundary(parentBundle, algorithm, boundaries) + .thenApply(splitBundlesPair -> splitBundlesPair.getLeft()); + } + } + + private CompletableFuture updateSplitNamespaceBundlesAsync( + NamespaceService namespaceService, + NamespaceBundleFactory bundleFactory, + NamespaceBundle parentBundle, + NamespaceBundles splitNamespaceBundles) { + var namespaceName = parentBundle.getNamespaceObject(); + return namespaceService.updateNamespaceBundles( + namespaceName, splitNamespaceBundles) + .thenCompose(__ -> namespaceService.updateNamespaceBundlesForPolicies( + namespaceName, splitNamespaceBundles)) + .thenAccept(__ -> { + bundleFactory.invalidateBundleCache(parentBundle.getNamespaceObject()); + if (debug()) { + log.info("Successfully updated split namespace bundles and namespace bundle cache."); + } + }); } public void handleMetadataSessionEvent(SessionEvent e) { @@ -1053,11 +1102,23 @@ private void scheduleCleanup(String broker, long delayInSecs) { broker, delayInSecs, cleanupJobs.size()); } + + private ServiceUnitStateData getOverrideInactiveBrokerStateData(ServiceUnitStateData orphanData, + String selectedBroker) { + if (orphanData.state() == Splitting) { + return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker, + Map.copyOf(orphanData.splitServiceUnitToDestBroker()), + true, getNextVersionId(orphanData)); + } else { + return new ServiceUnitStateData(Owned, selectedBroker, true, getNextVersionId(orphanData)); + } + } + private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData) { + Optional selectedBroker = selectBroker(serviceUnit); if (selectedBroker.isPresent()) { - var override = - new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData)); + var override = getOverrideInactiveBrokerStateData(orphanData, selectedBroker.get()); log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", serviceUnit, orphanData, override); publishOverrideEventAsync(serviceUnit, orphanData, override) @@ -1157,7 +1218,13 @@ private Optional getOverrideInFlightStateData( case Assigning: { return getRollForwardStateData(serviceUnit, nextVersionId); } - case Splitting, Releasing: { + case Splitting: { + return Optional.of(new ServiceUnitStateData(Splitting, + orphanData.dstBroker(), orphanData.sourceBroker(), + Map.copyOf(orphanData.splitServiceUnitToDestBroker()), + true, nextVersionId)); + } + case Releasing: { if (availableBrokers.contains(orphanData.sourceBroker())) { // rollback to the src return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); @@ -1219,7 +1286,6 @@ protected void monitorOwnerships(List brokers) { inactiveBrokers.add(dstBroker); } else if (isInFlightState(state) && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { - log.warn("Found orphan serviceUnit:{}, stateData:{}", serviceUnit, stateData); orphanServiceUnits.put(serviceUnit, stateData); } } else if (now - stateData.timestamp() > semiTerminalStateWaitingTimeInMillis) { @@ -1249,6 +1315,9 @@ protected void monitorOwnerships(List brokers) { var overrideData = getOverrideInFlightStateData( orphanServiceUnit, orphanData, activeBrokers); if (overrideData.isPresent()) { + log.info("Overriding in-flight state ownership serviceUnit:{} " + + "from orphanData:{} to overrideData:{}", + orphanServiceUnit, orphanData, overrideData); publishOverrideEventAsync(orphanServiceUnit, orphanData, overrideData.get()) .whenComplete((__, e) -> { if (e != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index 52c7e27d65033..307d3a4acb175 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -45,10 +45,25 @@ public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sou System.currentTimeMillis(), versionId); } + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, + Map> splitServiceUnitToDestBroker, boolean force, + long versionId) { + this(state, dstBroker, sourceBroker, splitServiceUnitToDestBroker, force, + System.currentTimeMillis(), versionId); + } + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, long versionId) { this(state, dstBroker, sourceBroker, null, false, System.currentTimeMillis(), versionId); } + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, boolean force, + long versionId) { + this(state, dstBroker, sourceBroker, null, force, + System.currentTimeMillis(), versionId); + } + + + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, long versionId) { this(state, dstBroker, null, null, false, System.currentTimeMillis(), versionId); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java index ac9a36e6dbfd3..690fac59bc99c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java @@ -30,12 +30,8 @@ public record Split( public Split { Objects.requireNonNull(serviceUnit); - if (splitServiceUnitToDestBroker != null && splitServiceUnitToDestBroker.size() != 2) { + if (splitServiceUnitToDestBroker == null || splitServiceUnitToDestBroker.size() != 2) { throw new IllegalArgumentException("Split service unit should be split into 2 service units."); } } - - public Split(String serviceUnit, String sourceBroker) { - this(serviceUnit, sourceBroker, null); - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java index d474011919d97..753f8a942ced0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java @@ -24,12 +24,18 @@ /** * Defines the information required to unload or transfer a service unit(e.g. bundle). */ -public record Unload(String sourceBroker, String serviceUnit, Optional destBroker) { +public record Unload(String sourceBroker, String serviceUnit, Optional destBroker, boolean force) { public Unload { Objects.requireNonNull(sourceBroker); Objects.requireNonNull(serviceUnit); } + + public Unload(String sourceBroker, String serviceUnit) { - this(sourceBroker, serviceUnit, Optional.empty()); + this(sourceBroker, serviceUnit, Optional.empty(), false); + } + + public Unload(String sourceBroker, String serviceUnit, Optional destBroker) { + this(sourceBroker, serviceUnit, destBroker, false); } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java index bd8b3aa66542a..7875c07b1224a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java @@ -26,18 +26,23 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; @@ -53,12 +58,14 @@ public class DefaultNamespaceBundleSplitStrategyImpl implements NamespaceBundleS private final Set decisionCache; private final Map namespaceBundleCount; private final Map splitConditionHitCounts; + private final Map splittingBundles; private final SplitCounter counter; public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) { decisionCache = new HashSet<>(); namespaceBundleCount = new HashMap<>(); splitConditionHitCounts = new HashMap<>(); + splittingBundles = new HashMap<>(); this.counter = counter; } @@ -67,6 +74,7 @@ public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) { public Set findBundlesToSplit(LoadManagerContext context, PulsarService pulsar) { decisionCache.clear(); namespaceBundleCount.clear(); + splittingBundles.clear(); final ServiceConfiguration conf = pulsar.getConfiguration(); int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles(); long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics(); @@ -78,6 +86,15 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS boolean debug = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled(); var channel = ServiceUnitStateChannelImpl.get(pulsar); + for (var etr : channel.getOwnershipEntrySet()) { + var eData = etr.getValue(); + if (eData.state() == ServiceUnitState.Splitting) { + String bundle = etr.getKey(); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + splittingBundles.put(bundle, bundleRange); + } + } + Map bundleStatsMap = pulsar.getBrokerService().getBundleStats(); NamespaceBundleFactory namespaceBundleFactory = pulsar.getNamespaceService().getNamespaceBundleFactory(); @@ -177,6 +194,27 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS continue; } + var ranges = bundleRange.split("_"); + var foundSplittingBundle = false; + for (var etr : splittingBundles.entrySet()) { + var splittingBundle = etr.getKey(); + if (splittingBundle.startsWith(namespace)) { + var splittingBundleRange = etr.getValue(); + if (splittingBundleRange.startsWith(ranges[0]) + || splittingBundleRange.endsWith(ranges[1])) { + if (debug) { + log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG + + " (parent) bundle:%s is in Splitting state.", bundle, splittingBundle)); + } + foundSplittingBundle = true; + break; + } + } + } + if (foundSplittingBundle) { + continue; + } + if (debug) { log.info(String.format( "Splitting bundle: %s. " @@ -193,7 +231,43 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS )); } var decision = new SplitDecision(); - decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId())); + var namespaceService = pulsar.getNamespaceService(); + var namespaceBundle = namespaceService.getNamespaceBundleFactory() + .getBundle(namespaceName, bundleRange); + NamespaceBundleSplitAlgorithm algorithm = + namespaceService.getNamespaceBundleSplitAlgorithmByName( + conf.getDefaultNamespaceBundleSplitAlgorithm()); + List splitBoundary = null; + try { + splitBoundary = namespaceService + .getSplitBoundary(namespaceBundle, null, algorithm) + .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + } catch (Throwable e) { + counter.update(Failure, Unknown); + log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG + " Failed to get split boundaries.", bundle, e)); + continue; + } + if (splitBoundary == null) { + counter.update(Failure, Unknown); + log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG + " The split boundaries is null.", bundle)); + continue; + } + if (splitBoundary.size() != 1) { + counter.update(Failure, Unknown); + log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG + " The size of split boundaries is not 1. " + + "splitBoundary:%s", bundle, splitBoundary)); + continue; + } + + var parentRange = namespaceBundle.getKeyRange(); + var leftChildBundle = namespaceBundleFactory.getBundle(namespaceBundle.getNamespaceObject(), + NamespaceBundleFactory.getRange(parentRange.lowerEndpoint(), splitBoundary.get(0))); + var rightChildBundle = namespaceBundleFactory.getBundle(namespaceBundle.getNamespaceObject(), + NamespaceBundleFactory.getRange(splitBoundary.get(0), parentRange.upperEndpoint())); + Map> splitServiceUnitToDestBroker = Map.of( + leftChildBundle.getBundleRange(), Optional.empty(), + rightChildBundle.getBundleRange(), Optional.empty()); + decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId(), splitServiceUnitToDestBroker)); decision.succeed(reason); decisionCache.add(decision); int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index d092ef04018c8..c9cd33d6d87c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -967,9 +967,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, */ public CompletableFuture>> getSplitBoundary( NamespaceBundle bundle, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm, List boundaries) { - BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config); - CompletableFuture> splitBoundary = - nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption); + CompletableFuture> splitBoundary = getSplitBoundary(bundle, boundaries, nsBundleSplitAlgorithm); return splitBoundary.thenCompose(splitBoundaries -> { if (splitBoundaries == null || splitBoundaries.size() == 0) { LOG.info("[{}] No valid boundary found in {} to split bundle {}", @@ -981,6 +979,12 @@ public CompletableFuture>> getSplit }); } + public CompletableFuture> getSplitBoundary( + NamespaceBundle bundle, List boundaries, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm) { + BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config); + return nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption); + } + private BundleSplitOption getBundleSplitOption(NamespaceBundle bundle, List boundaries, ServiceConfiguration config) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java index f0f7deb940da1..937d2763767b6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java @@ -272,8 +272,7 @@ public NamespaceBundle getBundle(String namespace, String bundleRange) { String[] boundaries = bundleRange.split("_"); Long lowerEndpoint = Long.decode(boundaries[0]); Long upperEndpoint = Long.decode(boundaries[1]); - Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint, - (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); + Range hashRange = getRange(lowerEndpoint, upperEndpoint); return getBundle(NamespaceName.get(namespace), hashRange); } @@ -414,4 +413,9 @@ public static String getNamespaceFromPoliciesPath(String path) { return Joiner.on("/").join(i); } + public static Range getRange(Long lowerEndpoint, Long upperEndpoint) { + return Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint, + (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); + } + } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java index 01a61d8166563..1fd6fbcd6ea12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java @@ -39,6 +39,9 @@ public interface NamespaceBundleSplitAlgorithm { NamespaceBundleSplitAlgorithm TOPIC_COUNT_EQUALLY_DIVIDE_ALGO = new TopicCountEquallyDivideBundleSplitAlgorithm(); NamespaceBundleSplitAlgorithm SPECIFIED_POSITIONS_DIVIDE_ALGO = new SpecifiedPositionsBundleSplitAlgorithm(); + + NamespaceBundleSplitAlgorithm SPECIFIED_POSITIONS_DIVIDE_FORCE_ALGO = + new SpecifiedPositionsBundleSplitAlgorithm(true); NamespaceBundleSplitAlgorithm FLOW_OR_QPS_EQUALLY_DIVIDE_ALGO = new FlowOrQpsEquallyDivideBundleSplitAlgorithm(); static NamespaceBundleSplitAlgorithm of(String algorithmName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java index 1fd9a85cfa782..4a5f8a831c269 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java @@ -29,6 +29,16 @@ * This algorithm divides the bundle into several parts by the specified positions. */ public class SpecifiedPositionsBundleSplitAlgorithm implements NamespaceBundleSplitAlgorithm{ + + private boolean force; + + public SpecifiedPositionsBundleSplitAlgorithm() { + force = false; + } + + public SpecifiedPositionsBundleSplitAlgorithm(boolean force) { + this.force = force; + } @Override public CompletableFuture> getSplitBoundary(BundleSplitOption bundleSplitOption) { NamespaceService service = bundleSplitOption.getService(); @@ -39,19 +49,28 @@ public CompletableFuture> getSplitBoundary(BundleSplitOption bundleSp } // sort all positions Collections.sort(positions); - return service.getOwnedTopicListForNamespaceBundle(bundle).thenCompose(topics -> { - if (topics == null || topics.size() <= 1) { - return CompletableFuture.completedFuture(null); - } - List splitBoundaries = positions - .stream() - .filter(position -> position > bundle.getLowerEndpoint() && position < bundle.getUpperEndpoint()) - .collect(Collectors.toList()); - - if (splitBoundaries.size() == 0) { - return CompletableFuture.completedFuture(null); - } - return CompletableFuture.completedFuture(splitBoundaries); - }); + if (force) { + return getBoundaries(bundle, positions); + } else { + return service.getOwnedTopicListForNamespaceBundle(bundle) + .thenCompose(topics -> { + if (topics == null || topics.size() <= 1) { + return CompletableFuture.completedFuture(null); + } + return getBoundaries(bundle, positions); + }); + } + } + + private CompletableFuture> getBoundaries(NamespaceBundle bundle, List positions) { + List splitBoundaries = positions + .stream() + .filter(position -> position > bundle.getLowerEndpoint() && position < bundle.getUpperEndpoint()) + .collect(Collectors.toList()); + + if (splitBoundaries.size() == 0) { + return CompletableFuture.completedFuture(null); + } + return CompletableFuture.completedFuture(splitBoundaries); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 57590d8d7da0e..ae7ceeed92838 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -133,6 +133,7 @@ public void setup() throws Exception { conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); conf.setLoadBalancerSheddingEnabled(false); + conf.setLoadBalancerDebugModeEnabled(true); super.internalSetup(conf); pulsar1 = pulsar; ServiceConfiguration defaultConf = getDefaultConf(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 2d98bc5cae6e5..77c80187a63e1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -107,9 +107,16 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { private String lookupServiceAddress1; private String lookupServiceAddress2; private String bundle; - private String bundle1; private String bundle2; + private String bundle3; + private String childBundle1Range; + private String childBundle2Range; + private String childBundle11; + private String childBundle12; + + private String childBundle31; + private String childBundle32; private PulsarTestContext additionalPulsarTestContext; private LoadManagerContext loadManagerContext; @@ -122,6 +129,7 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { @Override protected void setup() throws Exception { conf.setAllowAutoTopicCreation(true); + conf.setLoadBalancerDebugModeEnabled(true); conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10); super.internalSetup(conf); @@ -149,15 +157,23 @@ protected void setup() throws Exception { lookupServiceAddress2 = (String) FieldUtils.readDeclaredField(channel2, "lookupServiceAddress", true); - bundle = String.format("%s/%s", "public/default", "0x00000000_0xffffffff"); - bundle1 = String.format("%s/%s", "public/default", "0x00000000_0xfffffff0"); - bundle2 = String.format("%s/%s", "public/default", "0xfffffff0_0xffffffff"); + bundle = "public/default/0x00000000_0xffffffff"; + bundle1 = "public/default/0x00000000_0xfffffff0"; + bundle2 = "public/default/0xfffffff0_0xffffffff"; + bundle3 = "public/default3/0x00000000_0xffffffff"; + childBundle1Range = "0x7fffffff_0xffffffff"; + childBundle2Range = "0x00000000_0x7fffffff"; + + childBundle11 = "public/default/" + childBundle1Range; + childBundle12 = "public/default/" + childBundle2Range; + + childBundle31 = "public/default3/" + childBundle1Range; + childBundle32 = "public/default3/" + childBundle2Range; } @BeforeMethod protected void initChannels() throws Exception { - cleanTableView(channel1, bundle); - cleanTableView(channel2, bundle); + cleanTableViews(); cleanOwnershipMonitorCounters(channel1); cleanOwnershipMonitorCounters(channel2); cleanOpsCounters(channel1); @@ -299,7 +315,9 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } } try { - channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1)) + Split split = new Split(bundle, lookupServiceAddress1, Map.of( + childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); + channel.publishSplitEventAsync(split) .get(2, TimeUnit.SECONDS); } catch (ExecutionException e) { if (e.getCause() instanceof IllegalStateException) { @@ -496,7 +514,8 @@ public void transferTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests2.size()); // recovered, check the monitor update state : Assigned -> Owned - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))).when(loadManager).selectAsync(any()); + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))) + .when(loadManager).selectAsync(any()); FieldUtils.writeDeclaredField(channel2, "producer", producer, true); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 1 , true); @@ -564,32 +583,33 @@ public void splitAndRetryTest() throws Exception { doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); - Split split = new Split(bundle, ownerAddr1.get()); + // Assert child bundle ownerships in the channels. + + Split split = new Split(bundle, ownerAddr1.get(), Map.of( + childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); channel1.publishSplitEventAsync(split); waitUntilState(channel1, bundle, Deleted); waitUntilState(channel2, bundle, Deleted); - validateHandlerCounters(channel1, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0); - validateHandlerCounters(channel2, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0); + validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); + validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); // Verify the retry count verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount + 1)) .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), any(), any(), anyLong(), any()); - // Assert child bundle ownerships in the channels. - String childBundle1 = "public/default/0x7fffffff_0xffffffff"; - String childBundle2 = "public/default/0x00000000_0x7fffffff"; - waitUntilNewOwner(channel1, childBundle1, lookupServiceAddress1); - waitUntilNewOwner(channel1, childBundle2, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle1, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle2, lookupServiceAddress1); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle1).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle2).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle1).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle2).get()); + + waitUntilNewOwner(channel1, childBundle11, lookupServiceAddress1); + waitUntilNewOwner(channel1, childBundle12, lookupServiceAddress1); + waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress1); + waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress1); + assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle11).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle12).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle11).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle12).get()); // try the monitor and check the monitor moves `Deleted` -> `Init` @@ -620,10 +640,10 @@ public void splitAndRetryTest() throws Exception { 0, 0); - cleanTableView(channel1, childBundle1); - cleanTableView(channel2, childBundle1); - cleanTableView(channel1, childBundle2); - cleanTableView(channel2, childBundle2); + cleanTableView(channel1, childBundle11); + cleanTableView(channel2, childBundle11); + cleanTableView(channel1, childBundle12); + cleanTableView(channel2, childBundle12); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); @@ -714,8 +734,8 @@ public void handleBrokerDeletionEventTest() var owner1 = channel1.getOwnerAsync(bundle1); var owner2 = channel2.getOwnerAsync(bundle2); - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any()); - + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + .when(loadManager).selectAsync(any()); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); @@ -916,8 +936,12 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(140, TimeUnit.SECONDS) - .untilAsserted(() -> verify(compactor, times(1)) - .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any())); + .untilAsserted(() -> { + channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + verify(compactor, times(1)) + .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any()); + }); + var channel3 = createChannel(pulsar); channel3.start(); @@ -1049,7 +1073,7 @@ public void unloadTest() } @Test(priority = 13) - public void assignTestWhenDestBrokerFails() + public void assignTestWhenDestBrokerProducerFails() throws ExecutionException, InterruptedException, IllegalAccessException { Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); @@ -1076,7 +1100,8 @@ public void assignTestWhenDestBrokerFails() "inFlightStateWaitingTimeInMillis", 3 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any()); + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + .when(loadManager).selectAsync(any()); channel1.publishAssignEventAsync(bundle, lookupServiceAddress2); // channel1 is broken. the assign won't be complete. waitUntilState(channel1, bundle); @@ -1132,7 +1157,7 @@ public void assignTestWhenDestBrokerFails() } @Test(priority = 14) - public void splitTestWhenDestBrokerFails() + public void splitTestWhenProducerFails() throws ExecutionException, InterruptedException, IllegalAccessException { @@ -1165,7 +1190,12 @@ public void splitTestWhenDestBrokerFails() "inFlightStateWaitingTimeInMillis", 3 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - channel2.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1, null)); + // Assert child bundle ownerships in the channels. + + + Split split = new Split(bundle, lookupServiceAddress1, Map.of( + childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); + channel2.publishSplitEventAsync(split); // channel1 is broken. the split won't be complete. waitUntilState(channel1, bundle); waitUntilState(channel2, bundle); @@ -1180,29 +1210,19 @@ public void splitTestWhenDestBrokerFails() FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 1 , true); - ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); - ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); - var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); - var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + waitUntilStateWithMonitor(leader, bundle, Deleted); + waitUntilStateWithMonitor(channel1, bundle, Deleted); + waitUntilStateWithMonitor(channel2, bundle, Deleted); - assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + var ownerAddr1 = channel1.getOwnerAsync(bundle); + var ownerAddr2 = channel2.getOwnerAsync(bundle); + + assertTrue(ownerAddr1.isCompletedExceptionally()); + assertTrue(ownerAddr2.isCompletedExceptionally()); - var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; - validateMonitorCounters(leader, - 0, - 0, - 1, - 0, - 0, - 0, - 0); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); @@ -1273,6 +1293,243 @@ public void testIsOwner() throws IllegalAccessException { assertFalse(channel1.isOwner(bundle)); } + @Test(priority = 16) + public void splitAndRetryFailureTest() throws Exception { + channel1.publishAssignEventAsync(bundle3, lookupServiceAddress1); + waitUntilNewOwner(channel1, bundle3, lookupServiceAddress1); + waitUntilNewOwner(channel2, bundle3, lookupServiceAddress1); + var ownerAddr1 = channel1.getOwnerAsync(bundle3).get(); + var ownerAddr2 = channel2.getOwnerAsync(bundle3).get(); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr2, Optional.of(lookupServiceAddress1)); + assertTrue(ownerAddr1.isPresent()); + + NamespaceService namespaceService = spy(pulsar1.getNamespaceService()); + CompletableFuture future = new CompletableFuture<>(); + int badVersionExceptionCount = 10; + AtomicInteger count = new AtomicInteger(badVersionExceptionCount); + future.completeExceptionally(new MetadataStoreException.BadVersionException("BadVersion")); + doAnswer(invocationOnMock -> { + if (count.decrementAndGet() > 0) { + return future; + } + // Call the real method + reset(namespaceService); + doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) + .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); + return future; + }).when(namespaceService).updateNamespaceBundlesForPolicies(any(), any()); + doReturn(namespaceService).when(pulsar1).getNamespaceService(); + doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) + .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); + + // Assert child bundle ownerships in the channels. + + Split split = new Split(bundle3, ownerAddr1.get(), Map.of( + childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); + channel1.publishSplitEventAsync(split); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertEquals(3, count.get()); + }); + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + ((ServiceUnitStateChannelImpl) leader) + .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + waitUntilState(leader, bundle3, Deleted); + waitUntilState(channel1, bundle3, Deleted); + waitUntilState(channel2, bundle3, Deleted); + + + validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1, 0); + validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0); + validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); + validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); + + waitUntilNewOwner(channel1, childBundle31, lookupServiceAddress1); + waitUntilNewOwner(channel1, childBundle32, lookupServiceAddress1); + waitUntilNewOwner(channel2, childBundle31, lookupServiceAddress1); + waitUntilNewOwner(channel2, childBundle32, lookupServiceAddress1); + assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle31).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle32).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle31).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle32).get()); + + + // try the monitor and check the monitor moves `Deleted` -> `Init` + + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 1, true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + waitUntilState(channel1, bundle3, Init); + waitUntilState(channel2, bundle3, Init); + + validateMonitorCounters(leader, + 0, + 1, + 1, + 0, + 0, + 0, + 0); + + + cleanTableViews(); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + } + + @Test(priority = 17) + public void testOverrideInactiveBrokerStateData() + throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException { + + var leaderChannel = channel1; + var followerChannel = channel2; + String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + assertEquals(leader, leader2); + if (leader.equals(lookupServiceAddress2)) { + leaderChannel = channel2; + followerChannel = channel1; + } + + String broker = lookupServiceAddress1; + + // test override states + String releasingBundle = "public/releasing/0xfffffff0_0xffffffff"; + String splittingBundle = bundle; + String assigningBundle = "public/assigning/0xfffffff0_0xffffffff"; + String freeBundle = "public/free/0xfffffff0_0xffffffff"; + String deletedBundle = "public/deleted/0xfffffff0_0xffffffff"; + String ownedBundle = "public/owned/0xfffffff0_0xffffffff"; + overrideTableViews(releasingBundle, + new ServiceUnitStateData(Releasing, null, broker, 1)); + overrideTableViews(splittingBundle, + new ServiceUnitStateData(Splitting, null, broker, + Map.of(childBundle1Range, Optional.empty(), + childBundle2Range, Optional.empty()), 1)); + overrideTableViews(assigningBundle, + new ServiceUnitStateData(Assigning, broker, null, 1)); + overrideTableViews(freeBundle, + new ServiceUnitStateData(Free, null, broker, 1)); + overrideTableViews(deletedBundle, + new ServiceUnitStateData(Deleted, null, broker, 1)); + overrideTableViews(ownedBundle, + new ServiceUnitStateData(Owned, broker, null, 1)); + + // test stable metadata state + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + .when(loadManager).selectAsync(any()); + leaderChannel.handleMetadataSessionEvent(SessionReestablished); + followerChannel.handleMetadataSessionEvent(SessionReestablished); + FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp", + System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + FieldUtils.writeDeclaredField(followerChannel, "lastMetadataSessionEventTimestamp", + System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); + followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); + + waitUntilNewOwner(channel2, releasingBundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress2); + waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress2); + waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, ownedBundle, lookupServiceAddress2); + assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get()); + assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally()); + assertTrue(channel2.getOwnerAsync(splittingBundle).isCompletedExceptionally()); + + // clean-up + FieldUtils.writeDeclaredField(leaderChannel, "maxCleanupDelayTimeInSecs", 3 * 60, true); + cleanTableViews(); + + } + + @Test(priority = 18) + public void testOverrideOrphanStateData() + throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException { + + var leaderChannel = channel1; + var followerChannel = channel2; + String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + assertEquals(leader, leader2); + if (leader.equals(lookupServiceAddress2)) { + leaderChannel = channel2; + followerChannel = channel1; + } + + String broker = lookupServiceAddress1; + + // test override states + String releasingBundle = "public/releasing/0xfffffff0_0xffffffff"; + String splittingBundle = bundle; + String assigningBundle = "public/assigning/0xfffffff0_0xffffffff"; + String freeBundle = "public/free/0xfffffff0_0xffffffff"; + String deletedBundle = "public/deleted/0xfffffff0_0xffffffff"; + String ownedBundle = "public/owned/0xfffffff0_0xffffffff"; + overrideTableViews(releasingBundle, + new ServiceUnitStateData(Releasing, null, broker, 1)); + overrideTableViews(splittingBundle, + new ServiceUnitStateData(Splitting, null, broker, + Map.of(childBundle1Range, Optional.empty(), + childBundle2Range, Optional.empty()), 1)); + overrideTableViews(assigningBundle, + new ServiceUnitStateData(Assigning, broker, null, 1)); + overrideTableViews(freeBundle, + new ServiceUnitStateData(Free, null, broker, 1)); + overrideTableViews(deletedBundle, + new ServiceUnitStateData(Deleted, null, broker, 1)); + overrideTableViews(ownedBundle, + new ServiceUnitStateData(Owned, broker, null, 1)); + + // test stable metadata state + doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + .when(loadManager).selectAsync(any()); + FieldUtils.writeDeclaredField(leaderChannel, "inFlightStateWaitingTimeInMillis", + -1, true); + FieldUtils.writeDeclaredField(followerChannel, "inFlightStateWaitingTimeInMillis", + -1, true); + ((ServiceUnitStateChannelImpl) leaderChannel) + .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + + waitUntilNewOwner(channel2, releasingBundle, broker); + waitUntilNewOwner(channel2, childBundle11, broker); + waitUntilNewOwner(channel2, childBundle12, broker); + waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, ownedBundle, broker); + assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get()); + assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally()); + assertTrue(channel2.getOwnerAsync(splittingBundle).isCompletedExceptionally()); + + // clean-up + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + cleanTableViews(); + } + private static ConcurrentOpenHashMap>> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { @@ -1375,6 +1632,22 @@ private static void waitUntilState(ServiceUnitStateChannel channel, String key, }); } + private void waitUntilStateWithMonitor(ServiceUnitStateChannel channel, String key, ServiceUnitState expected) + throws IllegalAccessException { + TableViewImpl tv = (TableViewImpl) + FieldUtils.readField(channel, "tableview", true); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .until(() -> { // wait until true + ((ServiceUnitStateChannelImpl) channel) + .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + ServiceUnitStateData data = tv.get(key); + ServiceUnitState actual = state(data); + return actual == expected; + }); + } + private static void cleanTableView(ServiceUnitStateChannel channel, String serviceUnit) throws IllegalAccessException { var tv = (TableViewImpl) @@ -1384,6 +1657,26 @@ private static void cleanTableView(ServiceUnitStateChannel channel, String servi cache.remove(serviceUnit); } + private void cleanTableViews() + throws IllegalAccessException { + var tv1 = (TableViewImpl) + FieldUtils.readField(channel1, "tableview", true); + var cache1 = (ConcurrentMap) + FieldUtils.readField(tv1, "data", true); + cache1.clear(); + + var tv2 = (TableViewImpl) + FieldUtils.readField(channel2, "tableview", true); + var cache2 = (ConcurrentMap) + FieldUtils.readField(tv2, "data", true); + cache2.clear(); + } + + private void overrideTableViews(String serviceUnit, ServiceUnitStateData val) throws IllegalAccessException { + overrideTableView(channel1, serviceUnit, val); + overrideTableView(channel2, serviceUnit, val); + } + private static void overrideTableView(ServiceUnitStateChannel channel, String serviceUnit, ServiceUnitStateData val) throws IllegalAccessException { var tv = (TableViewImpl) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java index 1e23ea74dca5e..fd50f0d8aedc2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.extensions.channel.models; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -56,12 +55,9 @@ public void testInvalidSplitServiceUnitToDestBroker() { new Split("A", "B", map); } - @Test + @Test(expectedExceptions = IllegalArgumentException.class) public void testNullSplitServiceUnitToDestBroker() { - var split = new Split("A", "B"); - assertEquals(split.serviceUnit(), "A"); - assertEquals(split.sourceBroker(), "B"); - assertNull(split.splitServiceUnitToDestBroker()); + var split = new Split("A", "B", null); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java index f7726f8e7a1aa..f56ad90833bdb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java @@ -28,6 +28,8 @@ import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; @@ -56,6 +58,15 @@ public class SplitSchedulerTest { NamespaceBundleSplitStrategy strategy; String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF"; String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF"; + + String childBundle12 = "tenant/namespace/0x7fffffff_0xffffffff"; + + String childBundle11 = "tenant/namespace/0x00000000_0x7fffffff"; + + String childBundle22 = "tenant/namespace/0x7fffffff_0x0fffffff"; + + String childBundle21 = "tenant/namespace/0x00000000_0x7fffffff"; + String broker = "broker-1"; SplitDecision decision1; SplitDecision decision2; @@ -77,11 +88,15 @@ public void setUp() { doReturn(CompletableFuture.completedFuture(null)).when(channel).publishSplitEventAsync(any()); decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker)); + Split split = new Split(bundle1, broker, Map.of( + childBundle11, Optional.empty(), childBundle12, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.MsgRate); decision2 = new SplitDecision(); - decision2.setSplit(new Split(bundle2, broker)); + Split split2 = new Split(bundle2, broker, Map.of( + childBundle21, Optional.empty(), childBundle22, Optional.empty())); + decision2.setSplit(split2); decision2.succeed(SplitDecision.Reason.Sessions); Set decisions = Set.of(decision1, decision2); doReturn(decisions).when(strategy).findBundlesToSplit(any(), any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java index 8c765e7a3df64..8b489c92af0af 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java @@ -28,9 +28,13 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; +import com.google.common.hash.Hashing; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -38,15 +42,19 @@ import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; -import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.PulsarStats; import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -71,7 +79,12 @@ public class DefaultNamespaceBundleSplitStrategyTest { String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF"; String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF"; - + Long splitBoundary1 = 0x7fffffffL; + Long splitBoundary2 = 0x07ffffffL; + String childBundle12 = "0x7fffffff_0xffffffff"; + String childBundle11 = "0x00000000_0x7fffffff"; + String childBundle22 = "0x07ffffff_0x0fffffff"; + String childBundle21 = "0x00000000_0x07ffffff"; String broker = "broker-1"; @BeforeMethod @@ -90,19 +103,21 @@ void setup() { brokerService = mock(BrokerService.class); pulsarStats = mock(PulsarStats.class); namespaceService = mock(NamespaceService.class); - namespaceBundleFactory = mock(NamespaceBundleFactory.class); + loadManagerContext = mock(LoadManagerContext.class); brokerRegistry = mock(BrokerRegistry.class); loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class); loadManager = mock(ExtensibleLoadManagerImpl.class); channel = mock(ServiceUnitStateChannelImpl.class); + + doReturn(mock(MetadataStoreExtended.class)).when(pulsar).getLocalMetadataStore(); + namespaceBundleFactory = spy(new NamespaceBundleFactory(pulsar, Hashing.crc32())); doReturn(brokerService).when(pulsar).getBrokerService(); doReturn(config).when(pulsar).getConfiguration(); doReturn(pulsarStats).when(brokerService).getPulsarStats(); doReturn(namespaceService).when(pulsar).getNamespaceService(); doReturn(namespaceBundleFactory).when(namespaceService).getNamespaceBundleFactory(); - doReturn(true).when(namespaceBundleFactory).canSplitBundle(any()); doReturn(brokerRegistry).when(loadManagerContext).brokerRegistry(); doReturn(broker).when(brokerRegistry).getBrokerId(); doReturn(new AtomicReference(loadManagerWrapper)).when(pulsar).getLoadManager(); @@ -110,6 +125,18 @@ void setup() { doReturn(channel).when(loadManager).getServiceUnitStateChannel(); doReturn(true).when(channel).isOwner(any()); + var namespaceBundle1 = namespaceBundleFactory.getBundle( + LoadManagerShared.getNamespaceNameFromBundleName(bundle1), + LoadManagerShared.getBundleRangeFromBundleName(bundle1)); + var namespaceBundle2 = namespaceBundleFactory.getBundle( + LoadManagerShared.getNamespaceNameFromBundleName(bundle2), + LoadManagerShared.getBundleRangeFromBundleName(bundle2)); + doReturn(CompletableFuture.completedFuture( + List.of(splitBoundary1))).when(namespaceService).getSplitBoundary( + eq(namespaceBundle1), eq((List)null), any()); + doReturn(CompletableFuture.completedFuture( + List.of(splitBoundary2))).when(namespaceService).getSplitBoundary( + eq(namespaceBundle2), eq((List)null), any()); bundleStats = new LinkedHashMap<>(); NamespaceBundleStats stats1 = new NamespaceBundleStats(); @@ -184,6 +211,20 @@ public void testError() throws Exception { verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } + public void testSplittingBundle() { + var counter = spy(new SplitCounter()); + config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0); + bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1); + doReturn(Map.of("tenant/namespace/0x00000000_0xFFFFFFFF", + new ServiceUnitStateData(ServiceUnitState.Splitting, broker, 1)).entrySet()) + .when(channel).getOwnershipEntrySet(); + var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); + var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); + var expected = Set.of(); + assertEquals(actual, expected); + verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); + } + public void testMaxMsgRate() { var counter = spy(new SplitCounter()); var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter); @@ -196,14 +237,18 @@ public void testMaxMsgRate() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker)); + Split split = new Split(bundle1, broker, Map.of( + childBundle11, Optional.empty(), childBundle12, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.MsgRate); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker)); + Split split = new Split(bundle2, broker, Map.of( + childBundle21, Optional.empty(), childBundle22, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.MsgRate); assertEquals(actual, Set.of(decision1)); @@ -224,14 +269,18 @@ public void testMaxTopics() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker)); + Split split = new Split(bundle1, broker, Map.of( + childBundle11, Optional.empty(), childBundle12, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.Topics); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker)); + Split split = new Split(bundle2, broker, Map.of( + childBundle21, Optional.empty(), childBundle22, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.Topics); assertEquals(actual, Set.of(decision1)); @@ -255,14 +304,18 @@ public void testMaxSessions() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker)); + Split split = new Split(bundle1, broker, Map.of( + childBundle11, Optional.empty(), childBundle12, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.Sessions); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker)); + Split split = new Split(bundle2, broker, Map.of( + childBundle21, Optional.empty(), childBundle22, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.Sessions); assertEquals(actual, Set.of(decision1)); @@ -286,14 +339,18 @@ public void testMaxBandwidthMbytes() { var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar); if (i == threshold) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle1, broker)); + Split split = new Split(bundle1, broker, Map.of( + childBundle11, Optional.empty(), childBundle12, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.Bandwidth); assertEquals(actual, Set.of(decision1)); verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown)); } else if (i == threshold + 1) { SplitDecision decision1 = new SplitDecision(); - decision1.setSplit(new Split(bundle2, broker)); + Split split = new Split(bundle2, broker, Map.of( + childBundle21, Optional.empty(), childBundle22, Optional.empty())); + decision1.setSplit(split); decision1.succeed(SplitDecision.Reason.Bandwidth); assertEquals(actual, Set.of(decision1)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java index ec458def65bb7..80f9c279a55b7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java @@ -34,6 +34,21 @@ public class SpecifiedPositionsBundleSplitAlgorithmTest { + @Test + public void testEmptyTopicWithForce() { + SpecifiedPositionsBundleSplitAlgorithm algorithm = new SpecifiedPositionsBundleSplitAlgorithm(true); + NamespaceService mockNamespaceService = mock(NamespaceService.class); + NamespaceBundle mockNamespaceBundle = mock(NamespaceBundle.class); + doReturn(1L).when(mockNamespaceBundle).getLowerEndpoint(); + doReturn(1000L).when(mockNamespaceBundle).getUpperEndpoint(); + doReturn(CompletableFuture.completedFuture(List.of())) + .when(mockNamespaceService).getOwnedTopicListForNamespaceBundle(mockNamespaceBundle); + List splitPositions = + algorithm.getSplitBoundary(new BundleSplitOption(mockNamespaceService, mockNamespaceBundle, + Arrays.asList(1L, 2L))).join(); + assertEquals(splitPositions, Arrays.asList(2L)); + } + @Test public void testTotalTopicsSizeLessThan1() { SpecifiedPositionsBundleSplitAlgorithm algorithm = new SpecifiedPositionsBundleSplitAlgorithm(); From 90b9dd4c432ce14a28f76baf2e5a1bbd90b44b89 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Sat, 8 Apr 2023 15:41:43 -0500 Subject: [PATCH 072/280] [fix][fn] Make /version return correct version (#20047) ### Motivation The `/version` endpoint for functions currently returns `{"version":"version.number"}` instead of an actual version number. This PR fixes that. One observation is that the broker's `/version` endpoint returns a plain string: https://github.com/apache/pulsar/blob/82237d3684fe506bcb6426b3b23f413422e6e4fb/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java#L527-L535 ### Modifications * Return the actual version. * Use a static field to decrease the cost of this endpoint. ### Verifying this change This is a trivial change. ### Documentation - [x] `doc-not-needed` This fixes an endpoint, no docs are needed. ### Matching PR in forked repository PR in forked repository: Skipping PR since this is so trivial --- .../worker/rest/ConfigurationResource.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java index a801d8503d3ee..2218431770b66 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java @@ -19,30 +19,25 @@ package org.apache.pulsar.functions.worker.rest; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.PulsarVersion; @Path("/") public class ConfigurationResource { + + private static final String VERSION = "{\"version\":\"" + PulsarVersion.getVersion() + "\"}"; + @Path("version") @GET @Produces(MediaType.APPLICATION_JSON) public Response release() throws JsonProcessingException { - final ObjectMapper mapper = ObjectMapperFactory.getMapper().getObjectMapper(); - final ObjectNode node = mapper.createObjectNode(); - node.put("version", "version.number"); - return Response.ok() .type(MediaType.APPLICATION_JSON) - .entity(mapper - .writerWithDefaultPrettyPrinter() - .writeValueAsString(node)) + .entity(VERSION) .build(); } } From 470b674016c8718f2dfd0a0f93cf02d49af0fead Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 10 Apr 2023 15:06:33 +0800 Subject: [PATCH 073/280] [feat][sql] Bump Trino version to 368 and fix Decimal breaking change (#20016) Signed-off-by: tison --- pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 75 ++++++++++--------- pulsar-sql/presto-distribution/pom.xml | 4 + pulsar-sql/presto-pulsar/pom.xml | 7 +- .../decoder/avro/PulsarAvroColumnDecoder.java | 39 +++++++--- .../decoder/json/PulsarJsonFieldDecoder.java | 49 ++++++++---- .../json/PulsarJsonRowDecoderFactory.java | 14 ++-- .../sql/presto/decoder/DecoderTestUtil.java | 19 +++-- .../presto/decoder/avro/TestAvroDecoder.java | 36 +++++---- 10 files changed, 145 insertions(+), 102 deletions(-) diff --git a/pom.xml b/pom.xml index 4534036f3fd2c..7f20d478594ae 100644 --- a/pom.xml +++ b/pom.xml @@ -184,7 +184,7 @@ flexible messaging model and an intuitive client API. 2.4.10 1.2.4 8.5.2 - 363 + 368 1.9.7.Final 42.5.0 8.0.30 diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 4dbf7d5737335..1f8972bf22219 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -36,7 +36,7 @@ 3.14.9 1.17.2 - 208 + 213 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index c523fae7606e1..6d60765d459b1 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -303,28 +303,28 @@ The Apache Software License, Version 2.0 - bytecode-1.2.jar * Airlift - aircompressor-0.20.jar - - bootstrap-208.jar - - concurrent-208.jar - - configuration-208.jar - - discovery-208.jar + - bootstrap-213.jar + - concurrent-213.jar + - configuration-213.jar + - discovery-213.jar - discovery-server-1.30.jar - - event-208.jar - - event-http-208.jar - - http-client-208.jar - - http-server-208.jar - - jmx-208.jar - - jmx-http-208.jar - - jmx-http-rpc-208.jar + - event-213.jar + - event-http-213.jar + - http-client-213.jar + - http-server-213.jar + - jmx-213.jar + - jmx-http-213.jar + - jmx-http-rpc-213.jar - joni-2.1.5.3.jar - - json-208.jar - - log-208.jar - - log-manager-208.jar - - node-208.jar + - json-213.jar + - log-213.jar + - log-manager-213.jar + - node-213.jar - parameternames-1.4.jar - - security-208.jar - - slice-0.39.jar - - stats-208.jar - - trace-token-208.jar + - security-213.jar + - slice-0.41.jar + - stats-213.jar + - trace-token-213.jar - units-1.6.jar * Apache HTTP Client - httpclient-4.5.13.jar @@ -340,7 +340,9 @@ The Apache Software License, Version 2.0 * J2ObjC Annotations - j2objc-annotations-1.3.jar * JSON Web Token Support For The JVM - - jjwt-0.9.0.jar + - jjwt-api-0.11.1.jar + - jjwt-impl-0.11.1.jar + - jjwt-jackson-0.11.1.jar * Jmxutils - jmxutils-1.21.jar * LevelDB @@ -384,18 +386,18 @@ The Apache Software License, Version 2.0 * Okio - okio-1.17.2.jar * Trino - - trino-array-363.jar - - trino-cli-363.jar - - trino-client-363.jar - - trino-geospatial-toolkit-363.jar - - trino-main-363.jar - - trino-matching-363.jar - - trino-memory-context-363.jar - - trino-parser-363.jar - - trino-plugin-toolkit-363.jar - - trino-server-main-363.jar - - trino-spi-363.jar - - trino-record-decoder-363.jar + - trino-array-368.jar + - trino-cli-368.jar + - trino-client-368.jar + - trino-geospatial-toolkit-368.jar + - trino-main-368.jar + - trino-matching-368.jar + - trino-memory-context-368.jar + - trino-parser-368.jar + - trino-plugin-toolkit-368.jar + - trino-server-main-368.jar + - trino-spi-368.jar + - trino-record-decoder-368.jar * RocksDB JNI - rocksdbjni-6.29.4.1.jar * SnakeYAML @@ -456,6 +458,7 @@ The Apache Software License, Version 2.0 - javassist-3.25.0-GA.jar * Java Native Access - jna-5.12.1.jar + - jna-platform-5.10.0.jar * Java Object Layout: Core - jol-core-0.2.jar * Yahoo Datasketches @@ -493,7 +496,7 @@ BSD License * ANTLR 4 Runtime - antlr4-runtime-4.9.2.jar * ASM, a very small and fast Java bytecode manipulation framework - - asm-6.2.1.jar + - asm-9.1.jar - asm-analysis-6.2.1.jar - asm-tree-6.2.1.jar - asm-util-6.2.1.jar @@ -518,6 +521,8 @@ MIT License * ScribeJava - scribejava-apis-6.9.0.jar - scribejava-core-6.9.0.jar + * OSHI + - oshi-core-5.8.5.jar CDDL - 1.0 * OSGi Resource Locator @@ -533,8 +538,9 @@ CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt - hk2-utils-2.6.1.jar - aopalliance-repackaged-2.6.1.jar * Jersey - - jaxrs-208.jar + - jaxrs-213.jar - jersey-client-2.34.jar + - jersey-common-2.34.jar - jersey-container-servlet-2.34.jar - jersey-container-servlet-core-2.34.jar - jersey-entity-filtering-2.34.jar @@ -542,7 +548,6 @@ CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt - jersey-media-json-jackson-2.34.jar - jersey-media-multipart-2.34.jar - jersey-server-2.34.jar - - jersey-common-2.34.jar * JAXB - jaxb-api-2.3.1.jar - jaxb-runtime-2.3.4.jar diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 6e39bd50ad8be..48a1bf90662f6 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -97,6 +97,10 @@ com.google.inject.extensions guice-multibindings + + org.apache.logging.log4j + log4j-to-slf4j + diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 622c66e6d76ba..e61580f1fcf15 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -41,8 +41,13 @@ io.airlift bootstrap + + + org.apache.logging.log4j + log4j-to-slf4j + + - io.airlift json diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index 4616a60379971..73081f8948a51 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -41,8 +41,8 @@ import io.trino.spi.type.BooleanType; import io.trino.spi.type.DateType; import io.trino.spi.type.DecimalType; -import io.trino.spi.type.Decimals; import io.trino.spi.type.DoubleType; +import io.trino.spi.type.Int128; import io.trino.spi.type.IntegerType; import io.trino.spi.type.MapType; import io.trino.spi.type.RealType; @@ -66,11 +66,14 @@ import org.apache.avro.generic.GenericRecord; /** - * Copy from {@link io.trino.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) - * with A little bit pulsar's extensions. - * 1) support {@link io.trino.spi.type.TimestampType},{@link io.trino.spi.type.DateType}DATE, - * * {@link io.trino.spi.type.TimeType}. + * Copy from {@link io.trino.decoder.avro.AvroColumnDecoder} + * with A little pulsar's extensions. + * 1) support date and time types. + * {@link io.trino.spi.type.TimestampType} + * {@link io.trino.spi.type.DateType} + * {@link io.trino.spi.type.TimeType} * 2) support {@link io.trino.spi.type.RealType}. + * 3) support {@link io.trino.spi.type.DecimalType}. */ public class PulsarAvroColumnDecoder { private static final Set SUPPORTED_PRIMITIVE_TYPES = ImmutableSet.of( @@ -252,13 +255,6 @@ private static Slice getSlice(Object value, Type type, String columnName) { } } - // The returned Slice size must be equals to 18 Byte - if (type instanceof DecimalType) { - ByteBuffer buffer = (ByteBuffer) value; - BigInteger bigInteger = new BigInteger(buffer.array()); - return Decimals.encodeUnscaledValue(bigInteger); - } - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); @@ -274,6 +270,9 @@ private static Block serializeObject(BlockBuilder builder, Object value, Type ty if (type instanceof RowType) { return serializeRow(builder, value, type, columnName); } + if (type instanceof DecimalType && !((DecimalType) type).isShort()) { + return serializeLongDecimal(builder, value, type, columnName); + } serializePrimitive(builder, value, type, columnName); return null; } @@ -299,6 +298,22 @@ private static Block serializeList(BlockBuilder parentBlockBuilder, Object value return blockBuilder.build(); } + private static Block serializeLongDecimal( + BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + final BlockBuilder blockBuilder; + if (parentBlockBuilder != null) { + blockBuilder = parentBlockBuilder; + } else { + blockBuilder = type.createBlockBuilder(null, 1); + } + final ByteBuffer buffer = (ByteBuffer) value; + type.writeObject(blockBuilder, Int128.fromBigEndian(buffer.array())); + if (parentBlockBuilder == null) { + return blockBuilder.getSingleValueBlock(0); + } + return null; + } + private static void serializePrimitive(BlockBuilder blockBuilder, Object value, Type type, String columnName) { requireNonNull(blockBuilder, "parent blockBuilder is null"); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index c09dd43e24143..905e3bd6becb4 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -29,6 +29,7 @@ import static java.util.Objects.requireNonNull; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.DecimalNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; import io.airlift.log.Logger; @@ -45,8 +46,8 @@ import io.trino.spi.type.BooleanType; import io.trino.spi.type.DateType; import io.trino.spi.type.DecimalType; -import io.trino.spi.type.Decimals; import io.trino.spi.type.DoubleType; +import io.trino.spi.type.Int128; import io.trino.spi.type.IntegerType; import io.trino.spi.type.MapType; import io.trino.spi.type.RealType; @@ -59,21 +60,22 @@ import io.trino.spi.type.Type; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; -import java.math.BigInteger; import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.commons.lang3.tuple.Pair; /** - * Copy from {@link io.trino.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) - * with some pulsar's extensions. + * Copy from {@link io.trino.decoder.json.DefaultJsonFieldDecoder} with some pulsar's extensions. * 1) support {@link io.trino.spi.type.ArrayType}. * 2) support {@link io.trino.spi.type.MapType}. * 3) support {@link io.trino.spi.type.RowType}. - * 4) support {@link io.trino.spi.type.TimestampType},{@link io.trino.spi.type.DateType}, - * {@link io.trino.spi.type.TimeType}. + * 4) support date and time types. + * {@link io.trino.spi.type.TimestampType} + * {@link io.trino.spi.type.DateType} + * {@link io.trino.spi.type.TimeType} * 5) support {@link io.trino.spi.type.RealType}. + * 6) support {@link io.trino.spi.type.DecimalType}. */ public class PulsarJsonFieldDecoder implements JsonFieldDecoder { @@ -90,7 +92,6 @@ public PulsarJsonFieldDecoder(DecoderColumnHandle columnHandle) { Pair range = getNumRangeByType(columnHandle.getType()); minValue = range.getKey(); maxValue = range.getValue(); - } private static Pair getNumRangeByType(Type type) { @@ -221,7 +222,7 @@ public static long getLong(JsonNode value, Type type, String columnName, long mi } // If it is decimalType, need to eliminate the decimal point, - // and give it to presto to set the decimal point + // and give it to trino to set the decimal point if (type instanceof DecimalType) { String decimalLong = value.asText().replace(".", ""); return Long.parseLong(decimalLong); @@ -273,14 +274,6 @@ public static double getDouble(JsonNode value, Type type, String columnName) { private static Slice getSlice(JsonNode value, Type type, String columnName) { String textValue = value.isValueNode() ? value.asText() : value.toString(); - // If it is decimalType, need to eliminate the decimal point, - // and give it to presto to set the decimal point - if (type instanceof DecimalType) { - textValue = textValue.replace(".", ""); - BigInteger bigInteger = new BigInteger(textValue); - return Decimals.encodeUnscaledValue(bigInteger); - } - Slice slice = utf8Slice(textValue); if (type instanceof VarcharType) { slice = truncateToLength(slice, type); @@ -298,6 +291,9 @@ private Block serializeObject(BlockBuilder builder, Object value, Type type, Str if (type instanceof RowType) { return serializeRow(builder, value, type, columnName); } + if (type instanceof DecimalType && !((DecimalType) type).isShort()) { + return serializeLongDecimal(builder, value, type, columnName); + } serializePrimitive(builder, value, type, columnName); return null; } @@ -330,6 +326,27 @@ private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type return blockBuilder.build(); } + private static Block serializeLongDecimal( + BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + final BlockBuilder blockBuilder; + if (parentBlockBuilder != null) { + blockBuilder = parentBlockBuilder; + } else { + blockBuilder = type.createBlockBuilder(null, 1); + } + + assert value instanceof DecimalNode; + final DecimalNode node = (DecimalNode) value; + // For decimalType, need to eliminate the decimal point, + // and give it to trino to set the decimal point + type.writeObject(blockBuilder, Int128.valueOf(node.asText().replace(".", ""))); + + if (parentBlockBuilder == null) { + return blockBuilder.getSingleValueBlock(0); + } + return null; + } + private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type type, String columnName) { requireNonNull(blockBuilder, "parent blockBuilder is null"); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 5dd06fa939621..0d5cc2d262dfe 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -116,7 +116,7 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf } - private Type parseJsonPrestoType(String fieldname, Schema schema) { + private Type parseJsonPrestoType(String fieldName, Schema schema) { Schema.Type type = schema.getType(); LogicalType logicalType = schema.getLogicalType(); switch (type) { @@ -126,7 +126,7 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case NULL: throw new UnsupportedOperationException(format( "field '%s' NULL type code should not be reached , " - + "please check the schema or report the bug.", fieldname)); + + "please check the schema or report the bug.", fieldName)); case FIXED: case BYTES: // When the precision <= 0, throw Exception. @@ -157,10 +157,10 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case BOOLEAN: return BooleanType.BOOLEAN; case ARRAY: - return new ArrayType(parseJsonPrestoType(fieldname, schema.getElementType())); + return new ArrayType(parseJsonPrestoType(fieldName, schema.getElementType())); case MAP: //The key for an avro map must be string. - TypeSignature valueType = parseJsonPrestoType(fieldname, schema.getValueType()).getTypeSignature(); + TypeSignature valueType = parseJsonPrestoType(fieldName, schema.getValueType()).getTypeSignature(); return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter. typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); @@ -173,16 +173,16 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { } else { throw new UnsupportedOperationException(format( "field '%s' of record type has no fields, " - + "please check schema definition. ", fieldname)); + + "please check schema definition. ", fieldName)); } case UNION: for (Schema nestType : schema.getTypes()) { if (nestType.getType() != Schema.Type.NULL) { - return parseJsonPrestoType(fieldname, nestType); + return parseJsonPrestoType(fieldName, nestType); } } throw new UnsupportedOperationException(format( - "field '%s' of UNION type must contains not NULL type.", fieldname)); + "field '%s' of UNION type must contains not NULL type.", fieldName)); default: throw new UnsupportedOperationException(format( "Can't convert from schema type '%s' (%s) to presto type.", diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java index 84d93e780346a..60d6028f239d7 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java @@ -18,24 +18,23 @@ */ package org.apache.pulsar.sql.presto.decoder; +import static io.trino.testing.TestingConnectorSession.SESSION; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import io.airlift.slice.Slice; import io.trino.decoder.DecoderColumnHandle; import io.trino.decoder.FieldValueProvider; import io.trino.spi.block.Block; import io.trino.spi.type.ArrayType; import io.trino.spi.type.DecimalType; -import io.trino.spi.type.Decimals; +import io.trino.spi.type.Int128; import io.trino.spi.type.MapType; import io.trino.spi.type.RowType; import io.trino.spi.type.Type; import java.math.BigDecimal; -import java.math.BigInteger; import java.util.Map; -import static io.trino.spi.type.UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static org.testng.Assert.*; - /** * Abstract util superclass for XXDecoderTestUtil (e.g. AvroDecoderTestUtil 、JsonDecoderTestUtil) */ @@ -122,10 +121,10 @@ public void checkValue(Map decodedRow, FieldValueProvider provider = decodedRow.get(handle); DecimalType decimalType = (DecimalType) handle.getType(); BigDecimal actualDecimal; - if (decimalType.getFixedSize() == UNSCALED_DECIMAL_128_SLICE_LENGTH) { - Slice slice = provider.getSlice(); - BigInteger bigInteger = Decimals.decodeUnscaledValue(slice); - actualDecimal = new BigDecimal(bigInteger, decimalType.getScale()); + if (decimalType.getFixedSize() == Int128.SIZE) { + final Block block = provider.getBlock(); + final Int128 actualValue = (Int128) decimalType.getObject(block, 0); + actualDecimal = new BigDecimal(actualValue.toBigInteger(), decimalType.getScale()); } else { actualDecimal = BigDecimal.valueOf(provider.getLong(), decimalType.getScale()); } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java index 79ca0f5a65075..c4e7009b9465b 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java @@ -18,6 +18,19 @@ */ package org.apache.pulsar.sql.presto.decoder.avro; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.TimeType.TIME_MILLIS; +import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static java.lang.Float.floatToIntBits; +import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import com.google.common.collect.ImmutableList; import io.netty.buffer.ByteBuf; import io.trino.decoder.DecoderColumnHandle; @@ -33,6 +46,10 @@ import io.trino.spi.type.TypeSignatureParameter; import io.trino.spi.type.VarcharType; import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -47,25 +64,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.Float.floatToIntBits; -import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.expectThrows; - public class TestAvroDecoder extends AbstractDecoderTester { private AvroSchema schema; From 21c7c628d2a3fd7277f16fe7ba72154cfbf08128 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 10 Apr 2023 22:34:27 +0800 Subject: [PATCH 074/280] PIP-254: Support configuring client version (#20009) --- .../client/api/MutualAuthenticationTest.java | 48 ++++++++++++++++--- .../api/SimpleProducerConsumerTest.java | 34 ++++++++++++- .../pulsar/client/impl/ClientBuilderImpl.java | 23 +++++++++ .../apache/pulsar/client/impl/ClientCnx.java | 9 ++-- .../impl/conf/ClientConfigurationData.java | 6 +++ 5 files changed, 110 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java index 472af3e88cd36..2fc8aebf64a4a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java @@ -26,20 +26,26 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; +import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.common.api.AuthData; +import org.apache.pulsar.common.policies.data.PublisherStats; +import org.apache.pulsar.common.policies.data.TopicStats; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertEquals; /** * Test Mutual Authentication. @@ -182,7 +188,7 @@ public AuthenticationState newAuthState(AuthData authData, } } - @BeforeMethod(alwaysRun = true) + @BeforeClass(alwaysRun = true) @Override protected void setup() throws Exception { mutualAuth = new MutualAuthentication(); @@ -205,7 +211,7 @@ protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { clientBuilder.authentication(mutualAuth); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { internalCleanup(); @@ -214,12 +220,13 @@ protected void cleanup() throws Exception { @Test public void testAuthentication() throws Exception { log.info("-- Starting {} test --", methodName); + String topic = "persistent://my-property/my-ns/test-authentication"; - Consumer consumer = pulsarClient.newConsumer().topic("persistent://my-property/my-ns/my-topic1") + Consumer consumer = pulsarClient.newConsumer().topic(topic) .subscriptionName("my-subscriber-name") .subscribe(); Producer producer = pulsarClient.newProducer(Schema.BYTES) - .topic("persistent://my-property/my-ns/my-topic1") + .topic(topic) .create(); for (int i = 0; i < 10; i++) { @@ -239,4 +246,33 @@ public void testAuthentication() throws Exception { log.info("-- Exiting {} test --", methodName); } + + @Test + public void testClientVersion() throws Exception { + String defaultClientVersion = "Pulsar-Java-v" + PulsarVersion.getVersion(); + String topic = "persistent://my-property/my-ns/test-client-version"; + + Producer producer1 = pulsarClient.newProducer() + .topic(topic) + .create(); + TopicStats stats = admin.topics().getStats(topic); + assertEquals(stats.getPublishers().size(), 1); + assertEquals(stats.getPublishers().get(0).getClientVersion(), defaultClientVersion); + + PulsarClient client = ((ClientBuilderImpl) PulsarClient.builder()) + .description("my-java-client") + .serviceUrl(lookupUrl.toString()) + .authentication(mutualAuth) + .build(); + Producer producer2 = client.newProducer().topic(topic).create(); + stats = admin.topics().getStats(topic); + assertEquals(stats.getPublishers().size(), 2); + + assertEquals(stats.getPublishers().stream().map(PublisherStats::getClientVersion).collect(Collectors.toSet()), + Sets.newHashSet(defaultClientVersion, defaultClientVersion + "-my-java-client")); + + producer1.close(); + producer2.close(); + client.close(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 293e298fc6671..1bc437195d96d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -83,10 +83,12 @@ import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.ConsumerBase; import org.apache.pulsar.client.impl.ConsumerImpl; @@ -105,6 +107,8 @@ import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.PublisherStats; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; @@ -4581,4 +4585,32 @@ public void testSendMsgGreaterThanBatchingMaxBytes() throws Exception { // sendAsync should complete in time assertNotNull(producer.sendAsync(msg).get(timeoutSec, TimeUnit.SECONDS)); } -} \ No newline at end of file + + @Test + public void testClientVersion() throws Exception { + String defaultClientVersion = "Pulsar-Java-v" + PulsarVersion.getVersion(); + String topic = "persistent://my-property/my-ns/test-client-version"; + + Producer producer1 = pulsarClient.newProducer() + .topic(topic) + .create(); + TopicStats stats = admin.topics().getStats(topic); + assertEquals(stats.getPublishers().size(), 1); + assertEquals(stats.getPublishers().get(0).getClientVersion(), defaultClientVersion); + + PulsarClient client = ((ClientBuilderImpl) PulsarClient.builder()) + .description("my-java-client") + .serviceUrl(lookupUrl.toString()) + .build(); + Producer producer2 = client.newProducer().topic(topic).create(); + stats = admin.topics().getStats(topic); + assertEquals(stats.getPublishers().size(), 2); + + assertEquals(stats.getPublishers().stream().map(PublisherStats::getClientVersion).collect(Collectors.toSet()), + Sets.newHashSet(defaultClientVersion, defaultClientVersion + "-my-java-client")); + + producer1.close(); + producer2.close(); + client.close(); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java index 523acdace3950..7677045f0899b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java @@ -410,4 +410,27 @@ public ClientBuilder socks5ProxyPassword(String socks5ProxyPassword) { conf.setSocks5ProxyPassword(socks5ProxyPassword); return this; } + + /** + * Set the description. + * + *

By default, when the client connects to the broker, a version string like "Pulsar-Java-v" will be + * carried and saved by the broker. The client version string could be queried from the topic stats. + * + *

This method provides a way to add more description to a specific PulsarClient instance. If it's configured, + * the description will be appended to the original client version string, with '-' as the separator. + * + *

For example, if the client version is 3.0.0, and the description is "forked", the final client version string + * will be "Pulsar-Java-v3.0.0-forked". + * + * @param description the description of the current PulsarClient instance + * @throws IllegalArgumentException if the length of description exceeds 64 + */ + public ClientBuilder description(String description) { + if (description != null && description.length() > 64) { + throw new IllegalArgumentException("description should be at most 64 characters"); + } + conf.setDescription(description); + return this; + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 7780856c6948e..115c71307c4f2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -194,6 +194,8 @@ public class ClientCnx extends PulsarHandler { @Getter private long lastDisconnectedTimestamp; + private final String clientVersion; + protected enum State { None, SentConnectFrame, Ready, Failed, Connecting } @@ -252,6 +254,8 @@ public ClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, in this.state = State.None; this.protocolVersion = protocolVersion; this.idleState = new ClientCnxIdleState(this); + this.clientVersion = "Pulsar-Java-v" + PulsarVersion.getVersion() + + (conf.getDescription() == null ? "" : ("-" + conf.getDescription())); } @Override @@ -293,8 +297,7 @@ protected ByteBuf newConnectCommand() throws Exception { authenticationDataProvider = authentication.getAuthData(remoteHostName); AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); return Commands.newConnect(authentication.getAuthMethodName(), authData, this.protocolVersion, - String.format("Pulsar-Java-v%s", PulsarVersion.getVersion()), proxyToTargetBrokerAddress, null, null, - null); + clientVersion, proxyToTargetBrokerAddress, null, null, null); } @Override @@ -411,7 +414,7 @@ protected void handleAuthChallenge(CommandAuthChallenge authChallenge) { ByteBuf request = Commands.newAuthResponse(authentication.getAuthMethodName(), authData, this.protocolVersion, - String.format("Pulsar-Java-v%s", PulsarVersion.getVersion())); + clientVersion); if (log.isDebugEnabled()) { log.debug("{} Mutual auth {}", ctx.channel(), authentication.getAuthMethodName()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java index 1e7bc6f8221cf..7d94675ccba7d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java @@ -379,6 +379,12 @@ public class ClientConfigurationData implements Serializable, Cloneable { @Secret private String socks5ProxyPassword; + @ApiModelProperty( + name = "description", + value = "The extra description of the client version. The length cannot exceed 64." + ) + private String description; + /** * Gets the authentication settings for the client. * From 02c838c990543200c05f3d1479d74e685f895bba Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Mon, 10 Apr 2023 22:49:29 +0800 Subject: [PATCH 075/280] [fix][broker]Make LedgerOffloaderFactory can load the old nar. (#19913) After #13833, we change the signature of LedgerOffloaderFactory#create. But some users may use the old version LedgerOffloaderFactory in the offloader nar, when pulsar load the offloader nar, it will use the LedgerOffloaderFactory in the offloader nar, the LedgerOffloaderFactory#create has no param offloaderStats. Modifications: Make LedgerOffloaderFactory can load the old nar. --- .../mledger/LedgerOffloaderFactory.java | 39 ++++++++++++++++++- .../mledger/LedgerOffloaderStatsDisable.java | 4 +- .../FileSystemLedgerOffloaderFactory.java | 8 ++++ .../jcloud/JCloudLedgerOffloaderFactory.java | 7 ++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java index 42f92359f9a94..7ecb8f08d573d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java @@ -42,7 +42,7 @@ public interface LedgerOffloaderFactory { boolean isDriverSupported(String driverName); /** - * Create a ledger offloader with the provided configuration, user-metadata and scheduler. + * Create a ledger offloader with the provided configuration, user-metadata, scheduler and offloaderStats. * * @param offloadPolicies offload policies * @param userMetadata user metadata @@ -50,12 +50,29 @@ public interface LedgerOffloaderFactory { * @return the offloader instance * @throws IOException when fail to create an offloader */ + T create(OffloadPoliciesImpl offloadPolicies, + Map userMetadata, + OrderedScheduler scheduler) + throws IOException; + + + /** + * Create a ledger offloader with the provided configuration, user-metadata, scheduler and offloaderStats. + * + * @param offloadPolicies offload policies + * @param userMetadata user metadata + * @param scheduler scheduler + * @param offloaderStats offloaderStats + * @return the offloader instance + * @throws IOException when fail to create an offloader + */ T create(OffloadPoliciesImpl offloadPolicies, Map userMetadata, OrderedScheduler scheduler, LedgerOffloaderStats offloaderStats) throws IOException; + /** * Create a ledger offloader with the provided configuration, user-metadata, schema storage and scheduler. * @@ -66,6 +83,26 @@ T create(OffloadPoliciesImpl offloadPolicies, * @return the offloader instance * @throws IOException when fail to create an offloader */ + default T create(OffloadPoliciesImpl offloadPolicies, + Map userMetadata, + SchemaStorage schemaStorage, + OrderedScheduler scheduler) + throws IOException { + return create(offloadPolicies, userMetadata, scheduler); + } + + /** + * Create a ledger offloader with the provided configuration, user-metadata, schema storage, + * scheduler and offloaderStats. + * + * @param offloadPolicies offload policies + * @param userMetadata user metadata + * @param schemaStorage used for schema lookup in offloader + * @param scheduler scheduler + * @param offloaderStats offloaderStats + * @return the offloader instance + * @throws IOException when fail to create an offloader + */ default T create(OffloadPoliciesImpl offloadPolicies, Map userMetadata, SchemaStorage schemaStorage, diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java index 0fe0f453347bf..eeac9cfcfa994 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java @@ -20,9 +20,9 @@ import java.util.concurrent.TimeUnit; -class LedgerOffloaderStatsDisable implements LedgerOffloaderStats { +public class LedgerOffloaderStatsDisable implements LedgerOffloaderStats { - static final LedgerOffloaderStats INSTANCE = new LedgerOffloaderStatsDisable(); + public static final LedgerOffloaderStats INSTANCE = new LedgerOffloaderStatsDisable(); private LedgerOffloaderStatsDisable() { diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java index 8853ddf3da8ac..5649d264f579c 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java @@ -23,6 +23,7 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; +import org.apache.bookkeeper.mledger.LedgerOffloaderStatsDisable; import org.apache.bookkeeper.mledger.offload.filesystem.impl.FileSystemManagedLedgerOffloader; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; @@ -32,6 +33,13 @@ public boolean isDriverSupported(String driverName) { return FileSystemManagedLedgerOffloader.driverSupported(driverName); } + @Override + public FileSystemManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies, + Map userMetadata, OrderedScheduler scheduler) + throws IOException { + return create(offloadPolicies, userMetadata, scheduler, LedgerOffloaderStatsDisable.INSTANCE); + } + @Override public FileSystemManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies, Map userMetadata, diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java index e10575f5ddbb4..2c9165674444d 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java @@ -23,6 +23,7 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; +import org.apache.bookkeeper.mledger.LedgerOffloaderStatsDisable; import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider; import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration; @@ -44,6 +45,12 @@ public boolean isDriverSupported(String driverName) { return JCloudBlobStoreProvider.driverSupported(driverName); } + @Override + public BlobStoreManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies, Map userMetadata, + OrderedScheduler scheduler) throws IOException { + return create(offloadPolicies, userMetadata, scheduler, LedgerOffloaderStatsDisable.INSTANCE); + } + @Override public BlobStoreManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies, Map userMetadata, OrderedScheduler scheduler, From 7990948a73e2c4dfa0e9c99ff223f0ee90e82dc3 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 10 Apr 2023 12:13:13 -0500 Subject: [PATCH 076/280] [refactor][broker] Use AuthenticationParameters for rest producer (#20046) ### Motivation In #19975, we introduced a wrapper for all authentication parameters. This PR adds that wrapper to the Rest Producer. ### Modifications * Use `AuthenticationParameters` to simplify parameter management in Rest Producer. * Add method to the `AuthorizationService` that takes the `AuthenticationParameters`. * Update annotations on Rest Producer to indicate that a 401 is an expected response. ### Verifying this change This change is covered by the `TopicsAuthTest`. ### Documentation - [x] `doc-not-needed` This is an internal change that does not need to be documented. ### Matching PR in forked repository PR in forked repository: skipping PR since the relevant tests pass locally --- .../authorization/AuthorizationService.java | 7 ++++++ .../org/apache/pulsar/broker/rest/Topics.java | 4 +++ .../apache/pulsar/broker/rest/TopicsBase.java | 25 ++++++++++++++----- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 6c730f20092dd..706eadf0ec2dc 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -748,6 +748,13 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, } } + public CompletableFuture allowTopicOperationAsync(TopicName topicName, + TopicOperation operation, + AuthenticationParameters authParams) { + return allowTopicOperationAsync(topicName, operation, authParams.getOriginalPrincipal(), + authParams.getClientRole(), authParams.getClientAuthenticationDataSource()); + } + public CompletableFuture allowTopicOperationAsync(TopicName topicName, TopicOperation operation, String originalRole, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java index 5b17b05db209f..f1e7009df0257 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java @@ -49,6 +49,7 @@ public class Topics extends TopicsBase { @Path("/persistent/{tenant}/{namespace}/{topic}") @ApiOperation(value = "Produce message to a persistent topic.", response = String.class, responseContainer = "List") @ApiResponses(value = { + @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"), @ApiResponse(code = 412, message = "Namespace name is not valid"), @ApiResponse(code = 500, message = "Internal server error") }) @@ -76,6 +77,7 @@ public void produceOnPersistentTopic(@Suspended final AsyncResponse asyncRespons @ApiOperation(value = "Produce message to a partition of a persistent topic.", response = String.class, responseContainer = "List") @ApiResponses(value = { + @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"), @ApiResponse(code = 412, message = "Namespace name is not valid"), @ApiResponse(code = 500, message = "Internal server error") }) @@ -104,6 +106,7 @@ public void produceOnPersistentTopicPartition(@Suspended final AsyncResponse asy @Path("/non-persistent/{tenant}/{namespace}/{topic}") @ApiOperation(value = "Produce message to a persistent topic.", response = String.class, responseContainer = "List") @ApiResponses(value = { + @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"), @ApiResponse(code = 412, message = "Namespace name is not valid"), @ApiResponse(code = 500, message = "Internal server error") }) @@ -132,6 +135,7 @@ public void produceOnNonPersistentTopic(@Suspended final AsyncResponse asyncResp @ApiOperation(value = "Produce message to a partition of a persistent topic.", response = String.class, responseContainer = "List") @ApiResponses(value = { + @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"), @ApiResponse(code = 412, message = "Namespace name is not valid"), @ApiResponse(code = 500, message = "Internal server error") }) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 4614eb59a7016..7d3aa37fa4ec5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.rest; +import static java.util.concurrent.TimeUnit.SECONDS; import com.fasterxml.jackson.databind.ObjectReader; import io.netty.buffer.ByteBuf; import java.io.IOException; @@ -54,6 +55,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -81,6 +83,7 @@ import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; @@ -762,14 +765,24 @@ && pulsar().getBrokerService().isAuthorizationEnabled()) { if (!isClientAuthenticated(clientAppId())) { throw new RestException(Status.UNAUTHORIZED, "Need to authenticate to perform the request"); } + AuthenticationParameters authParams = authParams(); + boolean isAuthorized; + try { + isAuthorized = pulsar().getBrokerService().getAuthorizationService() + .allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, authParams) + .get(config().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } catch (InterruptedException e) { + log.warn("Time-out {} sec while checking authorization on {} ", + config().getMetadataStoreOperationTimeoutSeconds(), topicName); + throw new RestException(Status.INTERNAL_SERVER_ERROR, "Time-out while checking authorization"); + } catch (Exception e) { + log.warn("Producer-client with Role - {} {} failed to get permissions for topic - {}. {}", + authParams.getClientRole(), authParams.getOriginalPrincipal(), topicName, e.getMessage()); + throw new RestException(Status.INTERNAL_SERVER_ERROR, "Failed to get permissions"); + } - boolean isAuthorized = pulsar().getBrokerService().getAuthorizationService() - .canProduce(topicName, originalPrincipal() == null ? clientAppId() : originalPrincipal(), - clientAuthData()); if (!isAuthorized) { - throw new RestException(Status.UNAUTHORIZED, String.format("Unauthorized to produce to topic %s" - + " with clientAppId [%s] and authdata %s", topicName.toString(), - clientAppId(), clientAuthData())); + throw new RestException(Status.UNAUTHORIZED, "Unauthorized to produce to topic " + topicName); } } } From 25af808970fc02281a0b9345c5de205654d7b789 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Mon, 10 Apr 2023 10:27:58 -0700 Subject: [PATCH 077/280] [improve][client] Add retry when readTailMessages fails (#20044) --- .../pulsar/client/impl/TableViewTest.java | 41 +++++++++++++++++++ .../pulsar/client/impl/TableViewImpl.java | 29 ++++++------- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java index b6569d6a21dc1..6c6da5870aed9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.client.impl; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -31,6 +33,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -40,6 +43,7 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.TopicDomain; @@ -397,4 +401,41 @@ public void testTableViewWithEncryptedMessages() throws Exception { }); assertEquals(tv.keySet(), keys); } + + @Test(timeOut = 30 * 1000) + public void testTableViewTailMessageReadRetry() throws Exception { + String topic = "persistent://public/default/tableview-is-interrupted-test"; + admin.topics().createNonPartitionedTopic(topic); + @Cleanup + TableView tv = pulsarClient.newTableView(Schema.BYTES) + .topic(topic) + .autoUpdatePartitionsInterval(60, TimeUnit.SECONDS) + .create(); + + // inject failure on consumer.receiveAsync() + var reader = ((CompletableFuture>) + FieldUtils.readDeclaredField(tv, "reader", true)).join(); + var consumer = spy((ConsumerImpl) + FieldUtils.readDeclaredField(reader, "consumer", true)); + + var errorCnt = new AtomicInteger(3); + doAnswer(invocationOnMock -> { + if (errorCnt.decrementAndGet() > 0) { + return CompletableFuture.failedFuture(new RuntimeException()); + } + // Call the real method + reset(consumer); + return consumer.receiveAsync(); + }).when(consumer).receiveAsync(); + FieldUtils.writeDeclaredField(reader, "consumer", consumer, true); + + int msgCnt = 2; + this.publishMessages(topic, msgCnt, false, false); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertEquals(tv.size(), msgCnt); + }); + verify(consumer, times(msgCnt)).receiveAsync(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index 81771126f76ce..ff5b251ad55ff 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -240,9 +240,13 @@ private void readAllExistingMessages(Reader reader, CompletableFuture { - logException( - String.format("Reader %s was interrupted while reading existing messages", - reader.getTopic()), ex); + if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { + log.error("Reader {} was closed while reading existing messages.", + reader.getTopic(), ex); + } else { + log.warn("Reader {} was interrupted while reading existing messages. ", + reader.getTopic(), ex); + } future.completeExceptionally(ex); return null; }); @@ -266,18 +270,15 @@ private void readTailMessages(Reader reader) { handleMessage(msg); readTailMessages(reader); }).exceptionally(ex -> { - logException( - String.format("Reader %s was interrupted while reading tail messages.", - reader.getTopic()), ex); + if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { + log.error("Reader {} was closed while reading tail messages.", + reader.getTopic(), ex); + } else { + log.warn("Reader {} was interrupted while reading tail messages. " + + "Retrying..", reader.getTopic(), ex); + readTailMessages(reader); + } return null; }); } - - private void logException(String msg, Throwable ex) { - if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { - log.warn(msg, ex); - } else { - log.error(msg, ex); - } - } } From a7a605f13565edc380b8ae01808832c426f16d1f Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Tue, 11 Apr 2023 02:35:18 +0800 Subject: [PATCH 078/280] [improve] [broker] Upgrade the BookKeeper dependency to 4.16.0 (#20049) --- .../server/src/assemble/LICENSE.bin.txt | 54 ++++++++++--------- .../shell/src/assemble/LICENSE.bin.txt | 6 +-- pom.xml | 2 +- .../prometheus/metrics/LongAdderCounter.java | 9 +++- .../auth/SameThreadOrderedSafeExecutor.java | 5 +- .../broker/stats/PrometheusMetricsTest.java | 19 +------ pulsar-sql/presto-distribution/LICENSE | 29 +++++----- .../presto/PulsarConnectorMetricsTracker.java | 6 +-- .../bookkeeper/client/TestStatsProvider.java | 8 ++- 9 files changed, 70 insertions(+), 68 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 6b3455127b423..9205e072f7efe 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -343,35 +343,37 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar - org.apache.logging.log4j-log4j-web-2.18.0.jar * Java Native Access JNA - - net.java.dev.jna-jna-5.12.1.jar - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-proto-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-server-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.4.jar - - org.apache.bookkeeper-circe-checksum-4.15.4.jar - - org.apache.bookkeeper-cpu-affinity-4.15.4.jar - - org.apache.bookkeeper-statelib-4.15.4.jar - - org.apache.bookkeeper-stream-storage-api-4.15.4.jar - - org.apache.bookkeeper-stream-storage-common-4.15.4.jar - - org.apache.bookkeeper-stream-storage-java-client-4.15.4.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.15.4.jar - - org.apache.bookkeeper-stream-storage-proto-4.15.4.jar - - org.apache.bookkeeper-stream-storage-server-4.15.4.jar - - org.apache.bookkeeper-stream-storage-service-api-4.15.4.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.15.4.jar - - org.apache.bookkeeper.http-http-server-4.15.4.jar - - org.apache.bookkeeper.http-vertx-http-server-4.15.4.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.4.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.4.jar - - org.apache.distributedlog-distributedlog-common-4.15.4.jar - - org.apache.distributedlog-distributedlog-core-4.15.4-tests.jar - - org.apache.distributedlog-distributedlog-core-4.15.4.jar - - org.apache.distributedlog-distributedlog-protocol-4.15.4.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.4.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.0.jar + - org.apache.bookkeeper-circe-checksum-4.16.0.jar + - org.apache.bookkeeper-cpu-affinity-4.16.0.jar + - org.apache.bookkeeper-statelib-4.16.0.jar + - org.apache.bookkeeper-stream-storage-api-4.16.0.jar + - org.apache.bookkeeper-stream-storage-common-4.16.0.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.0.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.0.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.0.jar + - org.apache.bookkeeper-stream-storage-server-4.16.0.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.0.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.0.jar + - org.apache.bookkeeper.http-http-server-4.16.0.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.0.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.0.jar + - org.apache.distributedlog-distributedlog-common-4.16.0.jar + - org.apache.distributedlog-distributedlog-core-4.16.0-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.0.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.0.jar + - org.apache.bookkeeper-native-io-4.16.0.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 90896790b1fba..0ac08caa1c01d 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -390,9 +390,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.15.4.jar - - cpu-affinity-4.15.4.jar - - circe-checksum-4.15.4.jar + - bookkeeper-common-allocator-4.16.0.jar + - cpu-affinity-4.16.0.jar + - circe-checksum-4.16.0.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 7f20d478594ae..d6d7fdf44689d 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.15.4 + 4.16.0 3.8.1 1.5.0 1.10.0 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java index ff9d9302456d1..8ade2bc883f9a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.stats.Counter; @@ -50,10 +51,16 @@ public void dec() { } @Override - public void add(long delta) { + public void addCount(long delta) { counter.add(delta); } + @Override + public void addLatency(long eventLatency, TimeUnit unit) { + long valueMillis = unit.toMillis(eventLatency); + counter.add(valueMillis); + } + @Override public Long get() { return counter.sum(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java index 5f6a32a61bd25..258188a31f5d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java @@ -21,7 +21,6 @@ import io.netty.util.concurrent.DefaultThreadFactory; import org.apache.bookkeeper.common.util.OrderedExecutor; -import org.apache.bookkeeper.common.util.SafeRunnable; import org.apache.bookkeeper.stats.NullStatsLogger; public class SameThreadOrderedSafeExecutor extends OrderedExecutor { @@ -46,12 +45,12 @@ public void execute(Runnable r) { } @Override - public void executeOrdered(int orderingKey, SafeRunnable r) { + public void executeOrdered(int orderingKey, Runnable r) { r.run(); } @Override - public void executeOrdered(long orderingKey, SafeRunnable r) { + public void executeOrdered(long orderingKey, Runnable r) { r.run(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 04b0a5509b68c..f5ae8459f1803 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1322,24 +1322,7 @@ public void testManagedLedgerBookieClientStats() throws Exception { System.out.println(e.getKey() + ": " + e.getValue()) ); - List cm = (List) metrics.get(keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_completed_tasks")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( - keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_queue")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( - keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_total_tasks")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( + List cm = (List) metrics.get( keyNameBySubstrings(metrics, "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_threads")); assertEquals(cm.size(), 1); diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 6d60765d459b1..e101191bbf536 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -426,18 +426,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.15.4.jar - - bookkeeper-common-allocator-4.15.4.jar - - bookkeeper-proto-4.15.4.jar - - bookkeeper-server-4.15.4.jar - - bookkeeper-stats-api-4.15.4.jar - - bookkeeper-tools-framework-4.15.4.jar - - circe-checksum-4.15.4.jar - - codahale-metrics-provider-4.15.4.jar - - cpu-affinity-4.15.4.jar - - http-server-4.15.4.jar - - prometheus-metrics-provider-4.15.4.jar - - codahale-metrics-provider-4.15.4.jar + - bookkeeper-common-4.16.0.jar + - bookkeeper-common-allocator-4.16.0.jar + - bookkeeper-proto-4.16.0.jar + - bookkeeper-server-4.16.0.jar + - bookkeeper-stats-api-4.16.0.jar + - bookkeeper-tools-framework-4.16.0.jar + - circe-checksum-4.16.0.jar + - codahale-metrics-provider-4.16.0.jar + - cpu-affinity-4.16.0.jar + - http-server-4.16.0.jar + - prometheus-metrics-provider-4.16.0.jar + - codahale-metrics-provider-4.16.0.jar + - bookkeeper-slogger-api-4.16.0.jar + - bookkeeper-slogger-slf4j-4.16.0.jar + - native-io-4.16.0.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar @@ -479,6 +482,8 @@ The Apache Software License, Version 2.0 - amqp-client-5.5.3.jar * Stream Lib - stream-2.9.5.jar + * High Performance Primitive Collections for Java + - hppc-0.9.1.jar Protocol Buffers License diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java index b586063cc3c9a..12ee2da463c40 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java @@ -185,7 +185,7 @@ public void end_ENTRY_QUEUE_DEQUEUE_WAIT_TIME() { public void register_BYTES_READ(long bytes) { if (statsLogger != null) { bytesReadSum += bytes; - statsLoggerBytesRead.add(bytes); + statsLoggerBytesRead.addCount(bytes); } } @@ -220,7 +220,7 @@ public void end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME() { public void incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY() { if (statsLogger != null) { numMessagedDerserializedPerBatch++; - statsLoggerNumMessagesDeserialized.add(1); + statsLoggerNumMessagesDeserialized.addCount(1); } } @@ -295,7 +295,7 @@ public void end_RECORD_DESERIALIZE_TIME() { public void incr_NUM_RECORD_DESERIALIZED() { if (statsLogger != null) { - statsLoggerNumRecordDeserialized.add(1); + statsLoggerNumRecordDeserialized.addCount(1); } } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java index f015306cb048c..4d08a7f80df5b 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java @@ -59,10 +59,16 @@ public void dec() { } @Override - public void add(long delta) { + public void addCount(long delta) { updateMax(val.addAndGet(delta)); } + @Override + public void addLatency(long eventLatency, TimeUnit unit) { + long valueMillis = unit.toMillis(eventLatency); + updateMax(val.addAndGet(valueMillis)); + } + @Override public Long get() { return val.get(); From c3a92b213f04269678b68e5fe4b72a655955be1a Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 11 Apr 2023 02:44:03 +0800 Subject: [PATCH 079/280] [fix] [ml] Fix uncompleted future when remove cursor (#20050) --- .../mledger/impl/MetaStoreImpl.java | 20 ++++++++++--------- .../mledger/impl/ManagedCursorTest.java | 17 ++++++++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index 3bdaac1e8d80e..fb93e929f50c7 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -302,16 +302,18 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal } callback.operationComplete(null, null); }, executor.chooseThread(ledgerName)) - .exceptionallyAsync(ex -> { - Throwable actEx = FutureUtil.unwrapCompletionException(ex); - if (actEx instanceof MetadataStoreException.NotFoundException){ - log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName); - callback.operationComplete(null, null); - return null; - } - SafeRunnable.safeRun(() -> callback.operationFailed(getException(ex))); + .exceptionally(ex -> { + executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof MetadataStoreException.NotFoundException){ + log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName); + callback.operationComplete(null, null); + return; + } + callback.operationFailed(getException(ex)); + })); return null; - }, executor.chooseThread(ledgerName)); + }); } @Override diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 8dc726c249efc..1b1b5534256f9 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -48,6 +48,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -95,6 +96,7 @@ import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.IntRange; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -1060,6 +1062,21 @@ void removingCursor() throws Exception { Awaitility.await().until(() -> ledger.getNumberOfEntries() <= 2); } + @Test(timeOut = 10000) + void testRemoveCursorFail() throws Exception { + String mlName = UUID.randomUUID().toString().replaceAll("-", ""); + String cursorName = "c1"; + ManagedLedger ledger = factory.open(mlName); + ledger.openCursor(cursorName); + metadataStore.setAlwaysFail(new MetadataStoreException("123")); + try { + ledger.deleteCursor(cursorName); + fail("expected delete cursor failure."); + } catch (Exception ex) { + assertTrue(FutureUtil.unwrapCompletionException(ex).getMessage().contains("123")); + } + } + @Test(timeOut = 20000) void cursorPersistence() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); From 34b6e89269619afefa8120bdea3821d6b3811757 Mon Sep 17 00:00:00 2001 From: vineeth1995 Date: Mon, 10 Apr 2023 11:56:03 -0700 Subject: [PATCH 080/280] [feat] [broker] PIP-188 support blue-green cluster migration [part-2] (#19605) Co-authored-by: Vineeth --- .../pulsar/broker/service/Replicator.java | 2 + .../pulsar/broker/service/ServerCnx.java | 24 +- .../apache/pulsar/broker/service/Topic.java | 2 + .../NonPersistentReplicator.java | 2 +- .../nonpersistent/NonPersistentTopic.java | 5 + .../persistent/PersistentReplicator.java | 4 +- .../service/persistent/PersistentTopic.java | 21 +- .../auth/MockedPulsarServiceBaseTest.java | 8 +- .../broker/service/ClusterMigrationTest.java | 229 +++++++++++++++--- 9 files changed, 253 insertions(+), 44 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java index b8d13256d225a..482fa2cbd2300 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java @@ -49,4 +49,6 @@ default Optional getRateLimiter() { } boolean isConnected(); + + long getNumberOfEntriesInBacklog(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 973dec8980ea0..1b376492accc5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1579,13 +1579,23 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ if (ex.getCause() instanceof BrokerServiceException.TopicMigratedException) { Optional clusterURL = getMigratedClusterUrl(service.getPulsar()); if (clusterURL.isPresent()) { - log.info("[{}] redirect migrated producer to topic {}: producerId={}, {}", remoteAddress, topicName, - producerId, ex.getCause().getMessage()); - commandSender.sendTopicMigrated(ResourceType.Producer, producerId, - clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); - closeProducer(producer); - return null; - + if (topic.isReplicationBacklogExist()) { + log.info("Topic {} is migrated but replication backlog exist: " + + "producerId = {}, producerName = {}, {}", topicName, + producerId, producerName, ex.getCause().getMessage()); + } else { + log.info("[{}] redirect migrated producer to topic {}: " + + "producerId={}, producerName = {}, {}", remoteAddress, + topicName, producerId, producerName, ex.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, + clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("client doesn't support topic migration handling {}-{}-{}", topic, + remoteAddress, producerId); + } + closeProducer(producer); + return null; + } } else { log.warn("[{}] failed producer because migration url not configured topic {}: producerId={}, {}", remoteAddress, topicName, producerId, ex.getCause().getMessage()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index e6a29368dbb85..7657d77e1299f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -234,6 +234,8 @@ CompletableFuture createSubscription(String subscriptionName, Init boolean isBrokerPublishRateExceeded(); + boolean isReplicationBacklogExist(); + void disableCnxAutoRead(); void enableCnxAutoRead(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java index 7ad231926189b..514db4219db98 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java @@ -248,7 +248,7 @@ protected Position getReplicatorReadPosition() { } @Override - protected long getNumberOfEntriesInBacklog() { + public long getNumberOfEntriesInBacklog() { // No-op return 0; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index a0a8462a22753..317b8df6b9a82 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -236,6 +236,11 @@ public void checkMessageDeduplicationInfo() { // No-op } + @Override + public boolean isReplicationBacklogExist() { + return false; + } + @Override public void removeProducer(Producer producer) { checkArgument(producer.getTopic() == this); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index f38bcc71582a1..a556237f4342c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -165,8 +165,8 @@ protected Position getReplicatorReadPosition() { } @Override - protected long getNumberOfEntriesInBacklog() { - return cursor.getNumberOfEntriesInBacklog(false); + public long getNumberOfEntriesInBacklog() { + return cursor.getNumberOfEntriesInBacklog(true); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 18a662c4b7a38..0374fc98212f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -575,6 +575,7 @@ public void addComplete(Position pos, ByteBuf entryData, Object ctx) { @Override public synchronized void addFailed(ManagedLedgerException exception, Object ctx) { + PublishContext callback = (PublishContext) ctx; if (exception instanceof ManagedLedgerFencedException) { // If the managed ledger has been fenced, we cannot continue using it. We need to close and reopen close(); @@ -587,7 +588,11 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx) List> futures = new ArrayList<>(); // send migration url metadata to producers before disconnecting them if (isMigrated()) { - producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl())); + if (isReplicationBacklogExist()) { + log.info("Topic {} is migrated but replication backlog exists. Closing producers.", topic); + } else { + producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl())); + } } producers.forEach((__, producer) -> futures.add(producer.disconnect())); disconnectProducersFuture = FutureUtil.waitForAll(futures); @@ -599,8 +604,6 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx) return null; }); - PublishContext callback = (PublishContext) ctx; - if (exception instanceof ManagedLedgerAlreadyClosedException) { if (log.isDebugEnabled()) { log.debug("[{}] Failed to persist msg in store: {}", topic, exception.getMessage()); @@ -2510,6 +2513,18 @@ public CompletableFuture checkClusterMigration() { } } + public boolean isReplicationBacklogExist() { + ConcurrentOpenHashMap replicators = getReplicators(); + if (replicators != null) { + for (Replicator replicator : replicators.values()) { + if (replicator.getNumberOfEntriesInBacklog() != 0) { + return true; + } + } + } + return false; + } + @Override public void checkGC() { if (!isDeleteWhileInactive()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index cd31f9150e619..3fe8c22b1c4de 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -112,7 +112,7 @@ public abstract class MockedPulsarServiceBaseTest extends TestRetrySupport { protected URI lookupUrl; protected boolean isTcpLookup = false; - protected static final String configClusterName = "test"; + protected String configClusterName = "test"; protected boolean enableBrokerInterceptor = false; @@ -120,6 +120,12 @@ public MockedPulsarServiceBaseTest() { resetConfig(); } + protected void setupWithClusterName(String clusterName) throws Exception { + this.conf.setClusterName(clusterName); + this.configClusterName = clusterName; + this.internalSetup(); + } + protected PulsarService getPulsar() { return pulsar; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java index f376ea4541737..df4f66c43d2b4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java @@ -18,10 +18,12 @@ */ package org.apache.pulsar.broker.service; +import static java.lang.Thread.sleep; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import java.lang.reflect.Method; @@ -59,7 +61,8 @@ public class ClusterMigrationTest { protected String methodName; String namespace = "pulsar/migrationNs"; - TestBroker broker1, broker2; + TestBroker broker1, broker2, broker3, broker4; + URL url1; URL urlTls1; PulsarService pulsar1; @@ -71,6 +74,16 @@ public class ClusterMigrationTest { PulsarService pulsar2; PulsarAdmin admin2; + URL url3; + URL urlTls3; + PulsarService pulsar3; + PulsarAdmin admin3; + + URL url4; + URL urlTls4; + PulsarService pulsar4; + PulsarAdmin admin4; + @DataProvider(name = "TopicsubscriptionTypes") public Object[][] subscriptionTypes() { return new Object[][] { @@ -91,9 +104,10 @@ public void setup() throws Exception { log.info("--- Starting ReplicatorTestBase::setup ---"); - broker1 = new TestBroker(); - broker2 = new TestBroker(); - String clusterName = broker1.getClusterName(); + broker1 = new TestBroker("r1"); + broker2 = new TestBroker("r2"); + broker3 = new TestBroker("r3"); + broker4 = new TestBroker("r4"); pulsar1 = broker1.getPulsarService(); url1 = new URL(pulsar1.getWebServiceAddress()); @@ -105,32 +119,81 @@ public void setup() throws Exception { urlTls2 = new URL(pulsar2.getWebServiceAddressTls()); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); - // Start region 3 + pulsar3 = broker3.getPulsarService(); + url3 = new URL(pulsar3.getWebServiceAddress()); + urlTls3 = new URL(pulsar3.getWebServiceAddressTls()); + admin3 = PulsarAdmin.builder().serviceHttpUrl(url3.toString()).build(); - // Provision the global namespace - admin1.clusters().createCluster(clusterName, + pulsar4 = broker4.getPulsarService(); + url4 = new URL(pulsar4.getWebServiceAddress()); + urlTls4 = new URL(pulsar4.getWebServiceAddressTls()); + admin4 = PulsarAdmin.builder().serviceHttpUrl(url4.toString()).build(); + + + admin1.clusters().createCluster("r1", ClusterData.builder().serviceUrl(url1.toString()).serviceUrlTls(urlTls1.toString()) .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()).build()); - admin2.clusters().createCluster(clusterName, + admin3.clusters().createCluster("r1", + ClusterData.builder().serviceUrl(url1.toString()).serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()).build()); + admin2.clusters().createCluster("r2", + ClusterData.builder().serviceUrl(url2.toString()).serviceUrlTls(urlTls2.toString()) + .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()).build()); + admin4.clusters().createCluster("r2", ClusterData.builder().serviceUrl(url2.toString()).serviceUrlTls(urlTls2.toString()) .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()).build()); + admin1.clusters().createCluster("r3", + ClusterData.builder().serviceUrl(url3.toString()).serviceUrlTls(urlTls3.toString()) + .brokerServiceUrl(pulsar3.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()).build()); + + admin3.clusters().createCluster("r3", + ClusterData.builder().serviceUrl(url3.toString()).serviceUrlTls(urlTls3.toString()) + .brokerServiceUrl(pulsar3.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()).build()); + + admin2.clusters().createCluster("r4", + ClusterData.builder().serviceUrl(url4.toString()).serviceUrlTls(urlTls4.toString()) + .brokerServiceUrl(pulsar4.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()).build()); + admin4.clusters().createCluster("r4", + ClusterData.builder().serviceUrl(url4.toString()).serviceUrlTls(urlTls4.toString()) + .brokerServiceUrl(pulsar4.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()).build()); + + // Setting r3 as replication cluster for r1 admin1.tenants().createTenant("pulsar", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet(clusterName))); - admin1.namespaces().createNamespace(namespace, Sets.newHashSet(clusterName)); - + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3"))); + admin3.tenants().createTenant("pulsar", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3"))); + admin1.namespaces().createNamespace(namespace, Sets.newHashSet("r1", "r3")); + admin3.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r3")); + + // Setting r4 as replication cluster for r2 admin2.tenants().createTenant("pulsar", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet(clusterName))); - admin2.namespaces().createNamespace(namespace, Sets.newHashSet(clusterName)); - - assertEquals(admin1.clusters().getCluster(clusterName).getServiceUrl(), url1.toString()); - assertEquals(admin2.clusters().getCluster(clusterName).getServiceUrl(), url2.toString()); - assertEquals(admin1.clusters().getCluster(clusterName).getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl()); - assertEquals(admin2.clusters().getCluster(clusterName).getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()); - - Thread.sleep(100); + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4"))); + admin4.tenants().createTenant("pulsar", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4"))); + admin2.namespaces().createNamespace(namespace, Sets.newHashSet("r2", "r4")); + admin4.namespaces().createNamespace(namespace); + admin2.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r2", "r4")); + + assertEquals(admin1.clusters().getCluster("r1").getServiceUrl(), url1.toString()); + assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString()); + assertEquals(admin3.clusters().getCluster("r3").getServiceUrl(), url3.toString()); + assertEquals(admin4.clusters().getCluster("r4").getServiceUrl(), url4.toString()); + assertEquals(admin1.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl()); + assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()); + assertEquals(admin3.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl()); + assertEquals(admin4.clusters().getCluster("r4").getBrokerServiceUrl(), pulsar4.getBrokerServiceUrl()); + + sleep(100); log.info("--- ReplicatorTestBase::setup completed ---"); } @@ -140,6 +203,8 @@ protected void cleanup() throws Exception { log.info("--- Shutting down ---"); broker1.cleanup(); broker2.cleanup(); + broker3.cleanup(); + broker4.cleanup(); } @BeforeMethod(alwaysRun = true) @@ -154,20 +219,19 @@ public void beforeMethod(Method m) throws Exception { * (3) Migrate topic to cluster-2 * (4) Validate producer-1 is connected to cluster-2 * (5) create consumer1, drain backlog and migrate and reconnect to cluster-2 - * (6) Create new consumer2 with different subscription on cluster-1, + * (6) Create new consumer2 with different subscription on cluster-1, * which immediately migrate and reconnect to cluster-2 * (7) Create producer-2 directly to cluster-2 - * (8) Create producer-3 on cluster-1 which should be redirected to cluster-2 + * (8) Create producer-3 on cluster-1 which should be redirected to cluster-2 * (8) Publish messages using producer1, producer2, and producer3 * (9) Consume all messages by both consumer1 and consumer2 - * (10) Create Producer/consumer on non-migrated cluster and verify their connection with cluster-1 - * (11) Restart Broker-1 and connect producer/consumer on cluster-1 + * (10) Create Producer/consumer on non-migrated cluster and verify their connection with cluster-1 + * (11) Restart Broker-1 and connect producer/consumer on cluster-1 * @throws Exception */ @Test(dataProvider = "TopicsubscriptionTypes") public void testClusterMigration(boolean persistent, SubscriptionType subType) throws Exception { log.info("--- Starting ReplicatorTest::testClusterMigration ---"); - persistent = false; final String topicName = BrokerTestUtil .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic"); @@ -202,7 +266,7 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t assertFalse(topic2.getProducers().isEmpty()); ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); - admin1.clusters().updateClusterMigration(broker2.getClusterName(), true, migratedUrl); + admin1.clusters().updateClusterMigration("r1", true, migratedUrl); retryStrategically((test) -> { try { @@ -214,12 +278,16 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t return false; }, 10, 500); + topic1.checkClusterMigration().get(); + log.info("before sending message"); + sleep(1000); producer1.sendAsync("test1".getBytes()); // producer is disconnected from cluster-1 retryStrategically((test) -> topic1.getProducers().isEmpty(), 10, 500); + log.info("before asserting"); assertTrue(topic1.getProducers().isEmpty()); // create 3rd producer on cluster-1 which should be redirected to cluster-2 @@ -297,15 +365,116 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t log.info("Successfully consumed messages by migrated consumers"); } + @Test(dataProvider = "TopicsubscriptionTypes") + public void testClusterMigrationWithReplicationBacklog(boolean persistent, SubscriptionType subType) throws Exception { + log.info("--- Starting ReplicatorTest::testClusterMigrationWithReplicationBacklog ---"); + persistent = true; + final String topicName = BrokerTestUtil + .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic"); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + @Cleanup + PulsarClient client3 = PulsarClient.builder().serviceUrl(url3.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + // cluster-1 producer/consumer + Producer producer1 = client1.newProducer().topic(topicName).enableBatching(false) + .producerName("cluster1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + + // cluster-3 consumer + Consumer consumer3 = client3.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + AbstractTopic topic1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null).get(); + retryStrategically((test) -> !topic1.getProducers().isEmpty(), 5, 500); + retryStrategically((test) -> !topic1.getSubscriptions().isEmpty(), 5, 500); + assertFalse(topic1.getProducers().isEmpty()); + assertFalse(topic1.getSubscriptions().isEmpty()); + + // build backlog + consumer1.close(); + retryStrategically((test) -> topic1.getReplicators().size() == 1, 10, 3000); + assertEquals(topic1.getReplicators().size(), 1); + + // stop service in the replication cluster to build replication backlog + broker3.cleanup(); + retryStrategically((test) -> broker3.getPulsarService() == null, 10, 1000); + assertNull(pulsar3.getBrokerService()); + + //publish messages into topic in "r1" cluster + int n = 5; + for (int i = 0; i < n; i++) { + producer1.send("test1".getBytes()); + } + retryStrategically((test) -> topic1.isReplicationBacklogExist(), 10, 1000); + assertTrue(topic1.isReplicationBacklogExist()); + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + // cluster-2 producer/consumer + Producer producer2 = client2.newProducer().topic(topicName).enableBatching(false) + .producerName("cluster2-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + AbstractTopic topic2 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName, false).getNow(null).get(); + log.info("name of topic 2 - {}", topic2.getName()); + assertFalse(topic2.getProducers().isEmpty()); + + retryStrategically((test) -> topic2.getReplicators().size() == 1, 10, 2000); + log.info("replicators should be ready"); + ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); + admin1.clusters().updateClusterMigration("r1", true, migratedUrl); + log.info("update cluster migration called"); + retryStrategically((test) -> { + try { + topic1.checkClusterMigration().get(); + return true; + } catch (Exception e) { + // ok + } + return false; + }, 10, 500); + + topic1.checkClusterMigration().get(); + + producer1.sendAsync("test1".getBytes()); + + // producer is disconnected from cluster-1 + retryStrategically((test) -> topic1.getProducers().isEmpty(), 10, 500); + assertTrue(topic1.getProducers().isEmpty()); + + // verify that the disconnected producer is not redirected + // to replication cluster since there is replication backlog. + assertEquals(topic2.getProducers().size(), 1); + + // Restart the service in cluster "r3". + broker3.restart(); + retryStrategically((test) -> broker3.getPulsarService() != null, 10, 1000); + assertNotNull(broker3.getPulsarService()); + pulsar3 = broker3.getPulsarService(); + + // verify that the replication backlog drains once service in cluster "r3" is restarted. + retryStrategically((test) -> !topic1.isReplicationBacklogExist(), 10, 1000); + assertFalse(topic1.isReplicationBacklogExist()); + + // verify that the producer1 is now is now connected to migrated cluster "r2" since backlog is cleared. + retryStrategically((test) -> topic2.getProducers().size()==2, 10, 500); + assertEquals(topic2.getProducers().size(), 2); + } + static class TestBroker extends MockedPulsarServiceBaseTest { - public TestBroker() throws Exception { + private String clusterName; + + public TestBroker(String clusterName) throws Exception { + this.clusterName = clusterName; setup(); } @Override protected void setup() throws Exception { - super.internalSetup(); + super.setupWithClusterName(clusterName); } public PulsarService getPulsarService() { @@ -318,9 +487,9 @@ public String getClusterName() { @Override protected void cleanup() throws Exception { - internalCleanup(); + stopBroker(); } - + public void restart() throws Exception { restartBroker(); } From 08b28f5c5cbbdc63467455c40106173475ff01e5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 10 Apr 2023 22:56:11 +0300 Subject: [PATCH 081/280] [improve][broker] Prevent range conflicts with Key Shared sticky consumers when TCP/IP connections get orphaned (#20026) --- .../pulsar/broker/ServiceConfiguration.java | 9 + ...bstractDispatcherSingleActiveConsumer.java | 10 +- ...stentHashingStickyKeyConsumerSelector.java | 5 +- .../pulsar/broker/service/Dispatcher.java | 2 +- ...ngeAutoSplitStickyKeyConsumerSelector.java | 11 +- ...ngeExclusiveStickyKeyConsumerSelector.java | 53 +++-- .../pulsar/broker/service/ServerCnx.java | 48 ++++- .../service/StickyKeyConsumerSelector.java | 4 +- .../pulsar/broker/service/TransportCnx.java | 9 + ...PersistentDispatcherMultipleConsumers.java | 8 +- ...tStickyKeyDispatcherMultipleConsumers.java | 26 ++- .../NonPersistentSubscription.java | 7 +- ...PersistentDispatcherMultipleConsumers.java | 9 +- ...tStickyKeyDispatcherMultipleConsumers.java | 50 +++-- .../persistent/PersistentSubscription.java | 7 +- .../service/AbstractBaseDispatcherTest.java | 4 +- ...xclusiveStickyKeyConsumerSelectorTest.java | 81 +++++--- .../pulsar/common/protocol/PulsarHandler.java | 23 ++- pulsar-proxy/pom.xml | 5 + .../proxy/server/LookupProxyHandler.java | 7 +- .../pulsar/proxy/server/ProxyConnection.java | 2 +- .../pulsar/proxy/server/ProxyService.java | 4 + .../server/ProxyStuckConnectionTest.java | 194 ++++++++++++++++++ 23 files changed, 463 insertions(+), 115 deletions(-) create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index c86358b9b5b9e..cea159b74fd1d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2676,6 +2676,15 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "How often to check pulsar connection is still alive" ) private int keepAliveIntervalSeconds = 30; + @FieldContext( + category = CATEGORY_SERVER, + doc = "Timeout for connection liveness check used to check liveness of possible consumer or producer " + + "duplicates. Helps prevent ProducerFencedException with exclusive producer, " + + "ConsumerAssignException with range conflict for Key Shared with sticky hash ranges or " + + "ConsumerBusyException in the case of an exclusive consumer. Set to 0 to disable connection " + + "liveness check." + ) + private long connectionLivenessCheckTimeoutMillis = 5000L; @Deprecated @FieldContext( category = CATEGORY_POLICIES, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java index 1b6df5f809327..5098890242b6c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java @@ -36,6 +36,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; import org.apache.pulsar.client.impl.Murmur3Hash32; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Murmur3_32Hash; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,21 +158,21 @@ private NavigableMap makeHashRing(int consumerSize) { return Collections.unmodifiableNavigableMap(hashRing); } - public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + public synchronized CompletableFuture addConsumer(Consumer consumer) { if (IS_CLOSED_UPDATER.get(this) == TRUE) { log.warn("[{}] Dispatcher is already closed. Closing consumer {}", this.topicName, consumer); consumer.disconnect(); - return; + return CompletableFuture.completedFuture(null); } if (subscriptionType == SubType.Exclusive && !consumers.isEmpty()) { - throw new ConsumerBusyException("Exclusive consumer is already connected"); + return FutureUtil.failedFuture(new ConsumerBusyException("Exclusive consumer is already connected")); } if (subscriptionType == SubType.Failover && isConsumersExceededOnSubscription()) { log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit", this.topicName); - throw new ConsumerBusyException("Subscription reached max consumers limit"); + return FutureUtil.failedFuture(new ConsumerBusyException("Subscription reached max consumers limit")); } if (subscriptionType == SubType.Exclusive @@ -203,6 +204,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce } } + return CompletableFuture.completedFuture(null); } public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceException { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java index 30dc1248d0c7d..ea491bd40d332 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java @@ -26,9 +26,9 @@ import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; import org.apache.pulsar.client.api.Range; import org.apache.pulsar.common.util.Murmur3_32Hash; @@ -53,7 +53,7 @@ public ConsistentHashingStickyKeyConsumerSelector(int numberOfPoints) { } @Override - public void addConsumer(Consumer consumer) throws ConsumerAssignException { + public CompletableFuture addConsumer(Consumer consumer) { rwLock.writeLock().lock(); try { // Insert multiple points on the hash ring for every consumer @@ -73,6 +73,7 @@ public void addConsumer(Consumer consumer) throws ConsumerAssignException { } }); } + return CompletableFuture.completedFuture(null); } finally { rwLock.writeLock().unlock(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java index 9b0c4e885e64d..3ca06dc83d9aa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java @@ -27,7 +27,7 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; public interface Dispatcher { - void addConsumer(Consumer consumer) throws BrokerServiceException; + CompletableFuture addConsumer(Consumer consumer); void removeConsumer(Consumer consumer) throws BrokerServiceException; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java index 59b815197694e..a9fea5b39bf82 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java @@ -23,9 +23,11 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; import org.apache.pulsar.client.api.Range; +import org.apache.pulsar.common.util.FutureUtil; /** * This is a consumer selector based fixed hash range. @@ -77,13 +79,18 @@ public HashRangeAutoSplitStickyKeyConsumerSelector(int rangeSize) { } @Override - public synchronized void addConsumer(Consumer consumer) throws ConsumerAssignException { + public synchronized CompletableFuture addConsumer(Consumer consumer) { if (rangeMap.isEmpty()) { rangeMap.put(rangeSize, consumer); consumerRange.put(consumer, rangeSize); } else { - splitRange(findBiggestRange(), consumer); + try { + splitRange(findBiggestRange(), consumer); + } catch (ConsumerAssignException e) { + return FutureUtil.failedFuture(e); + } } + return CompletableFuture.completedFuture(null); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java index 45be6d65b09b4..78bad1b2c400e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java @@ -23,10 +23,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import org.apache.pulsar.client.api.Range; import org.apache.pulsar.common.api.proto.IntRange; import org.apache.pulsar.common.api.proto.KeySharedMeta; +import org.apache.pulsar.common.util.FutureUtil; /** * This is a sticky-key consumer selector based user provided range. @@ -52,8 +54,23 @@ public HashRangeExclusiveStickyKeyConsumerSelector(int rangeSize) { } @Override - public void addConsumer(Consumer consumer) throws BrokerServiceException.ConsumerAssignException { - validateKeySharedMeta(consumer); + public synchronized CompletableFuture addConsumer(Consumer consumer) { + return validateKeySharedMeta(consumer).thenRun(() -> { + try { + internalAddConsumer(consumer); + } catch (BrokerServiceException.ConsumerAssignException e) { + throw FutureUtil.wrapToCompletionException(e); + } + }); + } + + private synchronized void internalAddConsumer(Consumer consumer) + throws BrokerServiceException.ConsumerAssignException { + Consumer conflictingConsumer = findConflictingConsumer(consumer.getKeySharedMeta().getHashRangesList()); + if (conflictingConsumer != null) { + throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " + + conflictingConsumer); + } for (IntRange intRange : consumer.getKeySharedMeta().getHashRangesList()) { rangeMap.put(intRange.getStart(), consumer); rangeMap.put(intRange.getEnd(), consumer); @@ -101,31 +118,41 @@ public Consumer select(int hash) { } } - private void validateKeySharedMeta(Consumer consumer) throws BrokerServiceException.ConsumerAssignException { + private synchronized CompletableFuture validateKeySharedMeta(Consumer consumer) { if (consumer.getKeySharedMeta() == null) { - throw new BrokerServiceException.ConsumerAssignException("Must specify key shared meta for consumer."); + return FutureUtil.failedFuture( + new BrokerServiceException.ConsumerAssignException("Must specify key shared meta for consumer.")); } List ranges = consumer.getKeySharedMeta().getHashRangesList(); if (ranges.isEmpty()) { - throw new BrokerServiceException.ConsumerAssignException("Ranges for KeyShared policy must not be empty."); + return FutureUtil.failedFuture(new BrokerServiceException.ConsumerAssignException( + "Ranges for KeyShared policy must not be empty.")); } for (IntRange intRange : ranges) { - if (intRange.getStart() > intRange.getEnd()) { - throw new BrokerServiceException.ConsumerAssignException("Fixed hash range start > end"); + return FutureUtil.failedFuture( + new BrokerServiceException.ConsumerAssignException("Fixed hash range start > end")); } + } + Consumer conflictingConsumer = findConflictingConsumer(ranges); + if (conflictingConsumer != null) { + return conflictingConsumer.cnx().checkConnectionLiveness().thenRun(() -> {}); + } else { + return CompletableFuture.completedFuture(null); + } + } + private synchronized Consumer findConflictingConsumer(List ranges) { + for (IntRange intRange : ranges) { Map.Entry ceilingEntry = rangeMap.ceilingEntry(intRange.getStart()); Map.Entry floorEntry = rangeMap.floorEntry(intRange.getEnd()); if (floorEntry != null && floorEntry.getKey() >= intRange.getStart()) { - throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " - + floorEntry.getValue()); + return floorEntry.getValue(); } if (ceilingEntry != null && ceilingEntry.getKey() <= intRange.getEnd()) { - throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " - + ceilingEntry.getValue()); + return ceilingEntry.getValue(); } if (ceilingEntry != null && floorEntry != null && ceilingEntry.getValue().equals(floorEntry.getValue())) { @@ -134,12 +161,12 @@ private void validateKeySharedMeta(Consumer consumer) throws BrokerServiceExcept int start = Math.max(intRange.getStart(), range.getStart()); int end = Math.min(intRange.getEnd(), range.getEnd()); if (end >= start) { - throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " - + ceilingEntry.getValue()); + return ceilingEntry.getValue(); } } } } + return null; } Map getRangeConsumer() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 1b376492accc5..677ed25ef2163 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -164,6 +164,7 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.common.util.netty.NettyChannelUtil; +import org.apache.pulsar.common.util.netty.NettyFutureUtil; import org.apache.pulsar.functions.utils.Exceptions; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; @@ -236,6 +237,8 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private final long maxPendingBytesPerThread; private final long resumeThresholdPendingBytesPerThread; + private final long connectionLivenessCheckTimeoutMillis; + // Number of bytes pending to be published from a single specific IO thread. private static final FastThreadLocal pendingBytesPerThread = new FastThreadLocal() { @Override @@ -252,7 +255,6 @@ protected Set initialValue() throws Exception { } }; - enum State { Start, Connected, Failed, Connecting } @@ -272,6 +274,8 @@ public ServerCnx(PulsarService pulsar, String listenerName) { this.state = State.Start; ServiceConfiguration conf = pulsar.getConfiguration(); + this.connectionLivenessCheckTimeoutMillis = conf.getConnectionLivenessCheckTimeoutMillis(); + // This maps are not heavily contended since most accesses are within the cnx thread this.producers = ConcurrentLongHashMap.>newBuilder() .expectedItems(8) @@ -375,6 +379,11 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { }); this.topicListService.inactivate(); this.service.getPulsarStats().recordConnectionClose(); + + // complete possible pending connection check future + if (connectionCheckInProgress != null && !connectionCheckInProgress.isDone()) { + connectionCheckInProgress.complete(false); + } } @Override @@ -3325,6 +3334,43 @@ public String clientSourceAddress() { } } + CompletableFuture connectionCheckInProgress; + + @Override + public CompletableFuture checkConnectionLiveness() { + if (connectionLivenessCheckTimeoutMillis > 0) { + return NettyFutureUtil.toCompletableFuture(ctx.executor().submit(() -> { + if (connectionCheckInProgress != null) { + return connectionCheckInProgress; + } else { + final CompletableFuture finalConnectionCheckInProgress = new CompletableFuture<>(); + connectionCheckInProgress = finalConnectionCheckInProgress; + ctx.executor().schedule(() -> { + if (finalConnectionCheckInProgress == connectionCheckInProgress + && !finalConnectionCheckInProgress.isDone()) { + log.warn("[{}] Connection check timed out. Closing connection.", remoteAddress); + ctx.close(); + } + }, connectionLivenessCheckTimeoutMillis, TimeUnit.MILLISECONDS); + sendPing(); + return finalConnectionCheckInProgress; + } + })).thenCompose(java.util.function.Function.identity()); + } else { + // check is disabled + return CompletableFuture.completedFuture((Boolean) null); + } + } + + @Override + protected void messageReceived() { + super.messageReceived(); + if (connectionCheckInProgress != null && !connectionCheckInProgress.isDone()) { + connectionCheckInProgress.complete(true); + connectionCheckInProgress = null; + } + } + private static void logAuthException(SocketAddress remoteAddress, String operation, String principal, Optional topic, Throwable ex) { String topicString = topic.map(t -> ", topic=" + t.toString()).orElse(""); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java index 511445da71dbe..e0ed75020bc82 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.Map; -import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.client.api.Range; import org.apache.pulsar.common.util.Murmur3_32Hash; @@ -33,7 +33,7 @@ public interface StickyKeyConsumerSelector { * * @param consumer new consumer */ - void addConsumer(Consumer consumer) throws ConsumerAssignException; + CompletableFuture addConsumer(Consumer consumer); /** * Remove the consumer. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java index 2abacf80ef3d2..f1aaca2b290fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java @@ -21,6 +21,7 @@ import io.netty.handler.codec.haproxy.HAProxyMessage; import io.netty.util.concurrent.Promise; import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; public interface TransportCnx { @@ -78,4 +79,12 @@ public interface TransportCnx { String clientSourceAddress(); + /*** + * Check if the connection is still alive + * by actively sending a Ping message to the client. + * + * @return a completable future where the result is true if the connection is alive, false otherwise. The result + * is null if the connection liveness check is disabled. + */ + CompletableFuture checkConnectionLiveness(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java index d679106094178..c106b1603f6bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java @@ -35,6 +35,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.stats.Rate; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,20 +68,21 @@ public NonPersistentDispatcherMultipleConsumers(NonPersistentTopic topic, Subscr } @Override - public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + public synchronized CompletableFuture addConsumer(Consumer consumer) { if (IS_CLOSED_UPDATER.get(this) == TRUE) { log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer); consumer.disconnect(); - return; + return CompletableFuture.completedFuture(null); } if (isConsumersExceededOnSubscription()) { log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit", name); - throw new ConsumerBusyException("Subscription reached max consumers limit"); + return FutureUtil.failedFuture(new ConsumerBusyException("Subscription reached max consumers limit")); } consumerList.add(consumer); consumerSet.add(consumer); + return CompletableFuture.completedFuture(null); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java index 23814bfdbe22e..2cad253f96ee2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Entry; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -39,6 +40,7 @@ import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,20 +87,24 @@ public NonPersistentStickyKeyDispatcherMultipleConsumers(NonPersistentTopic topi } @Override - public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + public synchronized CompletableFuture addConsumer(Consumer consumer) { if (IS_CLOSED_UPDATER.get(this) == TRUE) { log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer); consumer.disconnect(); - return; - } - super.addConsumer(consumer); - try { - selector.addConsumer(consumer); - } catch (BrokerServiceException e) { - consumerSet.removeAll(consumer); - consumerList.remove(consumer); - throw e; + return CompletableFuture.completedFuture(null); } + return super.addConsumer(consumer).thenCompose(__ -> + selector.addConsumer(consumer).handle((value, ex) -> { + if (ex != null) { + synchronized (NonPersistentStickyKeyDispatcherMultipleConsumers.this) { + consumerSet.removeAll(consumer); + consumerList.remove(consumer); + } + throw FutureUtil.wrapToCompletionException(ex); + } else { + return value; + } + })); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java index 3af886a528153..fc6df4a340997 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java @@ -172,12 +172,7 @@ public synchronized CompletableFuture addConsumer(Consumer consumer) { } } - try { - dispatcher.addConsumer(consumer); - return CompletableFuture.completedFuture(null); - } catch (BrokerServiceException brokerServiceException) { - return FutureUtil.failedFuture(brokerServiceException); - } + return dispatcher.addConsumer(consumer); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 7ff6e72d02aed..9540f6efe0005 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -75,6 +75,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -153,11 +154,11 @@ public PersistentDispatcherMultipleConsumers(PersistentTopic topic, ManagedCurso } @Override - public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + public synchronized CompletableFuture addConsumer(Consumer consumer) { if (IS_CLOSED_UPDATER.get(this) == TRUE) { log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer); consumer.disconnect(); - return; + return CompletableFuture.completedFuture(null); } if (consumerList.isEmpty()) { if (havePendingRead || havePendingReplayRead) { @@ -178,7 +179,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce if (isConsumersExceededOnSubscription()) { log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit", name); - throw new ConsumerBusyException("Subscription reached max consumers limit"); + return FutureUtil.failedFuture(new ConsumerBusyException("Subscription reached max consumers limit")); } consumerList.add(consumer); @@ -187,6 +188,8 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce consumerList.sort(Comparator.comparingInt(Consumer::getPriorityLevel)); } consumerSet.add(consumer); + + return CompletableFuture.completedFuture(null); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 0c29421edd4ef..a0292729e734f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.NavigableSet; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -52,6 +53,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,31 +109,37 @@ public StickyKeyConsumerSelector getSelector() { } @Override - public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + public synchronized CompletableFuture addConsumer(Consumer consumer) { if (IS_CLOSED_UPDATER.get(this) == TRUE) { log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer); consumer.disconnect(); - return; - } - super.addConsumer(consumer); - try { - selector.addConsumer(consumer); - } catch (BrokerServiceException e) { - consumerSet.removeAll(consumer); - consumerList.remove(consumer); - throw e; - } - - PositionImpl readPositionWhenJoining = (PositionImpl) cursor.getReadPosition(); - consumer.setReadPositionWhenJoining(readPositionWhenJoining); - // If this was the 1st consumer, or if all the messages are already acked, then we - // don't need to do anything special - if (!allowOutOfOrderDelivery - && recentlyJoinedConsumers != null - && consumerList.size() > 1 - && cursor.getNumberOfEntriesSinceFirstNotAckedMessage() > 1) { - recentlyJoinedConsumers.put(consumer, readPositionWhenJoining); + return CompletableFuture.completedFuture(null); } + return super.addConsumer(consumer).thenCompose(__ -> + selector.addConsumer(consumer).handle((result, ex) -> { + if (ex != null) { + synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) { + consumerSet.removeAll(consumer); + consumerList.remove(consumer); + } + throw FutureUtil.wrapToCompletionException(ex); + } + return result; + }) + ).thenRun(() -> { + synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) { + PositionImpl readPositionWhenJoining = (PositionImpl) cursor.getReadPosition(); + consumer.setReadPositionWhenJoining(readPositionWhenJoining); + // If this was the 1st consumer, or if all the messages are already acked, then we + // don't need to do anything special + if (!allowOutOfOrderDelivery + && recentlyJoinedConsumers != null + && consumerList.size() > 1 + && cursor.getNumberOfEntriesSinceFirstNotAckedMessage() > 1) { + recentlyJoinedConsumers.put(consumer, readPositionWhenJoining); + } + } + }); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 4ed191a9b4f61..d283cac77c7b6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -293,12 +293,7 @@ public CompletableFuture addConsumer(Consumer consumer) { } } - try { - dispatcher.addConsumer(consumer); - return CompletableFuture.completedFuture(null); - } catch (BrokerServiceException brokerServiceException) { - return FutureUtil.failedFuture(brokerServiceException); - } + return dispatcher.addConsumer(consumer); } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java index cc2ec3444d54d..332cccc2d2c6a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java @@ -249,8 +249,8 @@ protected void reScheduleRead() { } @Override - public void addConsumer(Consumer consumer) throws BrokerServiceException { - + public CompletableFuture addConsumer(Consumer consumer) { + return CompletableFuture.completedFuture(null); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java index e36c656e3faea..f3828981c8edd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.broker.service; import static org.mockito.Mockito.mock; @@ -26,6 +27,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import org.apache.pulsar.client.api.Range; import org.apache.pulsar.common.api.proto.IntRange; import org.apache.pulsar.common.api.proto.KeySharedMeta; @@ -37,7 +40,7 @@ public class HashRangeExclusiveStickyKeyConsumerSelectorTest { @Test - public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignException { + public void testConsumerSelect() throws ExecutionException, InterruptedException { HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10); Consumer consumer1 = mock(Consumer.class); @@ -46,8 +49,8 @@ public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignExc keySharedMeta1.addHashRange().setStart(0).setEnd(2); when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1); Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1); - selector.addConsumer(consumer1); - Assert.assertEquals(selector.getRangeConsumer().size(),2); + selector.addConsumer(consumer1).get(); + Assert.assertEquals(selector.getRangeConsumer().size(), 2); Consumer selectedConsumer; for (int i = 0; i < 3; i++) { selectedConsumer = selector.select(i); @@ -62,8 +65,8 @@ public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignExc keySharedMeta2.addHashRange().setStart(3).setEnd(9); when(consumer2.getKeySharedMeta()).thenReturn(keySharedMeta2); Assert.assertEquals(consumer2.getKeySharedMeta(), keySharedMeta2); - selector.addConsumer(consumer2); - Assert.assertEquals(selector.getRangeConsumer().size(),4); + selector.addConsumer(consumer2).get(); + Assert.assertEquals(selector.getRangeConsumer().size(), 4); for (int i = 3; i < 10; i++) { selectedConsumer = selector.select(i); @@ -76,32 +79,46 @@ public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignExc } selector.removeConsumer(consumer1); - Assert.assertEquals(selector.getRangeConsumer().size(),2); + Assert.assertEquals(selector.getRangeConsumer().size(), 2); selectedConsumer = selector.select(1); Assert.assertNull(selectedConsumer); selector.removeConsumer(consumer2); - Assert.assertEquals(selector.getRangeConsumer().size(),0); + Assert.assertEquals(selector.getRangeConsumer().size(), 0); selectedConsumer = selector.select(5); Assert.assertNull(selectedConsumer); } - @Test(expectedExceptions = BrokerServiceException.ConsumerAssignException.class) - public void testEmptyRanges() throws BrokerServiceException.ConsumerAssignException { + @Test + public void testEmptyRanges() { HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10); Consumer consumer = mock(Consumer.class); KeySharedMeta keySharedMeta = new KeySharedMeta() .setKeySharedMode(KeySharedMode.STICKY); when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta); - selector.addConsumer(consumer); + try { + selector.addConsumer(consumer).get(); + Assert.fail("Should have failed"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerAssignException); + } } - @Test(expectedExceptions = BrokerServiceException.ConsumerAssignException.class) - public void testNullKeySharedMeta() throws BrokerServiceException.ConsumerAssignException { + @Test + public void testNullKeySharedMeta() throws ExecutionException, InterruptedException { HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10); Consumer consumer = mock(Consumer.class); when(consumer.getKeySharedMeta()).thenReturn(null); - selector.addConsumer(consumer); + try { + selector.addConsumer(consumer).get(); + Assert.fail("Should have failed"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerAssignException); + } } @Test(expectedExceptions = IllegalArgumentException.class) @@ -110,7 +127,7 @@ public void testInvalidRangeTotal() { } @Test - public void testGetConsumerKeyHashRanges() throws BrokerServiceException.ConsumerAssignException { + public void testGetConsumerKeyHashRanges() throws ExecutionException, InterruptedException { HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10); List consumerName = Arrays.asList("consumer1", "consumer2", "consumer3", "consumer4"); List range = Arrays.asList(new int[] {0, 2}, new int[] {3, 7}, new int[] {9, 12}, new int[] {15, 20}); @@ -125,7 +142,7 @@ public void testGetConsumerKeyHashRanges() throws BrokerServiceException.Consume when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta); when(consumer.consumerName()).thenReturn(consumerName.get(index)); Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta); - selector.addConsumer(consumer); + selector.addConsumer(consumer).get(); consumers.add(consumer); } @@ -157,7 +174,7 @@ public void testGetConsumerKeyHashRangesWithSameConsumerName() throws Exception when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta); when(consumer.consumerName()).thenReturn(consumerName); Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta); - selector.addConsumer(consumer); + selector.addConsumer(consumer).get(); consumers.add(consumer); } @@ -173,16 +190,19 @@ public void testGetConsumerKeyHashRangesWithSameConsumerName() throws Exception } @Test - public void testSingleRangeConflict() throws BrokerServiceException.ConsumerAssignException { + public void testSingleRangeConflict() throws ExecutionException, InterruptedException { HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10); Consumer consumer1 = mock(Consumer.class); + TransportCnx transportCnx = mock(TransportCnx.class); + when(consumer1.cnx()).thenReturn(transportCnx); + when(transportCnx.checkConnectionLiveness()).thenReturn(CompletableFuture.completedFuture(null)); KeySharedMeta keySharedMeta1 = new KeySharedMeta() .setKeySharedMode(KeySharedMode.STICKY); keySharedMeta1.addHashRange().setStart(2).setEnd(5); when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1); Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1); - selector.addConsumer(consumer1); - Assert.assertEquals(selector.getRangeConsumer().size(),2); + selector.addConsumer(consumer1).get(); + Assert.assertEquals(selector.getRangeConsumer().size(), 2); final List testRanges = new ArrayList<>(); testRanges.add(new IntRange().setStart(4).setEnd(6)); @@ -203,25 +223,29 @@ public void testSingleRangeConflict() throws BrokerServiceException.ConsumerAssi when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta); Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta); try { - selector.addConsumer(consumer); + selector.addConsumer(consumer).get(); Assert.fail("should be failed"); - } catch (BrokerServiceException.ConsumerAssignException ignore) { + } catch (ExecutionException | InterruptedException e) { + // ignore } - Assert.assertEquals(selector.getRangeConsumer().size(),2); + Assert.assertEquals(selector.getRangeConsumer().size(), 2); } } @Test - public void testMultipleRangeConflict() throws BrokerServiceException.ConsumerAssignException { + public void testMultipleRangeConflict() throws ExecutionException, InterruptedException { HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10); Consumer consumer1 = mock(Consumer.class); + TransportCnx transportCnx = mock(TransportCnx.class); + when(consumer1.cnx()).thenReturn(transportCnx); + when(transportCnx.checkConnectionLiveness()).thenReturn(CompletableFuture.completedFuture(null)); KeySharedMeta keySharedMeta1 = new KeySharedMeta() .setKeySharedMode(KeySharedMode.STICKY); keySharedMeta1.addHashRange().setStart(2).setEnd(5); when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1); Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1); - selector.addConsumer(consumer1); - Assert.assertEquals(selector.getRangeConsumer().size(),2); + selector.addConsumer(consumer1).get(); + Assert.assertEquals(selector.getRangeConsumer().size(), 2); final List> testRanges = new ArrayList<>(); testRanges.add(List.of( @@ -242,11 +266,12 @@ public void testMultipleRangeConflict() throws BrokerServiceException.ConsumerAs when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta); Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta); try { - selector.addConsumer(consumer); + selector.addConsumer(consumer).get(); Assert.fail("should be failed"); - } catch (BrokerServiceException.ConsumerAssignException ignore) { + } catch (ExecutionException | InterruptedException e) { + // ignore } - Assert.assertEquals(selector.getRangeConsumer().size(),2); + Assert.assertEquals(selector.getRangeConsumer().size(), 2); } } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java index b28f7a6028084..51cd61afd6362 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java @@ -19,6 +19,7 @@ package org.apache.pulsar.common.protocol; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.util.concurrent.ScheduledFuture; import java.net.SocketAddress; @@ -56,7 +57,7 @@ public PulsarHandler(int keepAliveInterval, TimeUnit unit) { } @Override - protected final void messageReceived() { + protected void messageReceived() { waitingForPingResponse = false; } @@ -120,14 +121,7 @@ private void handleKeepAliveTimeout() { log.debug("[{}] Sending ping message", ctx.channel()); } waitingForPingResponse = true; - ctx.writeAndFlush(Commands.newPing()) - .addListener(future -> { - if (!future.isSuccess()) { - log.warn("[{}] Forcing connection to close since cannot send a ping message.", - ctx.channel(), future.cause()); - ctx.close(); - } - }); + sendPing(); } else { if (log.isDebugEnabled()) { log.debug("[{}] Peer doesn't support keep-alive", ctx.channel()); @@ -135,6 +129,17 @@ private void handleKeepAliveTimeout() { } } + protected ChannelFuture sendPing() { + return ctx.writeAndFlush(Commands.newPing()) + .addListener(future -> { + if (!future.isSuccess()) { + log.warn("[{}] Forcing connection to close since cannot send a ping message.", + ctx.channel(), future.cause()); + ctx.close(); + } + }); + } + public void cancelKeepAliveTask() { if (keepAliveTask != null) { keepAliveTask.cancel(false); diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 03ec0aed8b56d..b9a68cba2c5e1 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -180,6 +180,11 @@ ipaddress ${seancfoley.ipaddress.version} + + org.testcontainers + testcontainers + test + diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java index 6ec597ec1cfc3..b62b3bacf0114 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java @@ -29,6 +29,7 @@ import java.util.concurrent.Semaphore; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.BinaryProtoLookupService; import org.apache.pulsar.common.api.proto.CommandGetSchema; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.api.proto.CommandLookupTopic; @@ -155,7 +156,7 @@ private void performLookup(long clientRequestId, String topic, String brokerServ writeAndFlush( Commands.newLookupErrorResponse(getServerError(t), t.getMessage(), clientRequestId)); } else { - String brokerUrl = connectWithTLS ? r.brokerUrlTls : r.brokerUrl; + String brokerUrl = resolveBrokerUrlFromLookupDataResult(r); if (r.redirect) { // Need to try the lookup again on a different broker performLookup(clientRequestId, topic, brokerUrl, r.authoritative, numberOfRetries - 1); @@ -186,6 +187,10 @@ private void performLookup(long clientRequestId, String topic, String brokerServ }); } + protected String resolveBrokerUrlFromLookupDataResult(BinaryProtoLookupService.LookupDataResult r) { + return connectWithTLS ? r.brokerUrlTls : r.brokerUrl; + } + public void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata partitionMetadata) { PARTITIONS_METADATA_REQUESTS.inc(); if (log.isDebugEnabled()) { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 9530389b524b3..36b3b4c6038fb 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -374,7 +374,7 @@ private synchronized void completeConnect() throws PulsarClientException { // and we'll take care of just topics and // partitions metadata lookups state = State.ProxyLookupRequests; - lookupProxyHandler = new LookupProxyHandler(service, this); + lookupProxyHandler = service.newLookupProxyHandler(this); final ByteBuf msg = Commands.newConnected(protocolVersionToAdvertise, false); writeAndFlush(msg); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index 4cca24f5f4892..a934b8b078426 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -528,4 +528,8 @@ public synchronized void addPrometheusRawMetricsProvider(PrometheusRawMetricsPro } private static final Logger LOG = LoggerFactory.getLogger(ProxyService.class); + + protected LookupProxyHandler newLookupProxyHandler(ProxyConnection proxyConnection) { + return new LookupProxyHandler(this, proxyConnection); + } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java new file mode 100644 index 0000000000000..97279659af626 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.proxy.server; + +import static org.mockito.Mockito.doReturn; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.KeySharedPolicy; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerAccessMode; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Range; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.BinaryProtoLookupService; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.SocatContainer; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class ProxyStuckConnectionTest extends MockedPulsarServiceBaseTest { + + private static final Logger log = LoggerFactory.getLogger(ProxyStuckConnectionTest.class); + + private ProxyService proxyService; + private ProxyConfiguration proxyConfig; + private SocatContainer socatContainer; + + private String brokerServiceUriSocat; + private volatile boolean useBrokerSocatProxy = true; + + @Override + @BeforeMethod + protected void setup() throws Exception { + useBrokerSocatProxy = true; + internalSetup(); + + int brokerPort = pulsar.getBrokerService().getListenPort().get(); + Testcontainers.exposeHostPorts(brokerPort); + + socatContainer = new SocatContainer(); + socatContainer.withTarget(brokerPort, "host.testcontainers.internal", brokerPort); + socatContainer.start(); + brokerServiceUriSocat = "pulsar://" + socatContainer.getHost() + ":" + socatContainer.getMappedPort(brokerPort); + + proxyConfig = new ProxyConfiguration(); + proxyConfig.setServicePort(Optional.ofNullable(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + + startProxyService(); + // use the same port for subsequent restarts + proxyConfig.setServicePort(proxyService.getListenPort()); + } + + private void startProxyService() throws Exception { + proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig))) { + @Override + protected LookupProxyHandler newLookupProxyHandler(ProxyConnection proxyConnection) { + return new TestLookupProxyHandler(this, proxyConnection); + } + }); + doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); + doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + proxyService.start(); + } + + @Override + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + internalCleanup(); + if (proxyService != null) { + proxyService.close(); + } + if (socatContainer != null) { + socatContainer.close(); + } + } + + private final class TestLookupProxyHandler extends LookupProxyHandler { + public TestLookupProxyHandler(ProxyService proxy, ProxyConnection proxyConnection) { + super(proxy, proxyConnection); + } + + @Override + protected String resolveBrokerUrlFromLookupDataResult(BinaryProtoLookupService.LookupDataResult r) { + return useBrokerSocatProxy ? brokerServiceUriSocat : super.resolveBrokerUrlFromLookupDataResult(r); + } + } + + @Test + public void testKeySharedStickyWithStuckConnection() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()) + // keep alive is set to 2 seconds to detect the dead connection on the client side + // the main focus of the test is to verify that the broker and proxy doesn't get stuck forever + // when there's a hanging connection from the proxy to the broker and that it doesn't cause issues + // such as hash range conflicts + .keepAliveInterval(2, TimeUnit.SECONDS) + .build(); + String topicName = BrokerTestUtil.newUniqueName("persistent://sample/test/local/test-topic"); + + @Cleanup + Consumer consumer = client.newConsumer() + .topic(topicName) + .subscriptionName("test-subscription") + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.stickyHashRange() + .ranges(Range.of(0, 65535))) + .receiverQueueSize(2) + .isAckReceiptEnabled(true) + .subscribe(); + + Set messages = new HashSet<>(); + + try (Producer producer = client.newProducer() + .topic(topicName) + .accessMode(ProducerAccessMode.Shared) + .enableBatching(false) + .create()) { + for (int i = 0; i < 10; i++) { + String message = "test" + i; + producer.newMessage().value(message.getBytes()) + .key("A") + .send(); + messages.add(message); + } + } + + int counter = 0; + while (true) { + counter++; + Message msg = consumer.receive(15, TimeUnit.SECONDS); + if (msg == null) { + break; + } + String msgString = new String(msg.getData()); + log.info("Received message {}", msgString); + try { + consumer.acknowledge(msg); + } catch (PulsarClientException e) { + log.error("Failed to ack message {}", msgString, e); + } + messages.remove(msgString); + log.info("Remaining messages {}", messages.size()); + if (messages.size() == 0) { + break; + } + if (counter == 2) { + log.info( + "Pausing connection between proxy and broker and making further connections from proxy " + + "directly to broker"); + useBrokerSocatProxy = false; + socatContainer.getDockerClient().pauseContainerCmd(socatContainer.getContainerId()).exec(); + } + } + + Assert.assertEquals(messages.size(), 0); + Assert.assertEquals(consumer.receive(1, TimeUnit.MILLISECONDS), null); + } +} From 11751b7da2316516a2c18c11b3bd4011641b93f4 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 10 Apr 2023 17:44:41 -0500 Subject: [PATCH 082/280] [feat] PIP-257: Add AuthenticationProviderOpenID (#19849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PIP: #19771 ### Motivation This is the primary PR for PIP 257 (#19771). It adds an OpenID Conenct `AuthenticationProvider` implementation. The implementation is intended to be compliant with the OpenID Specs defined here: https://openid.net/developers/specs/. We specifically implement the discovery and these two: > * [OpenID Connect Core](https://openid.net/specs/openid-connect-core-1_0.html) – Defines the core OpenID Connect functionality: authentication built on top of OAuth 2.0 and the use of claims to communicate information about the End-User > * [OpenID Connect Discovery](https://openid.net/specs/openid-connect-discovery-1_0.html) – Defines how clients dynamically discover information about OpenID Providers ### Modifications * Add new module `pulsar-broker-auth-oidc` * Add implementation that relies on auth0 client libraries to verify the signature and claims of the JWT * Use async http client for all http requests * Cache the provider metadata and the JWKS results * Support different types of `FallbackDiscoveryMode`s, as documented in the code. Essentially, this setting allows users to more easily integrate with k8s. We need this coupling with kubernetes to deal with some of the nuances of the k8s implementation. Note that this part of the code is experimental and is subject to change as requirements and cloud provider implementations change. One important reason we use the k8s client is because the API Server requires special configuration for authentication and TLS. Since these do not appear to be generic requirements, the K8s client simplifies this integration. Here is a reference to the decision that requires authentication by default for getting OIDC info https://github.com/kubernetes/kubernetes/pull/80724. That discussion also indicate to me that this is an isolated design decision in k8s. If we find that authentication is a generic requirement, we should easily be able to expand the existing feature at a later time. * Add metrics to help quantify success and failure. (I had thought I would add audit logging, but that is an independent feature that we can add to the Pulsar framework. It seems outside the scope of an Authentication Provider implementation to implement this feature.) ### Verifying this change There are many new tests to cover this new implementation. Some of the tests are unit tests while others are integration tests that rely on Wire Mock to return the public key information. ### Documentation - [x] `doc-required` This feature will need new docs. ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/35 --- distribution/server/pom.xml | 6 + .../server/src/assemble/LICENSE.bin.txt | 3 + pom.xml | 2 + pulsar-broker-auth-oidc/pom.xml | 179 ++++++ .../oidc/AuthenticationExceptionCode.java | 38 ++ .../oidc/AuthenticationProviderOpenID.java | 493 ++++++++++++++++ .../oidc/AuthenticationStateOpenID.java | 96 ++++ .../authentication/oidc/ConfigUtils.java | 143 +++++ .../oidc/FallbackDiscoveryMode.java | 61 ++ .../broker/authentication/oidc/JwksCache.java | 202 +++++++ .../oidc/OpenIDProviderMetadata.java | 53 ++ .../oidc/OpenIDProviderMetadataCache.java | 232 ++++++++ .../authentication/oidc/package-info.java | 19 + ...ticationProviderOpenIDIntegrationTest.java | 530 ++++++++++++++++++ .../AuthenticationProviderOpenIDTest.java | 387 +++++++++++++ .../oidc/AuthenticationStateOpenIDTest.java | 59 ++ .../authentication/oidc/ConfigUtilsTest.java | 151 +++++ .../test/java/resources/fakeKubeConfig.yaml | 37 ++ 18 files changed, 2691 insertions(+) create mode 100644 pulsar-broker-auth-oidc/pom.xml create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java create mode 100644 pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 2043da516cf40..6b225bfc00c0b 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -46,6 +46,12 @@ ${project.version} + + ${project.groupId} + pulsar-broker-auth-oidc + ${project.version} + + ${project.groupId} pulsar-broker-auth-sasl diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 9205e072f7efe..4cb1d1544a63e 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -519,6 +519,9 @@ MIT License - org.checkerframework-checker-qual-3.12.0.jar * oshi - com.github.oshi-oshi-core-java11-6.4.0.jar + * Auth0, Inc. + - com.auth0-java-jwt-4.3.0.jar + - com.auth0-jwks-rsa-0.22.0.jar Protocol Buffers License * Protocol Buffers - com.google.protobuf-protobuf-java-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt diff --git a/pom.xml b/pom.xml index d6d7fdf44689d..389d9f6f83487 100644 --- a/pom.xml +++ b/pom.xml @@ -2159,6 +2159,7 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-athenz pulsar-client-auth-athenz pulsar-sql + pulsar-broker-auth-oidc pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation @@ -2217,6 +2218,7 @@ flexible messaging model and an intuitive client API. pulsar-websocket pulsar-proxy pulsar-testclient + pulsar-broker-auth-oidc pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml new file mode 100644 index 0000000000000..9ad0363775a13 --- /dev/null +++ b/pulsar-broker-auth-oidc/pom.xml @@ -0,0 +1,179 @@ + + + + 4.0.0 + + org.apache.pulsar + pulsar + 3.0.0-SNAPSHOT + + + pulsar-broker-auth-oidc + jar + Open ID Connect authentication plugin for broker + + + 0.11.5 + + + + + + ${project.groupId} + pulsar-broker-common + ${project.version} + + + io.grpc + * + + + + + + com.auth0 + java-jwt + 4.3.0 + + + + com.auth0 + jwks-rsa + 0.22.0 + + + + com.github.ben-manes.caffeine + caffeine + + + + org.asynchttpclient + async-http-client + + + + io.kubernetes + client-java + ${kubernetesclient.version} + + + + io.prometheus + simpleclient_httpserver + + + + + + io.jsonwebtoken + jjwt-api + ${jsonwebtoken.version} + test + + + io.jsonwebtoken + jjwt-impl + ${jsonwebtoken.version} + test + + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} + pulsar-broker + ${project.version} + test + test-jar + + + + + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + src/test/java/resources/fakeKubeConfig.yaml + ${project.basedir}/target/kubeconfig.yaml + + + + + + diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java new file mode 100644 index 0000000000000..5f89f5f1370f1 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +/** + * Enum used to classify the types of exceptions encountered + * when attempting JWT verification. + */ +public enum AuthenticationExceptionCode { + UNSUPPORTED_ISSUER, + UNSUPPORTED_ALGORITHM, + ISSUER_MISMATCH, + ALGORITHM_MISMATCH, + INVALID_PUBLIC_KEY, + ERROR_RETRIEVING_PROVIDER_METADATA, + ERROR_RETRIEVING_PUBLIC_KEY, + ERROR_DECODING_JWT, + ERROR_VERIFYING_JWT, + ERROR_VERIFYING_JWT_SIGNATURE, + INVALID_JWT_CLAIM, + EXPIRED_JWT, +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java new file mode 100644 index 0000000000000..ae4774b6f6b6b --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java @@ -0,0 +1,493 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsBoolean; +import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt; +import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsSet; +import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsString; +import com.auth0.jwk.InvalidPublicKeyException; +import com.auth0.jwk.Jwk; +import com.auth0.jwt.JWT; +import com.auth0.jwt.JWTVerifier; +import com.auth0.jwt.RegisteredClaims; +import com.auth0.jwt.algorithms.Algorithm; +import com.auth0.jwt.exceptions.AlgorithmMismatchException; +import com.auth0.jwt.exceptions.InvalidClaimException; +import com.auth0.jwt.exceptions.JWTDecodeException; +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.exceptions.SignatureVerificationException; +import com.auth0.jwt.exceptions.TokenExpiredException; +import com.auth0.jwt.interfaces.Claim; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.jwt.interfaces.Verification; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.util.Config; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import java.io.File; +import java.io.IOException; +import java.net.SocketAddress; +import java.security.PublicKey; +import java.security.interfaces.ECPublicKey; +import java.security.interfaces.RSAPublicKey; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import javax.naming.AuthenticationException; +import javax.net.ssl.SSLSession; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; +import org.apache.pulsar.common.api.AuthData; +import org.asynchttpclient.AsyncHttpClient; +import org.asynchttpclient.AsyncHttpClientConfig; +import org.asynchttpclient.DefaultAsyncHttpClient; +import org.asynchttpclient.DefaultAsyncHttpClientConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An {@link AuthenticationProvider} implementation that supports the usage of a JSON Web Token (JWT) + * for client authentication. This implementation retrieves the PublicKey from the JWT issuer (assuming the + * issuer is in the configured allowed list) and then uses that Public Key to verify the validity of the JWT's + * signature. + * + * The Public Keys for a given provider are cached based on certain configured parameters to improve performance. + * The tradeoff here is that the longer Public Keys are cached, the longer an invalidated token could be used. One way + * to ensure caches are cleared is to restart all brokers. + * + * Class is called from multiple threads. The implementation must be thread safe. This class expects to be loaded once + * and then called concurrently for each new connection. The cache is backed by a GuavaCachedJwkProvider, which is + * thread-safe. + * + * Supported algorithms are: RS256, RS384, RS512, ES256, ES384, ES512 where the naming conventions follow + * this RFC: https://datatracker.ietf.org/doc/html/rfc7518#section-3.1. + */ +public class AuthenticationProviderOpenID implements AuthenticationProvider { + private static final Logger log = LoggerFactory.getLogger(AuthenticationProviderOpenID.class); + + private static final String SIMPLE_NAME = AuthenticationProviderOpenID.class.getSimpleName(); + + // Must match the value used by the OAuth2 Client Plugin. + private static final String AUTH_METHOD_NAME = "token"; + + // This is backed by an ObjectMapper, which is thread safe. It is an optimization + // to share this for decoding JWTs for all connections to this broker. + private final JWT jwtLibrary = new JWT(); + + private Set issuers; + + // This caches the map from Issuer URL to the jwks_uri served at the /.well-known/openid-configuration endpoint + private OpenIDProviderMetadataCache openIDProviderMetadataCache; + + // A cache used to store the results of getting the JWKS from the jwks_uri for an issuer. + private JwksCache jwksCache; + + private volatile AsyncHttpClient httpClient; + + // A list of supported algorithms. This is the "alg" field on the JWT. + // Source for strings: https://datatracker.ietf.org/doc/html/rfc7518#section-3.1. + private static final String ALG_RS256 = "RS256"; + private static final String ALG_RS384 = "RS384"; + private static final String ALG_RS512 = "RS512"; + private static final String ALG_ES256 = "ES256"; + private static final String ALG_ES384 = "ES384"; + private static final String ALG_ES512 = "ES512"; + + private long acceptedTimeLeewaySeconds; + private FallbackDiscoveryMode fallbackDiscoveryMode; + private String roleClaim = ROLE_CLAIM_DEFAULT; + private boolean isRoleClaimNotSubject; + + static final String ALLOWED_TOKEN_ISSUERS = "openIDAllowedTokenIssuers"; + static final String ISSUER_TRUST_CERTS_FILE_PATH = "openIDTokenIssuerTrustCertsFilePath"; + static final String FALLBACK_DISCOVERY_MODE = "openIDFallbackDiscoveryMode"; + static final String ALLOWED_AUDIENCES = "openIDAllowedAudiences"; + static final String ROLE_CLAIM = "openIDRoleClaim"; + static final String ROLE_CLAIM_DEFAULT = "sub"; + static final String ACCEPTED_TIME_LEEWAY_SECONDS = "openIDAcceptedTimeLeewaySeconds"; + static final int ACCEPTED_TIME_LEEWAY_SECONDS_DEFAULT = 0; + static final String CACHE_SIZE = "openIDCacheSize"; + static final int CACHE_SIZE_DEFAULT = 5; + static final String CACHE_REFRESH_AFTER_WRITE_SECONDS = "openIDCacheRefreshAfterWriteSeconds"; + static final int CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT = 18 * 60 * 60; + static final String CACHE_EXPIRATION_SECONDS = "openIDCacheExpirationSeconds"; + static final int CACHE_EXPIRATION_SECONDS_DEFAULT = 24 * 60 * 60; + static final String HTTP_CONNECTION_TIMEOUT_MILLIS = "openIDHttpConnectionTimeoutMillis"; + static final int HTTP_CONNECTION_TIMEOUT_MILLIS_DEFAULT = 10_000; + static final String HTTP_READ_TIMEOUT_MILLIS = "openIDHttpReadTimeoutMillis"; + static final int HTTP_READ_TIMEOUT_MILLIS_DEFAULT = 10_000; + static final String REQUIRE_HTTPS = "openIDRequireIssuersUseHttps"; + static final boolean REQUIRE_HTTPS_DEFAULT = true; + + // The list of audiences that are allowed to connect to this broker. A valid JWT must contain one of the audiences. + private String[] allowedAudiences; + + @Override + public void initialize(ServiceConfiguration config) throws IOException { + this.allowedAudiences = validateAllowedAudiences(getConfigValueAsSet(config, ALLOWED_AUDIENCES)); + this.roleClaim = getConfigValueAsString(config, ROLE_CLAIM, ROLE_CLAIM_DEFAULT); + this.isRoleClaimNotSubject = !ROLE_CLAIM_DEFAULT.equals(roleClaim); + this.acceptedTimeLeewaySeconds = getConfigValueAsInt(config, ACCEPTED_TIME_LEEWAY_SECONDS, + ACCEPTED_TIME_LEEWAY_SECONDS_DEFAULT); + boolean requireHttps = getConfigValueAsBoolean(config, REQUIRE_HTTPS, REQUIRE_HTTPS_DEFAULT); + this.fallbackDiscoveryMode = FallbackDiscoveryMode.valueOf(getConfigValueAsString(config, + FALLBACK_DISCOVERY_MODE, FallbackDiscoveryMode.DISABLED.name())); + this.issuers = validateIssuers(getConfigValueAsSet(config, ALLOWED_TOKEN_ISSUERS), requireHttps, + fallbackDiscoveryMode != FallbackDiscoveryMode.DISABLED); + + int connectionTimeout = getConfigValueAsInt(config, HTTP_CONNECTION_TIMEOUT_MILLIS, + HTTP_CONNECTION_TIMEOUT_MILLIS_DEFAULT); + int readTimeout = getConfigValueAsInt(config, HTTP_READ_TIMEOUT_MILLIS, HTTP_READ_TIMEOUT_MILLIS_DEFAULT); + String trustCertsFilePath = getConfigValueAsString(config, ISSUER_TRUST_CERTS_FILE_PATH, null); + SslContext sslContext = null; + if (trustCertsFilePath != null) { + // Use default settings for everything but the trust store. + sslContext = SslContextBuilder.forClient() + .trustManager(new File(trustCertsFilePath)) + .build(); + } + AsyncHttpClientConfig clientConfig = new DefaultAsyncHttpClientConfig.Builder() + .setConnectTimeout(connectionTimeout) + .setReadTimeout(readTimeout) + .setSslContext(sslContext) + .build(); + httpClient = new DefaultAsyncHttpClient(clientConfig); + ApiClient k8sApiClient = + fallbackDiscoveryMode != FallbackDiscoveryMode.DISABLED ? Config.defaultClient() : null; + this.openIDProviderMetadataCache = new OpenIDProviderMetadataCache(config, httpClient, k8sApiClient); + this.jwksCache = new JwksCache(config, httpClient, k8sApiClient); + } + + @Override + public String getAuthMethodName() { + return AUTH_METHOD_NAME; + } + + /** + * Authenticate the parameterized {@link AuthenticationDataSource} by verifying the issuer is an allowed issuer, + * then retrieving the JWKS URI from the issuer, then retrieving the Public key from the JWKS URI, and finally + * verifying the JWT signature and claims. + * + * @param authData - the authData passed by the Pulsar Broker containing the token. + * @return the role, if the JWT is authenticated, otherwise a failed future. + */ + @Override + public CompletableFuture authenticateAsync(AuthenticationDataSource authData) { + return authenticateTokenAsync(authData).thenApply(this::getRole); + } + + /** + * Authenticate the parameterized {@link AuthenticationDataSource} and return the decoded JWT. + * @param authData - the authData containing the token. + * @return a completed future with the decoded JWT, if the JWT is authenticated. Otherwise, a failed future. + */ + CompletableFuture authenticateTokenAsync(AuthenticationDataSource authData) { + String token; + try { + token = AuthenticationProviderToken.getToken(authData); + } catch (AuthenticationException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT); + return CompletableFuture.failedFuture(e); + } + return authenticateToken(token) + .whenComplete((jwt, e) -> { + if (jwt != null) { + AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + } + // Failure metrics are incremented within methods above + }); + } + + /** + * Get the role from a JWT at the configured role claim field. + * NOTE: does not do any verification of the JWT + * @param jwt - token to get the role from + * @return the role, or null, if it is not set on the JWT + */ + String getRole(DecodedJWT jwt) { + try { + Claim roleClaim = jwt.getClaim(this.roleClaim); + if (roleClaim.isNull()) { + // The claim was not present in the JWT + return null; + } + + String role = roleClaim.asString(); + if (role != null) { + // The role is non null only if the JSON node is a text field + return role; + } + + List roles = jwt.getClaim(this.roleClaim).asList(String.class); + if (roles == null || roles.size() == 0) { + return null; + } else if (roles.size() == 1) { + return roles.get(0); + } else { + log.debug("JWT for subject [{}] has multiple roles; using the first one.", jwt.getSubject()); + return roles.get(0); + } + } catch (JWTDecodeException e) { + log.error("Exception while retrieving role from JWT", e); + return null; + } + } + + /** + * Convert a JWT string into a {@link DecodedJWT} + * The benefit of using this method is that it utilizes the already instantiated {@link JWT} parser. + * WARNING: this method does not verify the authenticity of the token. It only decodes it. + * + * @param token - string JWT to be decoded + * @return a decoded JWT + * @throws AuthenticationException if the token string is null or if any part of the token contains + * an invalid jwt or JSON format of each of the jwt parts. + */ + DecodedJWT decodeJWT(String token) throws AuthenticationException { + if (token == null) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT); + throw new AuthenticationException("Invalid token: cannot be null"); + } + try { + return jwtLibrary.decodeJwt(token); + } catch (JWTDecodeException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT); + throw new AuthenticationException("Unable to decode JWT: " + e.getMessage()); + } + } + + /** + * Authenticate the parameterized JWT. + * + * @param token - a nonnull JWT to authenticate + * @return a fully authenticated JWT, or AuthenticationException if the JWT is proven to be invalid in any way + */ + private CompletableFuture authenticateToken(String token) { + if (token == null) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT); + return CompletableFuture.failedFuture(new AuthenticationException("JWT cannot be null")); + } + final DecodedJWT jwt; + try { + jwt = decodeJWT(token); + } catch (AuthenticationException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT); + return CompletableFuture.failedFuture(e); + } + return verifyIssuerAndGetJwk(jwt) + .thenCompose(jwk -> { + try { + if (!jwt.getAlgorithm().equals(jwk.getAlgorithm())) { + incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH); + return CompletableFuture.failedFuture( + new AuthenticationException("JWK's alg [" + jwk.getAlgorithm() + + "] does not match JWT's alg [" + jwt.getAlgorithm() + "]")); + } + // Verify the JWT signature + // Throws exception if any verification check fails + return CompletableFuture + .completedFuture(verifyJWT(jwk.getPublicKey(), jwt.getAlgorithm(), jwt)); + } catch (InvalidPublicKeyException e) { + incrementFailureMetric(AuthenticationExceptionCode.INVALID_PUBLIC_KEY); + return CompletableFuture.failedFuture( + new AuthenticationException("Invalid public key: " + e.getMessage())); + } catch (AuthenticationException e) { + return CompletableFuture.failedFuture(e); + } + }); + } + + /** + * Verify the JWT's issuer (iss) claim is one of the allowed issuers and then retrieve the JWK from the issuer. If + * not, see {@link FallbackDiscoveryMode} for the fallback behavior. + * @param jwt - the token to use to discover the issuer's JWKS URI, which is then used to retrieve the issuer's + * current public keys. + * @return a JWK that can be used to verify the JWT's signature + */ + private CompletableFuture verifyIssuerAndGetJwk(DecodedJWT jwt) { + if (jwt.getIssuer() == null) { + incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER); + return CompletableFuture.failedFuture(new AuthenticationException("Issuer cannot be null")); + } else if (this.issuers.contains(jwt.getIssuer())) { + // Retrieve the metadata: https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata + return openIDProviderMetadataCache.getOpenIDProviderMetadataForIssuer(jwt.getIssuer()) + .thenCompose(metadata -> jwksCache.getJwk(metadata.getJwksUri(), jwt.getKeyId())); + } else if (fallbackDiscoveryMode == FallbackDiscoveryMode.KUBERNETES_DISCOVER_TRUSTED_ISSUER) { + return openIDProviderMetadataCache.getOpenIDProviderMetadataForKubernetesApiServer(jwt.getIssuer()) + .thenCompose(metadata -> + openIDProviderMetadataCache.getOpenIDProviderMetadataForIssuer(metadata.getIssuer())) + .thenCompose(metadata -> jwksCache.getJwk(metadata.getJwksUri(), jwt.getKeyId())); + } else if (fallbackDiscoveryMode == FallbackDiscoveryMode.KUBERNETES_DISCOVER_PUBLIC_KEYS) { + return openIDProviderMetadataCache.getOpenIDProviderMetadataForKubernetesApiServer(jwt.getIssuer()) + .thenCompose(__ -> jwksCache.getJwkFromKubernetesApiServer(jwt.getKeyId())); + } else { + incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER); + return CompletableFuture + .failedFuture(new AuthenticationException("Issuer not allowed: " + jwt.getIssuer())); + } + } + + @Override + public AuthenticationState newAuthState(AuthData authData, SocketAddress remoteAddress, SSLSession sslSession) + throws AuthenticationException { + return new AuthenticationStateOpenID(this, remoteAddress, sslSession); + } + + @Override + public void close() throws IOException { + httpClient.close(); + } + + /** + * Build and return a validator for the parameters. + * + * @param publicKey - the public key to use when configuring the validator + * @param publicKeyAlg - the algorithm for the parameterized public key + * @param jwt - jwt to be verified and returned (only if verified) + * @return a validator to use for validating a JWT associated with the parameterized public key. + * @throws AuthenticationException if the Public Key's algorithm is not supported or if the algorithm param does not + * match the Public Key's actual algorithm. + */ + DecodedJWT verifyJWT(PublicKey publicKey, + String publicKeyAlg, + DecodedJWT jwt) throws AuthenticationException { + if (publicKeyAlg == null) { + incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ALGORITHM); + throw new AuthenticationException("PublicKey algorithm cannot be null"); + } + + Algorithm alg; + try { + switch (publicKeyAlg) { + case ALG_RS256: + alg = Algorithm.RSA256((RSAPublicKey) publicKey, null); + break; + case ALG_RS384: + alg = Algorithm.RSA384((RSAPublicKey) publicKey, null); + break; + case ALG_RS512: + alg = Algorithm.RSA512((RSAPublicKey) publicKey, null); + break; + case ALG_ES256: + alg = Algorithm.ECDSA256((ECPublicKey) publicKey, null); + break; + case ALG_ES384: + alg = Algorithm.ECDSA384((ECPublicKey) publicKey, null); + break; + case ALG_ES512: + alg = Algorithm.ECDSA512((ECPublicKey) publicKey, null); + break; + default: + incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ALGORITHM); + throw new AuthenticationException("Unsupported algorithm: " + publicKeyAlg); + } + } catch (ClassCastException e) { + incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH); + throw new AuthenticationException("Expected PublicKey alg [" + publicKeyAlg + "] does match actual alg."); + } + + // We verify issuer when retrieving the PublicKey, so it is not verified here. + // The claim presence requirements are based on https://openid.net/specs/openid-connect-basic-1_0.html#IDToken + Verification verifierBuilder = JWT.require(alg) + .acceptLeeway(acceptedTimeLeewaySeconds) + .withAnyOfAudience(allowedAudiences) + .withClaimPresence(RegisteredClaims.ISSUED_AT) + .withClaimPresence(RegisteredClaims.EXPIRES_AT) + .withClaimPresence(RegisteredClaims.NOT_BEFORE) + .withClaimPresence(RegisteredClaims.SUBJECT); + + if (isRoleClaimNotSubject) { + verifierBuilder = verifierBuilder.withClaimPresence(roleClaim); + } + + JWTVerifier verifier = verifierBuilder.build(); + + try { + return verifier.verify(jwt); + } catch (TokenExpiredException e) { + incrementFailureMetric(AuthenticationExceptionCode.EXPIRED_JWT); + throw new AuthenticationException("JWT expired: " + e.getMessage()); + } catch (SignatureVerificationException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_VERIFYING_JWT_SIGNATURE); + throw new AuthenticationException("JWT signature verification exception: " + e.getMessage()); + } catch (InvalidClaimException e) { + incrementFailureMetric(AuthenticationExceptionCode.INVALID_JWT_CLAIM); + throw new AuthenticationException("JWT contains invalid claim: " + e.getMessage()); + } catch (AlgorithmMismatchException e) { + incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH); + throw new AuthenticationException("JWT algorithm does not match Public Key algorithm: " + e.getMessage()); + } catch (JWTDecodeException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT); + throw new AuthenticationException("Error while decoding JWT: " + e.getMessage()); + } catch (JWTVerificationException | IllegalArgumentException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_VERIFYING_JWT); + throw new AuthenticationException("JWT verification failed: " + e.getMessage()); + } + } + + static void incrementFailureMetric(AuthenticationExceptionCode code) { + AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, "token", code.toString()); + } + + /** + * Validate the configured allow list of allowedIssuers. The allowedIssuers set must be nonempty in order for + * the plugin to authenticate any token. Thus, it fails initialization if the configuration is + * missing. Each issuer URL should use the HTTPS scheme. The plugin fails initialization if any + * issuer url is insecure, unless requireHttps is false. + * @param allowedIssuers - issuers to validate + * @param requireHttps - whether to require https for issuers. + * @param allowEmptyIssuers - whether to allow empty issuers. This setting only makes sense when kubernetes is used + * as a fallback issuer. + * @return the validated issuers + * @throws IllegalArgumentException if the allowedIssuers is empty, or contains insecure issuers when required + */ + private Set validateIssuers(Set allowedIssuers, boolean requireHttps, boolean allowEmptyIssuers) { + if (allowedIssuers == null || (allowedIssuers.isEmpty() && !allowEmptyIssuers)) { + throw new IllegalArgumentException("Missing configured value for: " + ALLOWED_TOKEN_ISSUERS); + } + for (String issuer : allowedIssuers) { + if (!issuer.toLowerCase().startsWith("https://")) { + log.warn("Allowed issuer is not using https scheme: {}", issuer); + if (requireHttps) { + throw new IllegalArgumentException("Issuer URL does not use https, but must: " + issuer); + } + } + } + return allowedIssuers; + } + + /** + * Validate the configured allow list of allowedAudiences. The allowedAudiences must be set because + * JWT must have an audience claim. + * See https://openid.net/specs/openid-connect-basic-1_0.html#IDTokenValidation. + * @param allowedAudiences + * @return the validated audiences + */ + String[] validateAllowedAudiences(Set allowedAudiences) { + if (allowedAudiences == null || allowedAudiences.isEmpty()) { + throw new IllegalArgumentException("Missing configured value for: " + ALLOWED_AUDIENCES); + } + return allowedAudiences.toArray(new String[0]); + } +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java new file mode 100644 index 0000000000000..3046a6dd0e3b4 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static java.nio.charset.StandardCharsets.UTF_8; +import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; +import javax.naming.AuthenticationException; +import javax.net.ssl.SSLSession; +import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.common.api.AuthData; + +/** + * Class representing the authentication state of a single connection. + */ +class AuthenticationStateOpenID implements AuthenticationState { + private final AuthenticationProviderOpenID provider; + private AuthenticationDataSource authenticationDataSource; + private volatile String role; + private final SocketAddress remoteAddress; + private final SSLSession sslSession; + private volatile long expiration; + + AuthenticationStateOpenID( + AuthenticationProviderOpenID provider, + SocketAddress remoteAddress, + SSLSession sslSession) { + this.provider = provider; + this.remoteAddress = remoteAddress; + this.sslSession = sslSession; + } + + @Override + public String getAuthRole() throws AuthenticationException { + if (role == null) { + throw new AuthenticationException("Authentication has not completed"); + } + return role; + } + + @Deprecated + @Override + public AuthData authenticate(AuthData authData) throws AuthenticationException { + // This method is not expected to be called and is subject to removal. + throw new AuthenticationException("Not supported"); + } + + @Override + public CompletableFuture authenticateAsync(AuthData authData) { + final String token = new String(authData.getBytes(), UTF_8); + this.authenticationDataSource = new AuthenticationDataCommand(token, remoteAddress, sslSession); + return provider + .authenticateTokenAsync(authenticationDataSource) + .thenApply(jwt -> { + this.role = provider.getRole(jwt); + // OIDC requires setting the exp claim, so this should never be null. + // We verify it is not null during token validation. + this.expiration = jwt.getExpiresAt().getTime(); + // Single stage authentication, so return null here + return null; + }); + } + + @Override + public AuthenticationDataSource getAuthDataSource() { + return authenticationDataSource; + } + + @Override + public boolean isComplete() { + return role != null; + } + + @Override + public boolean isExpired() { + return System.currentTimeMillis() > expiration; + } +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java new file mode 100644 index 0000000000000..f62bf9c818653 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ConfigUtils { + private static final Logger log = LoggerFactory.getLogger(ConfigUtils.class); + + /** + * Get configured property as a string. If not configured, return null. + * @param conf - the configuration map + * @param configProp - the property to get + * @return a string from the conf or null, if the configuration property was not set + */ + static String getConfigValueAsString(ServiceConfiguration conf, + String configProp) throws IllegalArgumentException { + String value = getConfigValueAsStringImpl(conf, configProp); + log.info("Configuration for [{}] is [{}]", configProp, value); + return value; + } + + /** + * Get configured property as a string. If not configured, return null. + * @param conf - the configuration map + * @param configProp - the property to get + * @param defaultValue - the value to use if the configuration value is not set + * @return a string from the conf or the default value + */ + static String getConfigValueAsString(ServiceConfiguration conf, String configProp, + String defaultValue) throws IllegalArgumentException { + String value = getConfigValueAsStringImpl(conf, configProp); + if (value == null) { + value = defaultValue; + } + log.info("Configuration for [{}] is [{}]", configProp, value); + return value; + } + + /** + * Get configured property as a set. Split using a comma delimiter and remove any extra whitespace surrounding + * the commas. If not configured, return the empty set. + * + * @param conf - the map of configuration properties + * @param configProp - the property (key) to get + * @return a set of strings from the conf + */ + static Set getConfigValueAsSet(ServiceConfiguration conf, String configProp) { + String value = getConfigValueAsStringImpl(conf, configProp); + if (StringUtils.isBlank(value)) { + log.info("Configuration for [{}] is the empty set.", configProp); + return Collections.emptySet(); + } + Set set = Arrays.stream(value.trim().split("\\s*,\\s*")).collect(Collectors.toSet()); + log.info("Configuration for [{}] is [{}].", configProp, String.join(", ", set)); + return set; + } + + private static String getConfigValueAsStringImpl(ServiceConfiguration conf, + String configProp) throws IllegalArgumentException { + Object value = conf.getProperty(configProp); + if (value instanceof String) { + return (String) value; + } else { + return null; + } + } + + /** + * Utility method to get an integer from the {@link ServiceConfiguration}. If the value is not a valid long or the + * key is not present in the conf, the default value will be used. + * + * @param conf - the map of configuration properties + * @param configProp - the property (key) to get + * @param defaultValue - the value to use if the property is missing from the conf + * @return a long + */ + static int getConfigValueAsInt(ServiceConfiguration conf, String configProp, int defaultValue) { + Object value = conf.getProperty(configProp); + if (value instanceof Integer) { + log.info("Configuration for [{}] is [{}]", configProp, value); + return (Integer) value; + } else if (value instanceof String) { + try { + return Integer.parseInt((String) value); + } catch (NumberFormatException numberFormatException) { + log.error("Expected configuration for [{}] to be a long, but got [{}]. Using default value: [{}]", + configProp, value, defaultValue, numberFormatException); + return defaultValue; + } + } else { + log.info("Configuration for [{}] is using the default value: [{}]", configProp, defaultValue); + return defaultValue; + } + } + + /** + * Utility method to get a boolean from the {@link ServiceConfiguration}. If the key is present in the conf, + * return the default value. If key is present the value is not a valid boolean, the result will be false. + * + * @param conf - the map of configuration properties + * @param configProp - the property (key) to get + * @param defaultValue - the value to use if the property is missing from the conf + * @return a boolean + */ + static boolean getConfigValueAsBoolean(ServiceConfiguration conf, String configProp, boolean defaultValue) { + Object value = conf.getProperty(configProp); + if (value instanceof Boolean) { + log.info("Configuration for [{}] is [{}]", configProp, value); + return (boolean) value; + } else if (value instanceof String) { + boolean result = Boolean.parseBoolean((String) value); + log.info("Configuration for [{}] is [{}]", configProp, result); + return result; + } else { + log.info("Configuration for [{}] is using the default value: [{}]", configProp, defaultValue); + return defaultValue; + } + } +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java new file mode 100644 index 0000000000000..5bf0c1b23fce6 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import org.apache.pulsar.common.classification.InterfaceStability; + +/** + * These are the modes available for configuring how the Open ID Connect Authentication Provider should handle a JWT + * that has an issuer that is not explicitly in the allowed issuers set configured by + * {@link AuthenticationProviderOpenID#ALLOWED_TOKEN_ISSUERS}. The current implementations rely on using the Kubernetes + * Api Server's Open ID Connect features to discover an additional issuer or additional public keys to trust. See the + * Kubernetes documentation for more information on how Service Accounts can integrate with Open ID Connect. + * https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-issuer-discovery + */ +@InterfaceStability.Evolving +public enum FallbackDiscoveryMode { + /** + * There will be no discovery of additional trusted issuers or public keys. This setting requires that operators + * explicitly allow all issuers that will be trusted. For the Kubernetes Service Account Token Projections to work, + * the operator must explicitly trust the issuer on the token's "iss" claim. This is the default setting because it + * is the only mode that explicitly follows the OIDC spec for verification of discovered provider configuration. + */ + DISABLED, + + /** + * The Kubernetes Api Server will be used to discover an additional trusted issuer by getting the issuer at the + * Api Server's /.well-known/openid-configuration endpoint, verifying that issuer matches the "iss" claim on the + * supplied token, then treating that issuer as a trusted issuer by discovering the jwks_uri via that issuer's + * /.well-known/openid-configuration endpoint. This mode can be helpful in EKS environments where the Api Server's + * public keys served at the /openid/v1/jwks endpoint are not the same as the public keys served at the issuer's + * jwks_uri. It fails to be OIDC compliant because the URL used to discover the provider configuration is not the + * same as the issuer claim on the token. + */ + KUBERNETES_DISCOVER_TRUSTED_ISSUER, + + /** + * The Kubernetes Api Server will be used to discover an additional set of valid public keys by getting the issuer + * at the Api Server's /.well-known/openid-configuration endpoint, verifying that issuer matches the "iss" claim on + * the supplied token, then calling the Api Server endpoint to get the public keys using a kubernetes client. This + * mode is currently useful getting the public keys from the Api Server because the Api Server requires custom TLS + * and authentication, and the kubernetes client automatically handles those. It fails to be OIDC compliant because + * the URL used to discover the provider configuration is not the same as the issuer claim on the token. + */ + KUBERNETES_DISCOVER_PUBLIC_KEYS, +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java new file mode 100644 index 0000000000000..12ea7ec6b906d --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric; +import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt; +import com.auth0.jwk.Jwk; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import io.kubernetes.client.openapi.ApiCallback; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.OpenidApi; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import javax.naming.AuthenticationException; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.asynchttpclient.AsyncHttpClient; + +public class JwksCache { + + // Map from an issuer's JWKS URI to its JWKS. When the Issuer is not empty, use the fallback client. + private final AsyncLoadingCache, List> cache; + + private final ObjectReader reader = new ObjectMapper().readerFor(HashMap.class); + private final AsyncHttpClient httpClient; + private final OpenidApi openidApi; + + JwksCache(ServiceConfiguration config, AsyncHttpClient httpClient, ApiClient apiClient) throws IOException { + // Store the clients + this.httpClient = httpClient; + this.openidApi = apiClient != null ? new OpenidApi(apiClient) : null; + + // Configure the cache + int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT); + int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS, + CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT); + int expireAfterSeconds = getConfigValueAsInt(config, CACHE_EXPIRATION_SECONDS, + CACHE_EXPIRATION_SECONDS_DEFAULT); + AsyncCacheLoader, List> loader = (jwksUri, executor) -> { + if (jwksUri.isPresent()) { + return getJwksFromJwksUri(jwksUri.get()); + } else { + return getJwksFromKubernetesApiServer(); + } + }; + this.cache = Caffeine.newBuilder() + .maximumSize(maxSize) + .refreshAfterWrite(refreshAfterWriteSeconds, TimeUnit.SECONDS) + .expireAfterWrite(expireAfterSeconds, TimeUnit.SECONDS) + .buildAsync(loader); + } + + CompletableFuture getJwk(String jwksUri, String keyId) { + if (jwksUri == null) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + return CompletableFuture.failedFuture(new IllegalArgumentException("jwksUri must not be null.")); + } + return cache.get(Optional.of(jwksUri)).thenApply(jwks -> getJwkForKID(jwks, keyId)); + } + + private CompletableFuture> getJwksFromJwksUri(String jwksUri) { + return httpClient + .prepareGet(jwksUri) + .execute() + .toCompletableFuture() + .thenCompose(result -> { + CompletableFuture> future = new CompletableFuture<>(); + try { + HashMap jwks = + reader.readValue(result.getResponseBodyAsBytes()); + future.complete(convertToJwks(jwksUri, jwks)); + } catch (AuthenticationException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + future.completeExceptionally(e); + } catch (Exception e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + future.completeExceptionally(new AuthenticationException( + "Error retrieving public key at " + jwksUri + ": " + e.getMessage())); + } + return future; + }); + } + + CompletableFuture getJwkFromKubernetesApiServer(String keyId) { + if (openidApi == null) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + return CompletableFuture.failedFuture(new AuthenticationException( + "Failed to retrieve public key from Kubernetes API server: Kubernetes fallback is not enabled.")); + } + return cache.get(Optional.empty(), (__, executor) -> getJwksFromKubernetesApiServer()) + .thenApply(jwks -> getJwkForKID(jwks, keyId)); + } + + private CompletableFuture> getJwksFromKubernetesApiServer() { + CompletableFuture> future = new CompletableFuture<>(); + try { + openidApi.getServiceAccountIssuerOpenIDKeysetAsync(new ApiCallback() { + @Override + public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + future.completeExceptionally( + new AuthenticationException("Failed to retrieve public key from Kubernetes API server: " + + e.getMessage())); + } + + @Override + public void onSuccess(String result, int statusCode, Map> responseHeaders) { + try { + HashMap jwks = reader.readValue(result); + future.complete(convertToJwks("Kubernetes API server", jwks)); + } catch (AuthenticationException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + future.completeExceptionally(e); + } catch (Exception e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + future.completeExceptionally(new AuthenticationException( + "Error retrieving public key at Kubernetes API server: " + e.getMessage())); + } + } + + @Override + public void onUploadProgress(long bytesWritten, long contentLength, boolean done) { + + } + + @Override + public void onDownloadProgress(long bytesRead, long contentLength, boolean done) { + + } + }); + } catch (ApiException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + future.completeExceptionally( + new AuthenticationException("Failed to retrieve public key from Kubernetes API server: " + + e.getMessage())); + } + return future; + } + + private Jwk getJwkForKID(List jwks, String keyId) { + for (Jwk jwk : jwks) { + if (jwk.getId().equals(keyId)) { + return jwk; + } + } + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + throw new IllegalArgumentException("No JWK found for Key ID " + keyId); + } + + /** + * The JWK Set is stored in the "keys" key see https://www.rfc-editor.org/rfc/rfc7517#section-5.1. + * + * @param jwksUri - the URI used to retrieve the JWKS + * @param jwks - the JWKS to convert + * @return a list of {@link Jwk} + */ + private List convertToJwks(String jwksUri, Map jwks) throws AuthenticationException { + try { + @SuppressWarnings("unchecked") + List> jwkList = (List>) jwks.get("keys"); + final List result = new ArrayList<>(); + for (Map jwk : jwkList) { + result.add(Jwk.fromValues(jwk)); + } + return result; + } catch (ClassCastException e) { + throw new AuthenticationException("Malformed JWKS returned by: " + jwksUri); + } + } +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java new file mode 100644 index 0000000000000..553b3b882dbee --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * A Simple Class representing the essential fields of the OpenID Provider Metadata. + * Spec: https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata + * Note that this class is only used for deserializing the JSON metadata response from + * calling a provider's /.well-known/openid-configuration endpoint. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class OpenIDProviderMetadata { + + private final String issuer; + private final String jwksUri; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public OpenIDProviderMetadata(@JsonProperty("issuer") String issuer, @JsonProperty("jwks_uri") String jwksUri) { + this.issuer = issuer; + this.jwksUri = jwksUri; + } + + @JsonGetter + public String getIssuer() { + return issuer; + } + + @JsonGetter + public String getJwksUri() { + return jwksUri; + } +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java new file mode 100644 index 0000000000000..33d11f35a349a --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric; +import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import io.kubernetes.client.openapi.ApiCallback; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.WellKnownApi; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; +import javax.naming.AuthenticationException; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.asynchttpclient.AsyncHttpClient; + +/** + * Class used to cache metadata responses from OpenID Providers. + */ +class OpenIDProviderMetadataCache { + + private final ObjectReader reader = new ObjectMapper().readerFor(OpenIDProviderMetadata.class); + private final AsyncHttpClient httpClient; + private final WellKnownApi wellKnownApi; + private final AsyncLoadingCache, OpenIDProviderMetadata> cache; + private static final String WELL_KNOWN_OPENID_CONFIG = ".well-known/openid-configuration"; + private static final String SLASH_WELL_KNOWN_OPENID_CONFIG = "/" + WELL_KNOWN_OPENID_CONFIG; + + OpenIDProviderMetadataCache(ServiceConfiguration config, AsyncHttpClient httpClient, ApiClient apiClient) { + int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT); + int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS, + CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT); + int expireAfterSeconds = getConfigValueAsInt(config, CACHE_EXPIRATION_SECONDS, + CACHE_EXPIRATION_SECONDS_DEFAULT); + this.httpClient = httpClient; + this.wellKnownApi = apiClient != null ? new WellKnownApi(apiClient) : null; + AsyncCacheLoader, OpenIDProviderMetadata> loader = (issuer, executor) -> { + if (issuer.isPresent()) { + return loadOpenIDProviderMetadataForIssuer(issuer.get()); + } else { + return loadOpenIDProviderMetadataForKubernetesApiServer(); + } + }; + this.cache = Caffeine.newBuilder() + .maximumSize(maxSize) + .refreshAfterWrite(refreshAfterWriteSeconds, TimeUnit.SECONDS) + .expireAfterWrite(expireAfterSeconds, TimeUnit.SECONDS) + .buildAsync(loader); + } + + /** + * Retrieve the OpenID Provider Metadata for the provided issuer. + *

+ * Note: this method does not do any validation on the parameterized issuer. The OpenID Connect discovery + * spec requires that the issuer use the HTTPS scheme: https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata. + * The {@link AuthenticationProviderOpenID} class handles this verification. + * + * @param issuer - authority from which to retrieve the OpenID Provider Metadata + * @return the {@link OpenIDProviderMetadata} for the given issuer. Fail the completable future with + * AuthenticationException if any exceptions occur while retrieving the metadata. + */ + CompletableFuture getOpenIDProviderMetadataForIssuer(@Nonnull String issuer) { + return cache.get(Optional.of(issuer)); + } + + /** + * A loader for the cache that retrieves the metadata from the issuer's /.well-known/openid-configuration endpoint. + * @return a connection to the issuer's /.well-known/openid-configuration endpoint. Fails with + * AuthenticationException if the URL is malformed or there is an exception while opening the connection + */ + private CompletableFuture loadOpenIDProviderMetadataForIssuer(String issuer) { + String url; + // TODO URI's normalization likely follows RFC2396 (library doesn't say so explicitly), whereas the spec + // https://openid.net/specs/openid-connect-discovery-1_0.html#NormalizationSteps + // calls for normalization according to RFC3986, which is supposed to obsolete RFC2396. Is this a problem? + if (issuer.endsWith("/")) { + url = issuer + WELL_KNOWN_OPENID_CONFIG; + } else { + url = issuer + SLASH_WELL_KNOWN_OPENID_CONFIG; + } + + return httpClient + .prepareGet(url) + .execute() + .toCompletableFuture() + .thenCompose(result -> { + CompletableFuture future = new CompletableFuture<>(); + try { + OpenIDProviderMetadata openIDProviderMetadata = + reader.readValue(result.getResponseBodyAsBytes()); + // We can verify this issuer once and cache the result because the issuer uniquely maps + // to the cached object. + verifyIssuer(issuer, openIDProviderMetadata, false); + future.complete(openIDProviderMetadata); + } catch (AuthenticationException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + future.completeExceptionally(e); + } catch (Exception e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + future.completeExceptionally(new AuthenticationException( + "Error retrieving OpenID Provider Metadata at " + issuer + ": " + e.getMessage())); + } + return future; + }); + } + + /** + * Retrieve the OpenID Provider Metadata for the Kubernetes API server. This method is used instead of + * {@link #getOpenIDProviderMetadataForIssuer(String)} because different validations are done. The Kubernetes + * API server does not technically implement the complete OIDC spec for discovery, but it does implement some of + * it, so this method validates what it can. Specifically, it skips validation that the Discovery Document + * provider's URI matches the issuer. It verifies that the issuer on the discovery document matches the issuer + * claim + * @return + */ + CompletableFuture getOpenIDProviderMetadataForKubernetesApiServer(String issClaim) { + return cache.get(Optional.empty()).thenCompose(openIDProviderMetadata -> { + CompletableFuture future = new CompletableFuture<>(); + try { + verifyIssuer(issClaim, openIDProviderMetadata, true); + future.complete(openIDProviderMetadata); + } catch (AuthenticationException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + future.completeExceptionally(e); + } + return future; + }); + } + + private CompletableFuture loadOpenIDProviderMetadataForKubernetesApiServer() { + CompletableFuture future = new CompletableFuture<>(); + try { + wellKnownApi.getServiceAccountIssuerOpenIDConfigurationAsync(new ApiCallback<>() { + @Override + public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + future.completeExceptionally(new AuthenticationException( + "Error retrieving OpenID Provider Metadata from Kubernetes API server: " + + e.getMessage())); + } + + @Override + public void onSuccess(String result, int statusCode, Map> responseHeaders) { + try { + // Validation that the token's issuer matches the issuer returned by the api server must be done + // after the cache load operation to ensure each token's issuer matches the fallback issuer + OpenIDProviderMetadata openIDProviderMetadata = reader.readValue(result); + future.complete(openIDProviderMetadata); + } catch (Exception e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + future.completeExceptionally(new AuthenticationException( + "Error retrieving OpenID Provider Metadata from Kubernetes API Server: " + + e.getMessage())); + } + } + + @Override + public void onUploadProgress(long bytesWritten, long contentLength, boolean done) { + + } + + @Override + public void onDownloadProgress(long bytesRead, long contentLength, boolean done) { + + } + }); + } catch (ApiException e) { + incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + future.completeExceptionally(new AuthenticationException( + "Error retrieving OpenID Provider Metadata from Kubernetes API server: " + e.getMessage())); + } + return future; + } + + /** + * Verify the issuer url, as required by the OpenID Connect spec: + * + * Per the OpenID Connect Discovery spec, the issuer value returned MUST be identical to the + * Issuer URL that was directly used to retrieve the configuration information. This MUST also + * be identical to the iss Claim value in ID Tokens issued from this Issuer. + * https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderConfigurationValidation + * + * @param issuer - the issuer used to retrieve the metadata + * @param metadata - the OpenID Provider Metadata + * @param isK8s - whether the issuer is represented by the Kubernetes API server. This affects error reporting. + * @throws AuthenticationException if the issuer does not exactly match the metadata issuer + */ + private void verifyIssuer(@Nonnull String issuer, OpenIDProviderMetadata metadata, + boolean isK8s) throws AuthenticationException { + if (!issuer.equals(metadata.getIssuer())) { + if (isK8s) { + incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER); + throw new AuthenticationException("Issuer not allowed: " + issuer); + } else { + incrementFailureMetric(AuthenticationExceptionCode.ISSUER_MISMATCH); + throw new AuthenticationException(String.format("Issuer URL mismatch: [%s] should match [%s]", + issuer, metadata.getIssuer())); + } + } + } +} diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java new file mode 100644 index 0000000000000..d28f255d1bea7 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java new file mode 100644 index 0000000000000..298492652c0a8 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java @@ -0,0 +1,530 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.github.tomakehurst.wiremock.WireMockServer; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.impl.DefaultJwtBuilder; +import io.jsonwebtoken.security.Keys; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.interfaces.RSAPublicKey; +import java.util.Base64; +import java.util.Date; +import java.util.HashMap; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import javax.naming.AuthenticationException; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; +import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.common.api.AuthData; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * An integration test relying on WireMock to simulate an OpenID Connect provider. + */ +public class AuthenticationProviderOpenIDIntegrationTest { + + AuthenticationProviderOpenID provider; + PrivateKey privateKey; + + // These are the kid values for JWKs in the /keys endpoint + String validJwk = "valid"; + String invalidJwk = "invalid"; + + // The valid issuer + String issuer; + String issuerWithTrailingSlash; + // This issuer is configured to return an issuer in the openid-configuration + // that does not match the issuer on the token + String issuerThatFails; + String issuerK8s; + WireMockServer server; + + @BeforeClass + void beforeClass() throws IOException { + + // Port matches the port supplied in the fakeKubeConfig.yaml resource, which makes the k8s integration + // tests work correctly. + server = new WireMockServer(wireMockConfig().port(0)); + server.start(); + issuer = server.baseUrl(); + issuerWithTrailingSlash = issuer + "/trailing-slash/"; + issuerThatFails = issuer + "/fail"; + issuerK8s = issuer + "/k8s"; + + // Set up a correct openid-configuration + server.stubFor( + get(urlEqualTo("/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "jwks_uri": "%s/keys" + } + """.replace("%s", server.baseUrl())))); + + // Set up a correct openid-configuration that the k8s integration test can use + // NOTE: integration tests revealed that the k8s client adds a trailing slash to the openid-configuration + // endpoint. + // NOTE: the jwks_uri is ignored, so we supply one that would fail here to ensure that we are not implicitly + // relying on the jwks_uri. + server.stubFor( + get(urlEqualTo("/k8s/.well-known/openid-configuration/")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "jwks_uri": "%s/no/keys/hosted/here" + } + """.formatted(issuer, issuer)))); + + // Set up a correct openid-configuration that has a trailing slash in the issuers URL. This is a + // behavior observed by Auth0. In this case, the token's iss claim also has a trailing slash. + // The server should normalize the URL and call the Authorization Server without the double slash. + // NOTE: the spec does not indicate that the jwks_uri must have the same prefix as the issuer, and that + // is used here to simplify the testing. + server.stubFor( + get(urlEqualTo("/trailing-slash/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "jwks_uri": "%s/keys" + } + """.formatted(issuerWithTrailingSlash, issuer)))); + + // Set up an incorrect openid-configuration where issuer does not match + server.stubFor( + get(urlEqualTo("/fail/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "https://wrong-issuer.com", + "jwks_uri": "%s/keys" + } + """.formatted(server.baseUrl())))); + + // Create the token key pair + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + privateKey = keyPair.getPrivate(); + RSAPublicKey rsaPublicKey = (RSAPublicKey) keyPair.getPublic(); + String n = Base64.getUrlEncoder().encodeToString(rsaPublicKey.getModulus().toByteArray()); + String e = Base64.getUrlEncoder().encodeToString(rsaPublicKey.getPublicExponent().toByteArray()); + + // Set up JWKS endpoint with a valid and an invalid public key + // The url matches are for both the normal and the k8s endpoints + server.stubFor( + get(urlMatching( "/keys|/k8s/openid/v1/jwks/")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody( + """ + { + "keys" : [ + { + "kid":"%s", + "kty":"RSA", + "alg":"RS256", + "n":"%s", + "e":"%s" + }, + { + "kid": "%s", + "kty":"RSA", + "n":"invalid-key", + "e":"AQAB" + } + ] + } + """.formatted(validJwk, n, e, invalidJwk)))); + + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer + "," + issuerWithTrailingSlash + + "," + issuerThatFails); + + // Create the fake kube config file. This file is configured via the env vars and is written to the + // target directory so maven clean will remove it. + byte[] template = Files.readAllBytes(Path.of(System.getenv("KUBECONFIG_TEMPLATE"))); + String kubeConfig = new String(template).replace("${WIRE_MOCK_PORT}", String.valueOf(server.port())); + Files.write(Path.of(System.getenv("KUBECONFIG")), kubeConfig.getBytes()); + + provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + } + + @AfterClass + void afterClass() { + server.stop(); + } + + @Test + public void testTokenWithValidJWK() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); + } + + @Test + public void testTokenWithTrailingSlashAndValidJWK() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, issuer + "/trailing-slash/", role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); + } + + @Test + public void testTokenWithInvalidJWK() throws Exception { + String role = "superuser"; + String token = generateToken(invalidJwk, issuer, role, "allowed-audience",0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + @Test + public void testAuthorizationServerReturnsIncorrectIssuerInOpenidConnectConfiguration() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, issuerThatFails, role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + @Test + public void testTokenWithInvalidAudience() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, issuer, role, "invalid-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + @Test + public void testTokenWithInvalidIssuer() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, "https://not-an-allowed-issuer.com", role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + @Test + public void testKubernetesApiServerAsDiscoverTrustedIssuerSuccess() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER"); + // Test requires that k8sIssuer is not in the allowed token issuers + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + // We use the normal issuer on the token because the /k8s endpoint is configured via the kube config file + // made as part of the test setup. The kube client then gets the issuer from the /k8s endpoint and discovers + // this issuer. + String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); + + // Ensure that a subsequent token with a different issuer still fails due to invalid issuer exception + String token2 = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token2)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("Issuer not allowed"), + "Unexpected error message: " + e.getMessage()); + } + } + + @Test + public void testKubernetesApiServerAsDiscoverTrustedIssuerFailsDueToMismatchedIssuerClaim() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + + @Test + public void testKubernetesApiServerAsDiscoverPublicKeySuccess() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS"); + // Test requires that k8sIssuer is not in the allowed token issuers + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); + + // Ensure that a subsequent token with a different issuer still fails due to invalid issuer exception + String token2 = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token2)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("Issuer not allowed"), + "Unexpected error message: " + e.getMessage()); + } + } + + @Test + public void testKubernetesApiServerAsDiscoverPublicKeyFailsDueToMismatchedIssuerClaim() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + @Test + public void testAuthenticationStateOpenIDForValidToken() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L); + AuthenticationState state = provider.newAuthState(null, null, null); + AuthData result = state.authenticateAsync(AuthData.of(token.getBytes())).get(); + assertNull(result); + assertEquals(state.getAuthRole(), role); + assertEquals(state.getAuthDataSource().getCommandData(), token); + assertFalse(state.isExpired()); + } + + @Test + public void testAuthenticationStateOpenIDForExpiredToken() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, -10000L); + AuthenticationState state = provider.newAuthState(null, null, null); + try { + state.authenticateAsync(AuthData.of(token.getBytes())).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + @Test + public void testAuthenticationStateOpenIDForValidTokenWithNoExp() throws Exception { + String role = "superuser"; + String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, null); + AuthenticationState state = provider.newAuthState(null, null, null); + try { + state.authenticateAsync(AuthData.of(token.getBytes())).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + @Test + public void testAuthenticationStateOpenIDForTokenExpiration() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); + // Use the leeway to allow the token to pass validation and then fail expiration + props.setProperty(AuthenticationProviderOpenID.ACCEPTED_TIME_LEEWAY_SECONDS, "10"); + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 0L); + AuthenticationState state = provider.newAuthState(null, null, null); + AuthData result = state.authenticateAsync(AuthData.of(token.getBytes())).get(); + assertNull(result); + assertEquals(state.getAuthRole(), role); + assertEquals(state.getAuthDataSource().getCommandData(), token); + assertTrue(state.isExpired()); + } + + @Test + void ensureRoleClaimForNonSubClaimReturnsRole() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "test"); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + + // Build a JWT with a custom claim + HashMap claims = new HashMap(); + claims.put("test", "my-role"); + String token = generateToken(validJwk, issuer, "not-my-role", "allowed-audience", 0L, + 0L, 10000L, claims); + assertEquals(provider.authenticateAsync(new AuthenticationDataCommand(token)).get(), "my-role"); + } + + @Test + void ensureRoleClaimForNonSubClaimFailsWhenClaimIsMissing() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "test"); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + + // Build a JWT without the "test" claim, which should cause the authentication to fail + String token = generateToken(validJwk, issuer, "not-my-role", "allowed-audience", 0L, + 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); + } + } + + // This test is somewhat counterintuitive. We allow the state object to change roles, but then we fail it + // in the ServerCnx handling of the state object. As such, it is essential that the state object allow + // the role to change. + @Test + public void testAuthenticationStateOpenIDAllowsRoleChange() throws Exception { + String role1 = "superuser"; + String token1 = generateToken(validJwk, issuer, role1, "allowed-audience", 0L, 0L, 10000L); + String role2 = "otheruser"; + String token2 = generateToken(validJwk, issuer, role2, "allowed-audience", 0L, 0L, 10000L); + AuthenticationState state = provider.newAuthState(null, null, null); + AuthData result1 = state.authenticateAsync(AuthData.of(token1.getBytes())).get(); + assertNull(result1); + assertEquals(state.getAuthRole(), role1); + assertEquals(state.getAuthDataSource().getCommandData(), token1); + assertFalse(state.isExpired()); + + AuthData result2 = state.authenticateAsync(AuthData.of(token2.getBytes())).get(); + assertNull(result2); + assertEquals(state.getAuthRole(), role2); + assertEquals(state.getAuthDataSource().getCommandData(), token2); + assertFalse(state.isExpired()); + } + + private String generateToken(String kid, String issuer, String subject, String audience, + Long iatOffset, Long nbfOffset, Long expOffset) { + return generateToken(kid, issuer, subject, audience, iatOffset, nbfOffset, expOffset, new HashMap<>()); + } + + private String generateToken(String kid, String issuer, String subject, String audience, + Long iatOffset, Long nbfOffset, Long expOffset, HashMap extraClaims) { + long now = System.currentTimeMillis(); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + defaultJwtBuilder.setHeaderParam("kid", kid); + defaultJwtBuilder.setHeaderParam("typ", "JWT"); + defaultJwtBuilder.setHeaderParam("alg", "RS256"); + defaultJwtBuilder.setIssuer(issuer); + defaultJwtBuilder.setSubject(subject); + defaultJwtBuilder.setAudience(audience); + defaultJwtBuilder.setIssuedAt(iatOffset != null ? new Date(now + iatOffset) : null); + defaultJwtBuilder.setNotBefore(nbfOffset != null ? new Date(now + nbfOffset) : null); + defaultJwtBuilder.setExpiration(expOffset != null ? new Date(now + expOffset) : null); + defaultJwtBuilder.addClaims(extraClaims); + defaultJwtBuilder.signWith(privateKey); + return defaultJwtBuilder.compact(); + } + +} diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java new file mode 100644 index 0000000000000..74abffe9c38e8 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java @@ -0,0 +1,387 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static org.testng.Assert.assertNull; +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.impl.DefaultJwtBuilder; +import io.jsonwebtoken.security.Keys; +import java.security.KeyPair; +import java.sql.Date; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import javax.naming.AuthenticationException; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Unit tests to cover the AuthenticationProviderOpenID without any network calls. + *

+ * This class only tests the verification of tokens. It does not test the integration to retrieve tokens + * from an identity provider. See {@link AuthenticationProviderOpenIDIntegrationTest} for the authorization + * server integration tests. + *

+ * Note: this class uses the io.jsonwebtoken library here because it has more utilities than the auth0 library. + * The jsonwebtoken library makes it easy to generate key pairs for many algorithms, and it also has an enum + * that can be used to assert that unsupported algorithms properly fail validation. + */ +public class AuthenticationProviderOpenIDTest { + + // https://www.rfc-editor.org/rfc/rfc7518#section-3.1 + @DataProvider(name = "supportedAlgorithms") + public static Object[][] supportedAlgorithms() { + return new Object[][] { + { SignatureAlgorithm.RS256 }, + { SignatureAlgorithm.RS384 }, + { SignatureAlgorithm.RS512 }, + { SignatureAlgorithm.ES256 }, + { SignatureAlgorithm.ES384 }, + { SignatureAlgorithm.ES512 } + }; + } + + // Provider to use in common tests that are not verifying the configuration of the provider itself. + AuthenticationProviderOpenID basicProvider; + final String basicProviderAudience = "my-special-audience"; + + @BeforeClass + public void setup() throws Exception { + Properties properties = new Properties(); + properties.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience); + properties.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://my-issuer.com"); + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setProperties(properties); + basicProvider = new AuthenticationProviderOpenID(); + basicProvider.initialize(conf); + } + + @Test + public void testNullToken() { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Assert.assertThrows(AuthenticationException.class, + () -> provider.authenticate(new AuthenticationDataCommand(null))); + } + + @Test + public void testThatNullAlgFails() { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Assert.assertThrows(AuthenticationException.class, + () -> provider.verifyJWT(null, null, null)); + } + + @Test + public void testThatUnsupportedAlgsThrowExceptions() { + Set unsupportedAlgs = new HashSet<>(Set.of(SignatureAlgorithm.values())); + Arrays.stream(supportedAlgorithms()).map(o -> (SignatureAlgorithm) o[0]).toList() + .forEach(unsupportedAlgs::remove); + unsupportedAlgs.forEach(unsupportedAlg -> { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + // We don't create a public key because it's irrelevant + Assert.assertThrows(AuthenticationException.class, + () -> provider.verifyJWT(null, unsupportedAlg.getValue(), null)); + }); + } + + @Test(dataProvider = "supportedAlgorithms") + public void testThatSupportedAlgsWork(SignatureAlgorithm alg) throws AuthenticationException { + KeyPair keyPair = Keys.keyPairFor(alg); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); + defaultJwtBuilder.signWith(keyPair.getPrivate()); + + // Convert to the right class + DecodedJWT expectedValue = JWT.decode(defaultJwtBuilder.compact()); + DecodedJWT actualValue = basicProvider.verifyJWT(keyPair.getPublic(), alg.getValue(), expectedValue); + Assert.assertEquals(expectedValue, actualValue); + } + + @Test + public void testThatSupportedAlgWithMismatchedPublicKeyFromDifferentAlgFamilyFails() { + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); + defaultJwtBuilder.signWith(keyPair.getPrivate()); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + // Choose a different algorithm from a different alg family + Assert.assertThrows(AuthenticationException.class, + () -> provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.ES512.getValue(), jwt)); + } + + @Test + public void testThatSupportedAlgWithMismatchedPublicKeyFromSameAlgFamilyFails() { + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); + defaultJwtBuilder.signWith(keyPair.getPrivate()); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + // Choose a different algorithm but within the same alg family as above + Assert.assertThrows(AuthenticationException.class, + () -> provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS512.getValue(), jwt)); + } + + @Test + public void ensureExpiredTokenFails() { + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); + defaultJwtBuilder.setExpiration(Date.from(Instant.now().minusSeconds(60))); + defaultJwtBuilder.signWith(keyPair.getPrivate()); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + Assert.assertThrows(AuthenticationException.class, + () -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), jwt)); + } + + @Test + public void ensureFutureNBFFails() throws Exception { + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); + // Override the exp set in the above method + defaultJwtBuilder.setNotBefore(Date.from(Instant.now().plusSeconds(60))); + defaultJwtBuilder.signWith(keyPair.getPrivate()); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + Assert.assertThrows(AuthenticationException.class, + () -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), jwt)); + } + + @Test + public void ensureFutureIATFails() throws Exception { + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); + // Override the exp set in the above method + defaultJwtBuilder.setIssuedAt(Date.from(Instant.now().plusSeconds(60))); + defaultJwtBuilder.signWith(keyPair.getPrivate()); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + Assert.assertThrows(AuthenticationException.class, + () -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), jwt)); + } + + @Test + public void ensureRecentlyExpiredTokenWithinConfiguredLeewaySucceeds() throws Exception { + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + + // Set up the provider + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ACCEPTED_TIME_LEEWAY_SECONDS, "10"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "leewayAudience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://localhost:8080"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + + // Build the JWT with an only recently expired token + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + addValidMandatoryClaims(defaultJwtBuilder, "leewayAudience"); + defaultJwtBuilder.setExpiration(Date.from(Instant.ofEpochMilli(System.currentTimeMillis() - 5000L))); + defaultJwtBuilder.signWith(keyPair.getPrivate()); + DecodedJWT expectedValue = JWT.decode(defaultJwtBuilder.compact()); + + // Test the verification + DecodedJWT actualValue = null; + try { + actualValue = provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), expectedValue); + } catch (Exception e) { + Assert.fail("Token verification should not have thrown an exception.", e); + } + Assert.assertEquals(expectedValue, actualValue); + } + + @Test + public void ensureEmptyIssuersFailsInitialization() { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + } + + @Test + public void ensureEmptyIssuersFailsInitializationWithDisabledDiscoveryMode() { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "DISABLED"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + } + + @Test + public void ensureEmptyIssuersWithK8sTrustedIssuerEnabledPassesInitialization() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "my-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + } + + @Test + public void ensureEmptyIssuersWithK8sPublicKeyEnabledPassesInitialization() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "my-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + } + + @Test + public void ensureNullIssuersFailsInitialization() { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + ServiceConfiguration config = new ServiceConfiguration(); + // Make sure this still defaults to null. + assertNull(config.getProperties().get(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS)); + Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + } + + @Test + public void ensureInsecureIssuerFailsInitialization() { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com,http://myissuer.com"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + } + + @Test void ensureMissingRoleClaimReturnsNull() throws Exception { + // Build an empty JWT + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + defaultJwtBuilder.setAudience(basicProviderAudience); + DecodedJWT jwtWithoutSub = JWT.decode(defaultJwtBuilder.compact()); + + // A JWT with an empty role claim must result in a null role + assertNull(basicProvider.getRole(jwtWithoutSub)); + } + + @Test void ensureRoleClaimForStringReturnsRole() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience); + props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "sub"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + + // Build an empty JWT + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + defaultJwtBuilder.setSubject("my-role"); + defaultJwtBuilder.setAudience(basicProviderAudience); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + + Assert.assertEquals("my-role", provider.getRole(jwt)); + } + + @Test void ensureRoleClaimForSingletonListReturnsRole() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience); + props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + + // Build an empty JWT + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + HashMap> claims = new HashMap(); + claims.put("roles", Collections.singletonList("my-role")); + defaultJwtBuilder.setClaims(claims); + defaultJwtBuilder.setAudience(basicProviderAudience); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + + Assert.assertEquals("my-role", provider.getRole(jwt)); + } + + @Test void ensureRoleClaimForMultiEntryListReturnsFirstRole() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience); + props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + + // Build an empty JWT + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + HashMap> claims = new HashMap<>(); + claims.put("roles", Arrays.asList("my-role-1", "my-role-2")); + defaultJwtBuilder.setClaims(claims); + defaultJwtBuilder.setAudience(basicProviderAudience); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + + Assert.assertEquals("my-role-1", provider.getRole(jwt)); + } + + @Test void ensureRoleClaimForEmptyListReturnsNull() throws Exception { + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + Properties props = new Properties(); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "no-role-audience-test"); + props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + provider.initialize(config); + + // Build an empty JWT + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + HashMap> claims = new HashMap<>(); + claims.put("roles", Collections.emptyList()); + defaultJwtBuilder.setClaims(claims); + defaultJwtBuilder.setAudience("no-role-audience-test"); + DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); + + // A JWT with an empty list role claim must result in a null role + assertNull(provider.getRole(jwt)); + } + + // Method simplifies adding the required claims. For the tests that need to verify invalid values for these + // claims, it is sufficient to set the values after calling this method. + private void addValidMandatoryClaims(DefaultJwtBuilder defaultJwtBuilder, String audience) { + defaultJwtBuilder.setExpiration(Date.from(Instant.now().plusSeconds(60))); + defaultJwtBuilder.setNotBefore(Date.from(Instant.now())); + defaultJwtBuilder.setIssuedAt(Date.from(Instant.now())); + defaultJwtBuilder.setAudience(audience); + defaultJwtBuilder.setSubject("my-role"); + } +} diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java new file mode 100644 index 0000000000000..c0945ae43c3e8 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; +import static org.testng.AssertJUnit.fail; +import javax.naming.AuthenticationException; +import org.testng.annotations.Test; + +/** + * Unit tests primarily focused on failure scenarios for {@link AuthenticationStateOpenID}. The happy path is covered + * by {@link AuthenticationProviderOpenIDIntegrationTest}. + */ +public class AuthenticationStateOpenIDTest { + @Test + void getRoleOnAuthStateShouldFailIfNotAuthenticated() { + AuthenticationStateOpenID state = new AuthenticationStateOpenID(null, null, null); + assertFalse(state.isComplete()); + assertThrows(AuthenticationException.class, state::getAuthRole); + } + + @Test + void getAuthDataOnAuthStateShouldBeNullIfNotAuthenticated() { + AuthenticationStateOpenID state = new AuthenticationStateOpenID(null, null, null); + assertNull(state.getAuthDataSource()); + } + + // We override this behavior to make it clear that this provider is only meant to be used asynchronously. + @SuppressWarnings("deprecation") + @Test + void authenticateShouldThrowNotImplementedException() { + AuthenticationStateOpenID state = new AuthenticationStateOpenID(null, null, null); + try { + state.authenticate(null); + fail("Expected AuthenticationException to be thrown"); + } catch (AuthenticationException e) { + assertEquals(e.getMessage(), "Not supported"); + } + } +} diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java new file mode 100644 index 0000000000000..ad06d4b0c109a --- /dev/null +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.oidc; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import java.util.Collections; +import java.util.Properties; +import java.util.Set; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.testng.annotations.Test; + +public class ConfigUtilsTest { + + @Test + public void testGetConfigValueAsStringWorks() { + Properties props = new Properties(); + props.setProperty("prop1", "audience"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + String actual = ConfigUtils.getConfigValueAsString(config, "prop1"); + assertEquals("audience", actual); + } + + @Test + public void testGetConfigValueAsStringReturnsNullIfMissing() { + Properties props = new Properties(); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + String actual = ConfigUtils.getConfigValueAsString(config, "prop1"); + assertNull(actual); + } + + @Test + public void testGetConfigValueAsStringWithDefaultWorks() { + Properties props = new Properties(); + props.setProperty("prop1", "audience"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + String actual = ConfigUtils.getConfigValueAsString(config, "prop1", "default"); + assertEquals("audience", actual); + } + + @Test + public void testGetConfigValueAsStringReturnsDefaultIfMissing() { + Properties props = new Properties(); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + String actual = ConfigUtils.getConfigValueAsString(config, "prop1", "default"); + assertEquals("default", actual); + } + + @Test + public void testGetConfigValueAsSetReturnsWorks() { + Properties props = new Properties(); + props.setProperty("prop1", "a, b, c"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + Set actual = ConfigUtils.getConfigValueAsSet(config, "prop1"); + // Trims all whitespace + assertEquals(Set.of("a", "b", "c"), actual); + } + + @Test + public void testGetConfigValueAsSetReturnsEmptySetIfMissing() { + Properties props = new Properties(); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + Set actual = ConfigUtils.getConfigValueAsSet(config, "prop1"); + assertEquals(Collections.emptySet(), actual); + } + + @Test + public void testGetConfigValueAsSetReturnsEmptySetIfBlankString() { + Properties props = new Properties(); + props.setProperty("prop1", " "); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + Set actual = ConfigUtils.getConfigValueAsSet(config, "prop1"); + assertEquals(Collections.emptySet(), actual); + } + + @Test + public void testGetConfigValueAsIntegerWorks() { + Properties props = new Properties(); + props.setProperty("prop1", "1234"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + int actual = ConfigUtils.getConfigValueAsInt(config, "prop1", 9); + assertEquals(1234, actual); + } + + @Test + public void testGetConfigValueAsIntegerReturnsDefaultIfNAN() { + Properties props = new Properties(); + props.setProperty("prop1", "non-a-number"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + int actual = ConfigUtils.getConfigValueAsInt(config, "prop1", 9); + assertEquals(9, actual); + } + + @Test + public void testGetConfigValueAsIntegerReturnsDefaultIfMissingProp() { + Properties props = new Properties(); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + int actual = ConfigUtils.getConfigValueAsInt(config, "prop1", 10); + assertEquals(10, actual); + } + + @Test + public void testGetConfigValueAsBooleanReturnsDefaultIfMissingProp() { + Properties props = new Properties(); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + boolean actualFalse = ConfigUtils.getConfigValueAsBoolean(config, "prop1", false); + assertFalse(actualFalse); + boolean actualTrue = ConfigUtils.getConfigValueAsBoolean(config, "prop1", true); + assertTrue(actualTrue); + } + + @Test + public void testGetConfigValueAsBooleanWorks() { + Properties props = new Properties(); + props.setProperty("prop1", "true"); + ServiceConfiguration config = new ServiceConfiguration(); + config.setProperties(props); + boolean actual = ConfigUtils.getConfigValueAsBoolean(config, "prop1", false); + assertTrue(actual); + } + +} diff --git a/pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml b/pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml new file mode 100644 index 0000000000000..ef3d373399d61 --- /dev/null +++ b/pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +current-context: wire-mock-server +apiVersion: v1 +clusters: +- cluster: + api-version: v1 + server: http://localhost:${WIRE_MOCK_PORT}/k8s + name: wire-mock-server +contexts: +- context: + cluster: wire-mock-server + namespace: pulsar + user: test-user + name: wire-mock-server +kind: Config +users: +- name: test-user + user: + token: fake-token \ No newline at end of file From bafecb2a9c0e73942de6a38df72dd5888d5afd66 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 10 Apr 2023 18:30:22 -0700 Subject: [PATCH 083/280] [improve] Removed usages of SafeRun (#20060) ### Motivation With BK 4.16, we don't need to pass `SafeRunnable` instances to the `OrderedExecutor` anymore. The executor has embedded the logic of checking and logging exceptions. Removing the SafeRun will avoid extra allocations in critical path and clutter of the code ### Verifying this change - [ ] Make sure that the change passes the CI checks. *(Please pick either of the following options)* This change is a trivial rework / code cleanup without any test coverage. *(or)* This change is already covered by existing tests, such as *(please describe tests)*. *(or)* This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Does this pull request potentially affect one of the following parts: *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [ ] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [ ] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [x] `doc-not-needed` - [ ] `doc-complete` ### Matching PR in forked repository PR in forked repository: (https://github.com/merlimat/pulsar/pull/6) --- .../mledger/impl/ManagedCursorImpl.java | 21 +++-- .../mledger/impl/ManagedLedgerImpl.java | 44 +++++----- .../mledger/impl/MetaStoreImpl.java | 27 +++--- .../bookkeeper/mledger/impl/OpAddEntry.java | 10 +-- .../bookkeeper/mledger/impl/OpReadEntry.java | 13 ++- .../mledger/impl/ShadowManagedLedgerImpl.java | 13 ++- .../cache/RangeEntryCacheManagerImpl.java | 5 +- .../bookkeeper/mledger/util/SafeRun.java | 57 ------------ .../impl/ManagedCursorConcurrencyTest.java | 5 +- .../prometheus/PrometheusMetricsServlet.java | 5 +- .../pulsar/broker/service/BrokerService.java | 87 +++++++++++-------- .../pulsar/broker/service/ServerCnx.java | 5 +- ...PersistentDispatcherMultipleConsumers.java | 11 ++- ...sistentDispatcherSingleActiveConsumer.java | 25 ++---- ...tStickyKeyDispatcherMultipleConsumers.java | 5 +- ...tStreamingDispatcherMultipleConsumers.java | 12 ++- ...reamingDispatcherSingleActiveConsumer.java | 10 +-- .../StreamingEntryReader.java | 19 ++-- .../coordination/impl/LeaderElectionImpl.java | 3 +- 19 files changed, 150 insertions(+), 227 deletions(-) delete mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 05cad09b01833..ef607fa7ed7cf 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -25,7 +25,6 @@ import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.DEFAULT_LEDGER_DELETE_RETRIES; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.createManagedLedgerException; import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.collect.Collections2; @@ -1358,7 +1357,7 @@ public void asyncResetCursor(Position newPos, boolean forceReset, AsyncCallbacks final PositionImpl newPosition = (PositionImpl) newPos; // order trim and reset operations on a ledger - ledger.getExecutor().execute(safeRun(() -> { + ledger.getExecutor().execute(() -> { PositionImpl actualPosition = newPosition; if (!ledger.isValidPosition(actualPosition) @@ -1375,7 +1374,7 @@ public void asyncResetCursor(Position newPos, boolean forceReset, AsyncCallbacks } internalResetCursor(actualPosition, callback); - })); + }); } @Override @@ -2055,7 +2054,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { + "is later.", mdEntry.newPosition, persistentMarkDeletePosition); } // run with executor to prevent deadlock - ledger.getExecutor().execute(safeRun(() -> mdEntry.triggerComplete())); + ledger.getExecutor().execute(() -> mdEntry.triggerComplete()); return; } @@ -2074,7 +2073,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { + "in progress {} is later.", mdEntry.newPosition, inProgressLatest); } // run with executor to prevent deadlock - ledger.getExecutor().execute(safeRun(() -> mdEntry.triggerComplete())); + ledger.getExecutor().execute(() -> mdEntry.triggerComplete()); return; } @@ -2611,8 +2610,8 @@ private boolean shouldPersistUnackRangesToLedger() { private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl position, Map properties, MetaStoreCallback callback, boolean persistIndividualDeletedMessageRanges) { if (state == State.Closed) { - ledger.getExecutor().execute(safeRun(() -> callback.operationFailed(new MetaStoreException( - new CursorAlreadyClosedException(name + " cursor already closed"))))); + ledger.getExecutor().execute(() -> callback.operationFailed(new MetaStoreException( + new CursorAlreadyClosedException(name + " cursor already closed")))); return; } @@ -2845,7 +2844,7 @@ private CompletableFuture doCreateNewMetadataLedger() { return; } - ledger.getExecutor().execute(safeRun(() -> { + ledger.getExecutor().execute(() -> { ledger.mbean.endCursorLedgerCreateOp(); if (rc != BKException.Code.OK) { log.warn("[{}] Error creating ledger for cursor {}: {}", ledger.getName(), name, @@ -2858,7 +2857,7 @@ private CompletableFuture doCreateNewMetadataLedger() { log.debug("[{}] Created ledger {} for cursor {}", ledger.getName(), lh.getId(), name); } future.complete(lh); - })); + }); }, LedgerMetadataUtils.buildAdditionalMetadataForCursor(name)); return future; @@ -3192,7 +3191,7 @@ private void asyncDeleteLedger(final LedgerHandle lh, int retry) { log.warn("[{}] Failed to delete ledger {}: {}", ledger.getName(), lh.getId(), BKException.getMessage(rc)); if (!isNoSuchLedgerExistsException(rc)) { - ledger.getScheduledExecutor().schedule(safeRun(() -> asyncDeleteLedger(lh, retry - 1)), + ledger.getScheduledExecutor().schedule(() -> asyncDeleteLedger(lh, retry - 1), DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS); } return; @@ -3227,7 +3226,7 @@ private void asyncDeleteCursorLedger(int retry) { log.warn("[{}][{}] Failed to delete ledger {}: {}", ledger.getName(), name, cursorLedger.getId(), BKException.getMessage(rc)); if (!isNoSuchLedgerExistsException(rc)) { - ledger.getScheduledExecutor().schedule(safeRun(() -> asyncDeleteCursorLedger(retry - 1)), + ledger.getScheduledExecutor().schedule(() -> asyncDeleteCursorLedger(retry - 1), DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS); } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 8376ee1bb8467..c74db884a95a9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -22,7 +22,6 @@ import static com.google.common.base.Preconditions.checkState; import static java.lang.Math.min; import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.BoundType; import com.google.common.collect.Lists; @@ -409,7 +408,7 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { if (!ledgers.isEmpty()) { final long id = ledgers.lastKey(); OpenCallback opencb = (rc, lh, ctx1) -> { - executor.execute(safeRun(() -> { + executor.execute(() -> { mbean.endDataLedgerOpenOp(); if (log.isDebugEnabled()) { log.debug("[{}] Opened ledger {}: {}", name, id, BKException.getMessage(rc)); @@ -439,7 +438,7 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { callback.initializeFailed(createManagedLedgerException(rc)); return; } - })); + }); }; if (log.isDebugEnabled()) { @@ -522,7 +521,7 @@ public void operationFailed(MetaStoreException e) { return; } - executor.execute(safeRun(() -> { + executor.execute(() -> { mbean.endDataLedgerCreateOp(); if (rc != BKException.Code.OK) { callback.initializeFailed(createManagedLedgerException(rc)); @@ -551,7 +550,7 @@ public void operationFailed(MetaStoreException e) { // Save it back to ensure all nodes exist store.asyncUpdateLedgerIds(name, getManagedLedgerInfo(), ledgersStat, storeLedgersCb); - })); + }); }, ledgerMetadata); } @@ -774,10 +773,10 @@ public void asyncAddEntry(ByteBuf buffer, AddEntryCallback callback, Object ctx) buffer.retain(); // Jump to specific thread to avoid contention from writers writing from different threads - executor.execute(safeRun(() -> { + executor.execute(() -> { OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, callback, ctx); internalAsyncAddEntry(addOperation); - })); + }); } @Override @@ -790,10 +789,10 @@ public void asyncAddEntry(ByteBuf buffer, int numberOfMessages, AddEntryCallback buffer.retain(); // Jump to specific thread to avoid contention from writers writing from different threads - executor.execute(safeRun(() -> { + executor.execute(() -> { OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, numberOfMessages, callback, ctx); internalAsyncAddEntry(addOperation); - })); + }); } protected synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { @@ -2374,7 +2373,7 @@ void notifyCursors() { break; } - executor.execute(safeRun(waitingCursor::notifyEntriesAvailable)); + executor.execute(waitingCursor::notifyEntriesAvailable); } } @@ -2385,7 +2384,7 @@ void notifyWaitingEntryCallBacks() { break; } - executor.execute(safeRun(cb::entriesAvailable)); + executor.execute(cb::entriesAvailable); } } @@ -2432,16 +2431,16 @@ private void trimConsumedLedgersInBackground() { @Override public void trimConsumedLedgersInBackground(CompletableFuture promise) { - executor.execute(safeRun(() -> internalTrimConsumedLedgers(promise))); + executor.execute(() -> internalTrimConsumedLedgers(promise)); } public void trimConsumedLedgersInBackground(boolean isTruncate, CompletableFuture promise) { - executor.execute(safeRun(() -> internalTrimLedgers(isTruncate, promise))); + executor.execute(() -> internalTrimLedgers(isTruncate, promise)); } private void scheduleDeferredTrimming(boolean isTruncate, CompletableFuture promise) { - scheduledExecutor.schedule(safeRun(() -> trimConsumedLedgersInBackground(isTruncate, promise)), 100, - TimeUnit.MILLISECONDS); + scheduledExecutor.schedule(() -> trimConsumedLedgersInBackground(isTruncate, promise), + 100, TimeUnit.MILLISECONDS); } private void maybeOffloadInBackground(CompletableFuture promise) { @@ -2456,7 +2455,7 @@ private void maybeOffloadInBackground(CompletableFuture promise) { final long offloadThresholdInSeconds = Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInSeconds()).orElse(-1L); if (offloadThresholdInBytes >= 0 || offloadThresholdInSeconds >= 0) { - executor.execute(safeRun(() -> maybeOffload(offloadThresholdInBytes, offloadThresholdInSeconds, promise))); + executor.execute(() -> maybeOffload(offloadThresholdInBytes, offloadThresholdInSeconds, promise)); } } @@ -2477,7 +2476,7 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS } if (!offloadMutex.tryLock()) { - scheduledExecutor.schedule(safeRun(() -> maybeOffloadInBackground(finalPromise)), + scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise), 100, TimeUnit.MILLISECONDS); return; } @@ -2956,7 +2955,7 @@ private void asyncDeleteLedger(long ledgerId, long retry) { log.warn("[{}] Ledger was already deleted {}", name, ledgerId); } else if (rc != BKException.Code.OK) { log.error("[{}] Error deleting ledger {} : {}", name, ledgerId, BKException.getMessage(rc)); - scheduledExecutor.schedule(safeRun(() -> asyncDeleteLedger(ledgerId, retry - 1)), + scheduledExecutor.schedule(() -> asyncDeleteLedger(ledgerId, retry - 1), DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS); } else { if (log.isDebugEnabled()) { @@ -3260,7 +3259,7 @@ private void tryTransformLedgerInfo(long ledgerId, LedgerInfoTransformation tran if (!metadataMutex.tryLock()) { // retry in 100 milliseconds scheduledExecutor.schedule( - safeRun(() -> tryTransformLedgerInfo(ledgerId, transformation, finalPromise)), 100, + () -> tryTransformLedgerInfo(ledgerId, transformation, finalPromise), 100, TimeUnit.MILLISECONDS); } else { // lock acquired CompletableFuture unlockingPromise = new CompletableFuture<>(); @@ -4011,9 +4010,8 @@ private void scheduleTimeoutTask() { timeoutSec = timeoutSec <= 0 ? Math.max(config.getAddEntryTimeoutSeconds(), config.getReadEntryTimeoutSeconds()) : timeoutSec; - this.timeoutTask = this.scheduledExecutor.scheduleAtFixedRate(safeRun(() -> { - checkTimeouts(); - }), timeoutSec, timeoutSec, TimeUnit.SECONDS); + this.timeoutTask = this.scheduledExecutor.scheduleAtFixedRate( + this::checkTimeouts, timeoutSec, timeoutSec, TimeUnit.SECONDS); } } @@ -4336,7 +4334,7 @@ protected void updateLastLedgerCreatedTimeAndScheduleRolloverTask() { checkLedgerRollTask.cancel(true); } this.checkLedgerRollTask = this.scheduledExecutor.schedule( - safeRun(this::rollCurrentLedgerIfFull), this.maximumRolloverTimeMs, TimeUnit.MILLISECONDS); + this::rollCurrentLedgerIfFull, this.maximumRolloverTimeMs, TimeUnit.MILLISECONDS); } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index fb93e929f50c7..fcce1f7766cc8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -42,7 +42,6 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; -import org.apache.bookkeeper.util.SafeRunnable; import org.apache.commons.lang.StringUtils; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.compression.CompressionCodec; @@ -156,7 +155,7 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map .exceptionally(ex -> { try { executor.executeOrdered(ledgerName, - SafeRunnable.safeRun(() -> callback.operationFailed(getException(ex)))); + () -> callback.operationFailed(getException(ex))); } catch (RejectedExecutionException e) { //executor maybe shutdown, use common pool to run callback. CompletableFuture.runAsync(() -> callback.operationFailed(getException(ex))); @@ -204,8 +203,8 @@ public void asyncUpdateLedgerIds(String ledgerName, ManagedLedgerInfo mlInfo, St .thenAcceptAsync(newVersion -> callback.operationComplete(null, newVersion), executor.chooseThread(ledgerName)) .exceptionally(ex -> { - executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback - .operationFailed(getException(ex)))); + executor.executeOrdered(ledgerName, + () -> callback.operationFailed(getException(ex))); return null; }); } @@ -221,8 +220,8 @@ public void getCursors(String ledgerName, MetaStoreCallback> callba .thenAcceptAsync(cursors -> callback.operationComplete(cursors, null), executor .chooseThread(ledgerName)) .exceptionally(ex -> { - executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback - .operationFailed(getException(ex)))); + executor.executeOrdered(ledgerName, + () -> callback.operationFailed(getException(ex))); return null; }); } @@ -249,8 +248,8 @@ public void asyncGetCursorInfo(String ledgerName, String cursorName, } }, executor.chooseThread(ledgerName)) .exceptionally(ex -> { - executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback - .operationFailed(getException(ex)))); + executor.executeOrdered(ledgerName, + () -> callback.operationFailed(getException(ex))); return null; }); } @@ -284,8 +283,8 @@ public void asyncUpdateCursorInfo(String ledgerName, String cursorName, ManagedC .thenAcceptAsync(optStat -> callback.operationComplete(null, optStat), executor .chooseThread(ledgerName)) .exceptionally(ex -> { - executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback - .operationFailed(getException(ex)))); + executor.executeOrdered(ledgerName, + () -> callback.operationFailed(getException(ex))); return null; }); } @@ -303,7 +302,7 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal callback.operationComplete(null, null); }, executor.chooseThread(ledgerName)) .exceptionally(ex -> { - executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> { + executor.executeOrdered(ledgerName, () -> { Throwable actEx = FutureUtil.unwrapCompletionException(ex); if (actEx instanceof MetadataStoreException.NotFoundException){ log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName); @@ -311,7 +310,7 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal return; } callback.operationFailed(getException(ex)); - })); + }); return null; }); } @@ -329,8 +328,8 @@ public void removeManagedLedger(String ledgerName, MetaStoreCallback callb callback.operationComplete(null, null); }, executor.chooseThread(ledgerName)) .exceptionally(ex -> { - executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback - .operationFailed(getException(ex)))); + executor.executeOrdered(ledgerName, + () -> callback.operationFailed(getException(ex))); return null; }); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index c56123c24cac1..ae2beafb64374 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -35,8 +35,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; -import org.apache.bookkeeper.mledger.util.SafeRun; -import org.apache.bookkeeper.util.SafeRunnable; /** @@ -44,7 +42,7 @@ * */ @Slf4j -public class OpAddEntry extends SafeRunnable implements AddCallback, CloseCallback { +public class OpAddEntry implements AddCallback, CloseCallback, Runnable { protected ManagedLedgerImpl ml; LedgerHandle ledger; private long entryId; @@ -212,7 +210,7 @@ public void addComplete(int rc, final LedgerHandle lh, long entryId, Object ctx) // Called in executor hashed on managed ledger name, once the add operation is complete @Override - public void safeRun() { + public void run() { if (payloadProcessorHandle != null) { payloadProcessorHandle.release(); } @@ -328,11 +326,11 @@ void handleAddFailure(final LedgerHandle lh) { ManagedLedgerImpl finalMl = this.ml; finalMl.mbean.recordAddEntryError(); - finalMl.getExecutor().execute(SafeRun.safeRun(() -> { + finalMl.getExecutor().execute(() -> { // Force the creation of a new ledger. Doing it in a background thread to avoid acquiring ML lock // from a BK callback. finalMl.ledgerClosed(lh); - })); + }); } void close() { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java index 81b14359514b9..19211553a5f74 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java @@ -18,7 +18,6 @@ */ package org.apache.bookkeeper.mledger.impl; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; import java.util.ArrayList; @@ -108,10 +107,10 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { if (!entries.isEmpty()) { // There were already some entries that were read before, we can return them - cursor.ledger.getExecutor().execute(safeRun(() -> { + cursor.ledger.getExecutor().execute(() -> { callback.readEntriesComplete(entries, ctx); recycle(); - })); + }); } else if (cursor.config.isAutoSkipNonRecoverableData() && exception instanceof NonRecoverableLedgerException) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", cursor.ledger.getName(), cursor.getName(), readPosition, exception.getMessage()); @@ -161,20 +160,20 @@ void checkReadCompletion() { && maxPosition.compareTo(readPosition) > 0) { // We still have more entries to read from the next ledger, schedule a new async operation - cursor.ledger.getExecutor().execute(safeRun(() -> { + cursor.ledger.getExecutor().execute(() -> { readPosition = cursor.ledger.startReadOperationOnLedger(nextReadPosition); cursor.ledger.asyncReadEntries(OpReadEntry.this); - })); + }); } else { // The reading was already completed, release resources and trigger callback try { cursor.readOperationCompleted(); } finally { - cursor.ledger.getExecutor().execute(safeRun(() -> { + cursor.ledger.getExecutor().execute(() -> { callback.readEntriesComplete(entries, ctx); recycle(); - })); + }); } } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java index b1f239413472f..b33dd87543f77 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java @@ -19,7 +19,6 @@ package org.apache.bookkeeper.mledger.impl; import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -66,13 +65,13 @@ public ShadowManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper book @Override synchronized void initialize(ManagedLedgerInitializeLedgerCallback callback, Object ctx) { log.info("Opening shadow managed ledger {} with source={}", name, sourceMLName); - executor.execute(safeRun(() -> doInitialize(callback, ctx))); + executor.execute(() -> doInitialize(callback, ctx)); } private void doInitialize(ManagedLedgerInitializeLedgerCallback callback, Object ctx) { // Fetch the list of existing ledgers in the source managed ledger store.watchManagedLedgerInfo(sourceMLName, (managedLedgerInfo, stat) -> - executor.execute(safeRun(() -> processSourceManagedLedgerInfo(managedLedgerInfo, stat))) + executor.execute(() -> processSourceManagedLedgerInfo(managedLedgerInfo, stat)) ); store.getManagedLedgerInfo(sourceMLName, false, null, new MetaStore.MetaStoreCallback<>() { @Override @@ -106,7 +105,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) final long lastLedgerId = ledgers.lastKey(); mbean.startDataLedgerOpenOp(); - AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> executor.execute(safeRun(() -> { + AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> executor.execute(() -> { mbean.endDataLedgerOpenOp(); if (log.isDebugEnabled()) { log.debug("[{}] Opened source ledger {}", name, lastLedgerId); @@ -145,7 +144,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) BKException.getMessage(rc)); callback.initializeFailed(createManagedLedgerException(rc)); } - })); + }); //open ledger in readonly mode. bookKeeper.asyncOpenLedgerNoRecovery(lastLedgerId, digestType, config.getPassword(), opencb, null); @@ -317,7 +316,7 @@ private synchronized void processSourceManagedLedgerInfo(MLDataFormats.ManagedLe mbean.startDataLedgerOpenOp(); //open ledger in readonly mode. bookKeeper.asyncOpenLedgerNoRecovery(lastLedgerId, digestType, config.getPassword(), - (rc, lh, ctx1) -> executor.execute(safeRun(() -> { + (rc, lh, ctx1) -> executor.execute(() -> { mbean.endDataLedgerOpenOp(); if (log.isDebugEnabled()) { log.debug("[{}] Opened new source ledger {}", name, lastLedgerId); @@ -342,7 +341,7 @@ private synchronized void processSourceManagedLedgerInfo(MLDataFormats.ManagedLe log.error("[{}] Failed to open source ledger {}: {}", name, lastLedgerId, BKException.getMessage(rc)); } - })), null); + }), null); } //handle old ledgers deleted. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java index 080c70b5873cd..d5a3019855cb5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java @@ -18,7 +18,6 @@ */ package org.apache.bookkeeper.mledger.impl.cache; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import java.util.concurrent.ConcurrentHashMap; @@ -116,7 +115,7 @@ boolean hasSpaceInCache() { // Trigger a single eviction in background. While the eviction is running we stop inserting entries in the cache if (currentSize > evictionTriggerThreshold && evictionInProgress.compareAndSet(false, true)) { - mlFactory.getScheduledExecutor().execute(safeRun(() -> { + mlFactory.getScheduledExecutor().execute(() -> { // Trigger a new cache eviction cycle to bring the used memory below the cacheEvictionWatermark // percentage limit long sizeToEvict = currentSize - (long) (maxSize * cacheEvictionWatermark); @@ -136,7 +135,7 @@ boolean hasSpaceInCache() { mlFactoryMBean.recordCacheEviction(); evictionInProgress.set(false); } - })); + }); } return currentSize < maxSize; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java deleted file mode 100644 index 570cb7ae735ab..0000000000000 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bookkeeper.mledger.util; - -import java.util.function.Consumer; -import org.apache.bookkeeper.util.SafeRunnable; - -/** - * Static builders for {@link SafeRunnable}s. - */ -public class SafeRun { - public static SafeRunnable safeRun(Runnable runnable) { - return new SafeRunnable() { - @Override - public void safeRun() { - runnable.run(); - } - }; - } - - /** - * - * @param runnable - * @param exceptionHandler - * handler that will be called when there are any exception - * @return - */ - public static SafeRunnable safeRun(Runnable runnable, Consumer exceptionHandler) { - return new SafeRunnable() { - @Override - public void safeRun() { - try { - runnable.run(); - } catch (Throwable t) { - exceptionHandler.accept(t); - throw t; - } - } - }; - } -} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java index 3fa0234e13a55..7558f07db76ca 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java @@ -18,7 +18,6 @@ */ package org.apache.bookkeeper.mledger.impl; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; @@ -383,7 +382,7 @@ public void testConcurrentIndividualDeletesWithGetNthEntry() throws Exception { final AtomicInteger iteration = new AtomicInteger(0); for (int i = 0; i < deleteEntries; i++) { - executor.submit(safeRun(() -> { + executor.submit(() -> { try { cursor.asyncDelete(addedEntries.get(iteration.getAndIncrement()), new DeleteCallback() { @Override @@ -403,7 +402,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { } finally { counter.countDown(); } - })); + }); } counter.await(); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java index 136fe77d77aff..64d1fcdab6f14 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.stats.prometheus; -import static org.apache.bookkeeper.util.SafeRunnable.safeRun; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.EOFException; import java.io.IOException; @@ -61,7 +60,7 @@ public void init() throws ServletException { protected void doGet(HttpServletRequest request, HttpServletResponse response) { AsyncContext context = request.startAsync(); context.setTimeout(metricsServletTimeoutMs); - executor.execute(safeRun(() -> { + executor.execute(() -> { long start = System.currentTimeMillis(); HttpServletResponse res = (HttpServletResponse) context.getResponse(); try { @@ -92,7 +91,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { + "this is likely due to metricsServletTimeoutMs: {} ms elapsed: {}", time, e + ""); } } - })); + }); } protected void generateMetrics(String cluster, ServletOutputStream outputStream) throws IOException { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index a6345f4a56a71..bb08734298f04 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -20,7 +20,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.ManagedLedgerConfig.PROPERTY_SOURCE_TOPIC_KEY; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import static org.apache.commons.collections4.CollectionUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; @@ -56,7 +55,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -76,6 +74,7 @@ import lombok.Getter; import lombok.Setter; import org.apache.bookkeeper.common.util.OrderedExecutor; +import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback; import org.apache.bookkeeper.mledger.LedgerOffloader; @@ -324,8 +323,10 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.acceptorGroup = EventLoopUtil.newEventLoopGroup( pulsar.getConfiguration().getNumAcceptorThreads(), false, acceptorThreadFactory); this.workerGroup = eventLoopGroup; - this.statsUpdater = Executors.newSingleThreadScheduledExecutor( - new ExecutorProvider.ExtendedThreadFactory("pulsar-stats-updater")); + this.statsUpdater = OrderedScheduler.newSchedulerBuilder() + .name("pulsar-stats-updater") + .numThreads(1) + .build(); this.authorizationService = new AuthorizationService( pulsar.getConfiguration(), pulsar().getPulsarResources()); this.entryFilterProvider = new EntryFilterProvider(pulsar.getConfiguration()); @@ -333,22 +334,32 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws pulsar.getLocalMetadataStore().registerListener(this::handleMetadataChanges); pulsar.getConfigurationMetadataStore().registerListener(this::handleMetadataChanges); - this.inactivityMonitor = Executors.newSingleThreadScheduledExecutor( - new ExecutorProvider.ExtendedThreadFactory("pulsar-inactivity-monitor")); - this.messageExpiryMonitor = Executors.newSingleThreadScheduledExecutor( - new ExecutorProvider.ExtendedThreadFactory("pulsar-msg-expiry-monitor")); - this.compactionMonitor = - Executors.newSingleThreadScheduledExecutor( - new ExecutorProvider.ExtendedThreadFactory("pulsar-compaction-monitor")); - this.consumedLedgersMonitor = Executors.newSingleThreadScheduledExecutor( - new ExecutorProvider.ExtendedThreadFactory("consumed-Ledgers-monitor")); + + this.inactivityMonitor = OrderedScheduler.newSchedulerBuilder() + .name("pulsar-inactivity-monitor") + .numThreads(1) + .build(); + this.messageExpiryMonitor = OrderedScheduler.newSchedulerBuilder() + .name("pulsar-msg-expiry-monitor") + .numThreads(1) + .build(); + this.compactionMonitor = OrderedScheduler.newSchedulerBuilder() + .name("pulsar-compaction-monitor") + .numThreads(1) + .build(); + this.consumedLedgersMonitor = OrderedScheduler.newSchedulerBuilder() + .name("pulsar-consumed-ledgers-monitor") + .numThreads(1) + .build(); this.topicPublishRateLimiterMonitor = new PublishRateLimiterMonitor("pulsar-topic-publish-rate-limiter-monitor"); this.brokerPublishRateLimiterMonitor = new PublishRateLimiterMonitor("pulsar-broker-publish-rate-limiter-monitor"); this.backlogQuotaManager = new BacklogQuotaManager(pulsar); - this.backlogQuotaChecker = Executors.newSingleThreadScheduledExecutor( - new ExecutorProvider.ExtendedThreadFactory("pulsar-backlog-quota-checker")); + this.backlogQuotaChecker = OrderedScheduler.newSchedulerBuilder() + .name("pulsar-backlog-quota-checker") + .numThreads(1) + .build(); this.authenticationService = new AuthenticationService(pulsar.getConfiguration()); this.blockedDispatchers = ConcurrentOpenHashSet.newBuilder().build(); @@ -367,7 +378,7 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws log.info("Enabling per-broker unack-message limit {} and dispatcher-limit {} on blocked-broker", maxUnackedMessages, maxUnackedMsgsPerDispatcher); // block misbehaving dispatcher by checking periodically - pulsar.getExecutor().scheduleAtFixedRate(safeRun(this::checkUnAckMessageDispatching), + pulsar.getExecutor().scheduleAtFixedRate(this::checkUnAckMessageDispatching, 600, 30, TimeUnit.SECONDS); } else { this.maxUnackedMessages = 0; @@ -557,7 +568,7 @@ public void start() throws Exception { } protected void startStatsUpdater(int statsUpdateInitialDelayInSecs, int statsUpdateFrequencyInSecs) { - statsUpdater.scheduleAtFixedRate(safeRun(this::updateRates), + statsUpdater.scheduleAtFixedRate(this::updateRates, statsUpdateInitialDelayInSecs, statsUpdateFrequencyInSecs, TimeUnit.SECONDS); // Ensure the broker starts up with initial stats @@ -567,11 +578,12 @@ protected void startStatsUpdater(int statsUpdateInitialDelayInSecs, int statsUpd protected void startDeduplicationSnapshotMonitor() { int interval = pulsar().getConfiguration().getBrokerDeduplicationSnapshotFrequencyInSeconds(); if (interval > 0 && pulsar().getConfiguration().isBrokerDeduplicationEnabled()) { - this.deduplicationSnapshotMonitor = - Executors.newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory( - "deduplication-snapshot-monitor")); - deduplicationSnapshotMonitor.scheduleAtFixedRate(safeRun(() -> forEachTopic( - Topic::checkDeduplicationSnapshot)) + this.deduplicationSnapshotMonitor = OrderedScheduler.newSchedulerBuilder() + .name("deduplication-snapshot-monitor") + .numThreads(1) + .build(); + deduplicationSnapshotMonitor.scheduleAtFixedRate(() -> forEachTopic( + Topic::checkDeduplicationSnapshot) , interval, interval, TimeUnit.SECONDS); } } @@ -579,14 +591,14 @@ protected void startDeduplicationSnapshotMonitor() { protected void startInactivityMonitor() { if (pulsar().getConfiguration().isBrokerDeleteInactiveTopicsEnabled()) { int interval = pulsar().getConfiguration().getBrokerDeleteInactiveTopicsFrequencySeconds(); - inactivityMonitor.scheduleAtFixedRate(safeRun(() -> checkGC()), interval, interval, + inactivityMonitor.scheduleAtFixedRate(() -> checkGC(), interval, interval, TimeUnit.SECONDS); } // Deduplication info checker long duplicationCheckerIntervalInSeconds = TimeUnit.MINUTES .toSeconds(pulsar().getConfiguration().getBrokerDeduplicationProducerInactivityTimeoutMinutes()) / 3; - inactivityMonitor.scheduleAtFixedRate(safeRun(this::checkMessageDeduplicationInfo), + inactivityMonitor.scheduleAtFixedRate(this::checkMessageDeduplicationInfo, duplicationCheckerIntervalInSeconds, duplicationCheckerIntervalInSeconds, TimeUnit.SECONDS); @@ -595,7 +607,7 @@ protected void startInactivityMonitor() { long subscriptionExpiryCheckIntervalInSeconds = TimeUnit.MINUTES.toSeconds(pulsar().getConfiguration() .getSubscriptionExpiryCheckIntervalInMinutes()); - inactivityMonitor.scheduleAtFixedRate(safeRun(this::checkInactiveSubscriptions), + inactivityMonitor.scheduleAtFixedRate(this::checkInactiveSubscriptions, subscriptionExpiryCheckIntervalInSeconds, subscriptionExpiryCheckIntervalInSeconds, TimeUnit.SECONDS); } @@ -603,21 +615,21 @@ protected void startInactivityMonitor() { // check cluster migration int interval = pulsar().getConfiguration().getClusterMigrationCheckDurationSeconds(); if (interval > 0) { - inactivityMonitor.scheduleAtFixedRate(safeRun(() -> checkClusterMigration()), interval, interval, + inactivityMonitor.scheduleAtFixedRate(() -> checkClusterMigration(), interval, interval, TimeUnit.SECONDS); } } protected void startMessageExpiryMonitor() { int interval = pulsar().getConfiguration().getMessageExpiryCheckIntervalInMinutes(); - messageExpiryMonitor.scheduleAtFixedRate(safeRun(this::checkMessageExpiry), interval, interval, + messageExpiryMonitor.scheduleAtFixedRate(this::checkMessageExpiry, interval, interval, TimeUnit.MINUTES); } protected void startCheckReplicationPolicies() { int interval = pulsar.getConfig().getReplicationPolicyCheckDurationSeconds(); if (interval > 0) { - messageExpiryMonitor.scheduleAtFixedRate(safeRun(this::checkReplicationPolicies), interval, interval, + messageExpiryMonitor.scheduleAtFixedRate(this::checkReplicationPolicies, interval, interval, TimeUnit.SECONDS); } } @@ -625,16 +637,16 @@ protected void startCheckReplicationPolicies() { protected void startCompactionMonitor() { int interval = pulsar().getConfiguration().getBrokerServiceCompactionMonitorIntervalInSeconds(); if (interval > 0) { - compactionMonitor.scheduleAtFixedRate(safeRun(() -> checkCompaction()), - interval, interval, TimeUnit.SECONDS); + compactionMonitor.scheduleAtFixedRate(this::checkCompaction, + interval, interval, TimeUnit.SECONDS); } } protected void startConsumedLedgersMonitor() { int interval = pulsar().getConfiguration().getRetentionCheckIntervalInSeconds(); if (interval > 0) { - consumedLedgersMonitor.scheduleAtFixedRate(safeRun(this::checkConsumedLedgers), - interval, interval, TimeUnit.SECONDS); + consumedLedgersMonitor.scheduleAtFixedRate(this::checkConsumedLedgers, + interval, interval, TimeUnit.SECONDS); } } @@ -642,7 +654,7 @@ protected void startBacklogQuotaChecker() { if (pulsar().getConfiguration().isBacklogQuotaCheckEnabled()) { final int interval = pulsar().getConfiguration().getBacklogQuotaCheckIntervalInSeconds(); log.info("Scheduling a thread to check backlog quota after [{}] seconds in background", interval); - backlogQuotaChecker.scheduleAtFixedRate(safeRun(this::monitorBacklogQuota), interval, interval, + backlogQuotaChecker.scheduleAtFixedRate(this::monitorBacklogQuota, interval, interval, TimeUnit.SECONDS); } else { log.info("Backlog quota check monitoring is disabled"); @@ -702,12 +714,15 @@ synchronized void startOrUpdate(long tickTimeMs, Runnable checkTask, Runnable re stop(); } //start monitor. - scheduler = Executors.newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory(name)); + scheduler = OrderedScheduler.newSchedulerBuilder() + .name(name) + .numThreads(1) + .build(); // schedule task that sums up publish-rate across all cnx on a topic , // and check the rate limit exceeded or not. - scheduler.scheduleAtFixedRate(safeRun(checkTask), tickTimeMs, tickTimeMs, TimeUnit.MILLISECONDS); + scheduler.scheduleAtFixedRate(checkTask, tickTimeMs, tickTimeMs, TimeUnit.MILLISECONDS); // schedule task that refreshes rate-limiting bucket - scheduler.scheduleAtFixedRate(safeRun(refreshTask), 1, 1, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(refreshTask, 1, 1, TimeUnit.SECONDS); this.tickTimeMs = tickTimeMs; this.refreshTask = refreshTask; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 677ed25ef2163..bfd7c001e4cfb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -67,7 +67,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.util.SafeRun; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.mutable.MutableInt; @@ -1674,9 +1673,9 @@ protected void handleSend(CommandSend send, ByteBuf headersAndPayload) { final long producerId = send.getProducerId(); final long sequenceId = send.getSequenceId(); final long highestSequenceId = send.getHighestSequenceId(); - service.getTopicOrderedExecutor().executeOrdered(producer.getTopic().getName(), SafeRun.safeRun(() -> { + service.getTopicOrderedExecutor().executeOrdered(producer.getTopic().getName(), () -> { commandSender.sendSendReceiptResponse(producerId, sequenceId, highestSequenceId, -1, -1); - })); + }); producer.recordMessageDrop(send.getNumMessages()); return; } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 9540f6efe0005..4ac755860fc7b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.persistent; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; import com.google.common.collect.Lists; import com.google.common.collect.Range; @@ -236,9 +235,9 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE @Override public void consumerFlow(Consumer consumer, int additionalNumberOfMessages) { - topic.getBrokerService().executor().execute(safeRun(() -> { + topic.getBrokerService().executor().execute(() -> { internalConsumerFlow(consumer, additionalNumberOfMessages); - })); + }); } private synchronized void internalConsumerFlow(Consumer consumer, int additionalNumberOfMessages) { @@ -264,7 +263,7 @@ private synchronized void internalConsumerFlow(Consumer consumer, int additional * */ public void readMoreEntriesAsync() { - topic.getBrokerService().executor().execute(safeRun(this::readMoreEntries)); + topic.getBrokerService().executor().execute(this::readMoreEntries); } public synchronized void readMoreEntries() { @@ -584,14 +583,14 @@ public final synchronized void readEntriesComplete(List entries, Object c // setting sendInProgress here, because sendMessagesToConsumers will be executed // in a separate thread, and we want to prevent more reads acquireSendInProgress(); - dispatchMessagesThread.execute(safeRun(() -> { + dispatchMessagesThread.execute(() -> { if (sendMessagesToConsumers(readType, entries, false)) { updatePendingBytesToDispatch(-size); readMoreEntries(); } else { updatePendingBytesToDispatch(-size); } - })); + }); } else { if (sendMessagesToConsumers(readType, entries, true)) { updatePendingBytesToDispatch(-size); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 03dce58af3a92..7cbf7bd2c787a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -38,7 +38,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.util.SafeRun; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.Consumer; @@ -149,9 +148,7 @@ protected void cancelPendingRead() { @Override public void readEntriesComplete(final List entries, Object obj) { - topicExecutor.execute(SafeRun.safeRun(() -> { - internalReadEntriesComplete(entries, obj); - })); + topicExecutor.execute(() -> internalReadEntriesComplete(entries, obj)); } public synchronized void internalReadEntriesComplete(final List entries, Object obj) { @@ -229,21 +226,19 @@ protected void dispatchEntriesToConsumer(Consumer currentConsumer, List e sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes()); // Schedule a new read batch operation only after the previous batch has been written to the socket. - topicExecutor.execute(SafeRun.safeRun(() -> { + topicExecutor.execute(() -> { synchronized (PersistentDispatcherSingleActiveConsumer.this) { Consumer newConsumer = getActiveConsumer(); readMoreEntries(newConsumer); } - })); + }); } }); } @Override public void consumerFlow(Consumer consumer, int additionalNumberOfMessages) { - topicExecutor.execute(SafeRun.safeRun(() -> { - internalConsumerFlow(consumer); - })); + topicExecutor.execute(() -> internalConsumerFlow(consumer)); } private synchronized void internalConsumerFlow(Consumer consumer) { @@ -272,9 +267,7 @@ private synchronized void internalConsumerFlow(Consumer consumer) { @Override public void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) { - topicExecutor.execute(SafeRun.safeRun(() -> { - internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch); - })); + topicExecutor.execute(() -> internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch)); } private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) { @@ -466,9 +459,7 @@ protected Pair calculateToRead(Consumer consumer) { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - topicExecutor.execute(SafeRun.safeRun(() -> { - internalReadEntriesFailed(exception, ctx); - })); + topicExecutor.execute(() -> internalReadEntriesFailed(exception, ctx)); } private synchronized void internalReadEntriesFailed(ManagedLedgerException exception, Object ctx) { @@ -516,7 +507,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep topic.getBrokerService().executor().schedule(() -> { // Jump again into dispatcher dedicated thread - topicExecutor.execute(SafeRun.safeRun(() -> { + topicExecutor.execute(() -> { synchronized (PersistentDispatcherSingleActiveConsumer.this) { Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); // we should retry the read if we have an active consumer and there is no pending read @@ -533,7 +524,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep currentConsumer, havePendingRead); } } - })); + }); }, waitTimeMillis, TimeUnit.MILLISECONDS); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index a0292729e734f..1a8c6e180a2a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.persistent; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; @@ -415,7 +414,7 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List en public void markDeletePositionMoveForward() { // Execute the notification in different thread to avoid a mutex chain here // from the delete operation that was completed - topic.getBrokerService().getTopicOrderedExecutor().execute(safeRun(() -> { + topic.getBrokerService().getTopicOrderedExecutor().execute(() -> { synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) { if (recentlyJoinedConsumers != null && !recentlyJoinedConsumers.isEmpty() && removeConsumersFromRecentJoinedConsumers()) { @@ -424,7 +423,7 @@ && removeConsumersFromRecentJoinedConsumers()) { readMoreEntries(); } } - })); + }); } private boolean removeConsumersFromRecentJoinedConsumers() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java index 5df1fc2c6db26..44e8a423344cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.persistent; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import com.google.common.collect.Lists; import java.util.Set; import java.util.concurrent.Executor; @@ -30,7 +29,6 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.util.SafeRun; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.streamingdispatch.PendingReadEntryRequest; @@ -105,13 +103,13 @@ public synchronized void readEntryComplete(Entry entry, PendingReadEntryRequest // setting sendInProgress here, because sendMessagesToConsumers will be executed // in a separate thread, and we want to prevent more reads acquireSendInProgress(); - dispatchMessagesThread.execute(safeRun(() -> { + dispatchMessagesThread.execute(() -> { if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), false)) { readMoreEntries(); } else { updatePendingBytesToDispatch(-size); } - })); + }); } else { if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), true)) { readMoreEntriesAsync(); @@ -129,7 +127,7 @@ public synchronized void readEntryComplete(Entry entry, PendingReadEntryRequest public void canReadMoreEntries(boolean withBackoff) { havePendingRead = false; topic.getBrokerService().executor().schedule(() -> { - topicExecutor.execute(SafeRun.safeRun(() -> { + topicExecutor.execute(() -> { synchronized (PersistentStreamingDispatcherMultipleConsumers.this) { if (!havePendingRead) { log.info("[{}] Scheduling read operation", name); @@ -138,7 +136,7 @@ public void canReadMoreEntries(boolean withBackoff) { log.info("[{}] Skipping read since we have pendingRead", name); } } - })); + }); }, withBackoff ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS); } @@ -216,7 +214,7 @@ public synchronized void readMoreEntries() { havePendingReplayRead = false; // We should not call readMoreEntries() recursively in the same thread // as there is a risk of StackOverflowError - topic.getBrokerService().executor().execute(safeRun(this::readMoreEntries)); + topic.getBrokerService().executor().execute(this::readMoreEntries); } } else if (BLOCKED_DISPATCHER_ON_UNACKMSG_UPDATER.get(this) == TRUE) { log.debug("[{}] Dispatcher read is blocked due to unackMessages {} reached to max {}", name, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java index 9000850ed69ed..efe9de778a3e7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.persistent; -import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import com.google.common.collect.Lists; import java.util.concurrent.Executor; @@ -29,7 +28,6 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.util.SafeRun; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.EntryBatchIndexesAcks; @@ -68,7 +66,7 @@ public PersistentStreamingDispatcherSingleActiveConsumer(ManagedCursor cursor, S public void canReadMoreEntries(boolean withBackoff) { havePendingRead = false; topic.getBrokerService().executor().schedule(() -> { - topicExecutor.execute(SafeRun.safeRun(() -> { + topicExecutor.execute(() -> { synchronized (PersistentStreamingDispatcherSingleActiveConsumer.this) { Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); if (currentConsumer != null && !havePendingRead) { @@ -81,7 +79,7 @@ public void canReadMoreEntries(boolean withBackoff) { currentConsumer, havePendingRead); } } - })); + }); }, withBackoff ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS); } @@ -115,9 +113,7 @@ public String getName() { */ @Override public void readEntryComplete(Entry entry, PendingReadEntryRequest ctx) { - dispatcherExecutor.execute(safeRun(() -> { - internalReadEntryComplete(entry, ctx); - })); + dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx)); } public synchronized void internalReadEntryComplete(Entry entry, PendingReadEntryRequest ctx) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java index 0f75538ee88f0..6ffc5ba0f623a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java @@ -35,7 +35,6 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.util.SafeRun; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.impl.Backoff; @@ -157,9 +156,7 @@ public synchronized void asyncReadEntries(int numEntriesToRead, long maxReadSize @Override public void readEntryComplete(Entry entry, Object ctx) { // Don't block caller thread, complete read entry with dispatcher dedicated thread. - dispatcherExecutor.execute(SafeRun.safeRun(() -> { - internalReadEntryComplete(entry, ctx); - })); + dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx)); } private void internalReadEntryComplete(Entry entry, Object ctx) { @@ -198,9 +195,7 @@ private void internalReadEntryComplete(Entry entry, Object ctx) { @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { // Don't block caller thread, complete read entry fail with dispatcher dedicated thread. - dispatcherExecutor.execute(SafeRun.safeRun(() -> { - internalReadEntryFailed(exception, ctx); - })); + dispatcherExecutor.execute(() -> internalReadEntryFailed(exception, ctx)); } private void internalReadEntryFailed(ManagedLedgerException exception, Object ctx) { @@ -257,13 +252,13 @@ private void internalCancelReadRequests() { public boolean cancelReadRequests() { if (STATE_UPDATER.compareAndSet(this, State.Issued, State.Canceling)) { // Don't block caller thread, complete cancel read with dispatcher dedicated thread. - topicExecutor.execute(SafeRun.safeRun(() -> { + topicExecutor.execute(() -> { synchronized (StreamingEntryReader.this) { if (STATE_UPDATER.compareAndSet(this, State.Canceling, State.Canceled)) { internalCancelReadRequests(); } } - })); + }); return true; } return false; @@ -282,16 +277,16 @@ private void cleanQueue(Queue queue) { private void retryReadRequest(PendingReadEntryRequest pendingReadEntryRequest, long delay) { topic.getBrokerService().executor().schedule(() -> { // Jump again into dispatcher dedicated thread - dispatcherExecutor.execute(SafeRun.safeRun(() -> { + dispatcherExecutor.execute(() -> { ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger(); managedLedger.asyncReadEntry(pendingReadEntryRequest.position, this, pendingReadEntryRequest); - })); + }); }, delay, TimeUnit.MILLISECONDS); } @Override public void entriesAvailable() { - dispatcherExecutor.execute(SafeRun.safeRun(this::internalEntriesAvailable)); + dispatcherExecutor.execute(this::internalEntriesAvailable); } private synchronized void internalEntriesAvailable() { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index 11ae62226e7cd..9e6a9b94c42a3 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -30,7 +30,6 @@ import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.concurrent.FutureUtils; -import org.apache.bookkeeper.common.util.SafeRunnable; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataCache; @@ -89,7 +88,7 @@ private enum InternalState { this.sequencer = FutureUtil.Sequencer.create(); store.registerListener(this::handlePathNotification); store.registerSessionListener(this::handleSessionNotification); - updateCachedValueFuture = executor.scheduleWithFixedDelay(SafeRunnable.safeRun(this::getLeaderValue), + updateCachedValueFuture = executor.scheduleWithFixedDelay(this::getLeaderValue, metadataCacheConfig.getRefreshAfterWriteMillis() / 2, metadataCacheConfig.getRefreshAfterWriteMillis(), TimeUnit.MILLISECONDS); } From a91865140893f7e9737f6ce9ffc052584c721884 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 11 Apr 2023 10:25:57 +0800 Subject: [PATCH 084/280] [improve][client] PIP-224: Add getLastMessageIds API (#20040) Co-authored-by: Baodi Shi --- .../pulsar/client/api/TopicReaderTest.java | 6 +++ .../client/impl/TopicsConsumerImplTest.java | 37 +++++++++++++++++++ .../apache/pulsar/client/api/Consumer.java | 21 +++++++++++ .../pulsar/client/impl/ConsumerBase.java | 15 ++++++++ .../pulsar/client/impl/ConsumerImpl.java | 7 ++++ .../client/impl/MultiMessageIdImpl.java | 1 + .../client/impl/MultiTopicsConsumerImpl.java | 13 +++++++ 7 files changed, 100 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java index c2eb957ee605d..424081b904c81 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java @@ -1096,6 +1096,12 @@ public void testHasMessageAvailable() throws Exception { assertFalse(lastMsgId instanceof BatchMessageIdImpl); assertEquals(lastMsgId.getLedgerId(), messageId.getLedgerId()); assertEquals(lastMsgId.getEntryId(), messageId.getEntryId()); + List lastMsgIds = reader.getConsumer().getLastMessageIds(); + assertEquals(lastMsgIds.size(), 1); + assertEquals(lastMsgIds.get(0).getOwnerTopic(), topicName); + MessageIdAdv lastMsgIdAdv = (MessageIdAdv) lastMsgIds.get(0); + assertEquals(lastMsgIdAdv.getLedgerId(), messageId.getLedgerId()); + assertEquals(lastMsgIdAdv.getEntryId(), messageId.getEntryId()); reader.close(); CountDownLatch latch = new CountDownLatch(numOfMessage); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java index ce4a0ae86ac4e..73fe97996424c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java @@ -33,6 +33,7 @@ import org.apache.pulsar.client.api.ConsumerEventListener; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageRouter; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -42,7 +43,9 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.TopicMetadata; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PartitionedTopicStats; import org.apache.pulsar.common.policies.data.SubscriptionStats; @@ -1097,6 +1100,11 @@ public void testGetLastMessageId() throws Exception { admin.topics().createPartitionedTopic(topicName2, 2); admin.topics().createPartitionedTopic(topicName3, 3); + final Set topics = new HashSet<>(); + topics.add(topicName1); + IntStream.range(0, 2).forEach(i -> topics.add(topicName2 + TopicName.PARTITIONED_TOPIC_SUFFIX + i)); + IntStream.range(0, 3).forEach(i -> topics.add(topicName3 + TopicName.PARTITIONED_TOPIC_SUFFIX + i)); + // 1. producer connect Producer producer1 = pulsarClient.newProducer().topic(topicName1) .enableBatching(false) @@ -1146,12 +1154,27 @@ public void testGetLastMessageId() throws Exception { } }); + List msgIds = consumer.getLastMessageIds(); + assertEquals(msgIds.size(), 6); + assertEquals(msgIds.stream().map(TopicMessageId::getOwnerTopic).collect(Collectors.toSet()), topics); + for (TopicMessageId msgId : msgIds) { + int numMessages = (int) ((MessageIdAdv) msgId).getEntryId() + 1; + if (msgId.getOwnerTopic().equals(topicName1)) { + assertEquals(numMessages, totalMessages); + } else if (msgId.getOwnerTopic().startsWith(topicName2)) { + assertEquals(numMessages, totalMessages / 2); + } else { + assertEquals(numMessages, totalMessages / 3); + } + } + for (int i = 0; i < totalMessages; i++) { producer1.send((messagePredicate + "producer1-" + i).getBytes()); producer2.send((messagePredicate + "producer2-" + i).getBytes()); producer3.send((messagePredicate + "producer3-" + i).getBytes()); } + messageId = consumer.getLastMessageId(); assertTrue(messageId instanceof MultiMessageIdImpl); MultiMessageIdImpl multiMessageId2 = (MultiMessageIdImpl) messageId; @@ -1170,6 +1193,20 @@ public void testGetLastMessageId() throws Exception { } }); + msgIds = consumer.getLastMessageIds(); + assertEquals(msgIds.size(), 6); + assertEquals(msgIds.stream().map(TopicMessageId::getOwnerTopic).collect(Collectors.toSet()), topics); + for (TopicMessageId msgId : msgIds) { + int numMessages = (int) ((MessageIdAdv) msgId).getEntryId() + 1; + if (msgId.getOwnerTopic().equals(topicName1)) { + assertEquals(numMessages, totalMessages * 2); + } else if (msgId.getOwnerTopic().startsWith(topicName2)) { + assertEquals(numMessages, totalMessages); + } else { + assertEquals(numMessages, totalMessages / 3 * 2); + } + } + consumer.unsubscribe(); consumer.close(); producer1.close(); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java index 694099004965a..88ad24fe1f484 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.api; import java.io.Closeable; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -539,16 +540,36 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message, * Get the last message id available for consume. * * @return the last message id. + * @apiNote If the consumer is a multi-topics consumer, the returned value cannot be used anywhere. + * @deprecated Use {@link Consumer#getLastMessageIds()} instead. */ + @Deprecated MessageId getLastMessageId() throws PulsarClientException; /** * Get the last message id available for consume. * * @return a future that can be used to track the completion of the operation. + * @deprecated Use {@link Consumer#getLastMessageIdsAsync()}} instead. */ + @Deprecated CompletableFuture getLastMessageIdAsync(); + /** + * Get all the last message id of the topics the consumer subscribed. + * + * @return the list of TopicMessageId instances of all the topics that the consumer subscribed + * @throws PulsarClientException if failed to get last message id. + * @apiNote It's guaranteed that the owner topic of each TopicMessageId in the returned list is different from owner + * topics of other TopicMessageId instances + */ + List getLastMessageIds() throws PulsarClientException; + + /** + * The asynchronous version of {@link Consumer#getLastMessageIds()}. + */ + CompletableFuture> getLastMessageIdsAsync(); + /** * @return Whether the consumer is connected to the broker */ diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 973b3302f4199..0db2a8e0ab9f5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -53,6 +53,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.transaction.TransactionImpl; @@ -730,6 +731,7 @@ public void close() throws PulsarClientException { public abstract CompletableFuture closeAsync(); + @Deprecated @Override public MessageId getLastMessageId() throws PulsarClientException { try { @@ -742,9 +744,22 @@ public MessageId getLastMessageId() throws PulsarClientException { } } + @Deprecated @Override public abstract CompletableFuture getLastMessageIdAsync(); + @Override + public List getLastMessageIds() throws PulsarClientException { + try { + return getLastMessageIdsAsync().get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw PulsarClientException.unwrap(e); + } catch (ExecutionException e) { + throw PulsarClientException.unwrap(e); + } + } + private boolean isCumulativeAcknowledgementAllowed(SubscriptionType type) { return SubscriptionType.Shared != type && SubscriptionType.Key_Shared != type; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index fb372566426d3..cc01609319698 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2336,11 +2336,18 @@ private static final class GetLastMessageIdResponse { } } + @Deprecated @Override public CompletableFuture getLastMessageIdAsync() { return internalGetLastMessageIdAsync().thenApply(r -> r.lastMessageId); } + @Override + public CompletableFuture> getLastMessageIdsAsync() { + return getLastMessageIdAsync() + .thenApply(msgId -> Collections.singletonList(TopicMessageId.create(topic, msgId))); + } + public CompletableFuture internalGetLastMessageIdAsync() { if (getState() == State.Closing || getState() == State.Closed) { return FutureUtil diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java index 6e60239ffe537..f40e3476dd0e0 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java @@ -29,6 +29,7 @@ * This is useful when MessageId is need for partition/multi-topics/pattern consumer. * e.g. seek(), ackCumulative(), getLastMessageId(). */ +@Deprecated public class MultiMessageIdImpl implements MessageId { @Getter private Map map; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 5fe0e4a82b840..ef0345de919ea 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -1468,6 +1468,7 @@ public Timeout getPartitionsAutoUpdateTimeout() { return partitionsAutoUpdateTimeout; } + @Deprecated @Override public CompletableFuture getLastMessageIdAsync() { CompletableFuture returnFuture = new CompletableFuture<>(); @@ -1496,6 +1497,18 @@ public CompletableFuture getLastMessageIdAsync() { return returnFuture; } + @Override + public CompletableFuture> getLastMessageIdsAsync() { + final List>> futures = consumers.values().stream() + .map(ConsumerImpl::getLastMessageIdsAsync) + .collect(Collectors.toList()); + return FutureUtil.waitForAll(futures).thenApply(__ -> { + final List messageIds = new ArrayList<>(); + futures.stream().map(CompletableFuture::join).forEach(messageIds::addAll); + return messageIds; + }); + } + private static final Logger log = LoggerFactory.getLogger(MultiTopicsConsumerImpl.class); public static boolean isIllegalMultiTopicsMessageId(MessageId messageId) { From 25dea3dc9dd6f5c2956cb7b9376450515f225a6b Mon Sep 17 00:00:00 2001 From: Mathieu Grenonville Date: Tue, 11 Apr 2023 04:52:11 +0200 Subject: [PATCH 085/280] [fix] Use scheduled executor in BinaryProtoLookupService (#20043) Motivation: PulsarClient using ExecutorService as ScheduledExecutorService Modifications: Use exactly ScheduledExecutorService --- .../java/org/apache/pulsar/client/impl/PulsarClientImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index f37709f3d84f6..6c749a8cf4354 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -205,7 +205,7 @@ private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopG lookup = new HttpLookupService(conf, this.eventLoopGroup); } else { lookup = new BinaryProtoLookupService(this, conf.getServiceUrl(), conf.getListenerName(), - conf.isUseTls(), this.externalExecutorProvider.getExecutor()); + conf.isUseTls(), this.scheduledExecutorProvider.getExecutor()); } if (timer == null) { this.timer = new HashedWheelTimer(getThreadFactory("pulsar-timer"), 1, TimeUnit.MILLISECONDS); From b7f7e040b6ad27827ba5aa31bf51be621d506560 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 11 Apr 2023 14:18:51 +0800 Subject: [PATCH 086/280] [fix] [admin] PIP-259: Admin API can not work if uri too large (#19514) Motivation: If automatic topic creation is enabled, producers and consumers can create topics with long names, but once created, such topics cannot be managed by the Admin API, it will respond "414 URI is too large". Modifications: add a config `pulsar.conf.httpMaxRequestHeaderSize` to make the jetty can accept the request with a longer Uri --- conf/broker.conf | 5 ++ conf/proxy.conf | 5 ++ .../pulsar/broker/ServiceConfiguration.java | 11 ++++ .../apache/pulsar/broker/web/WebService.java | 8 ++- .../naming/ServiceConfigurationTest.java | 1 + .../configurations/pulsar_broker_test.conf | 1 + .../proxy/server/AdminProxyHandler.java | 4 +- .../proxy/server/ProxyConfiguration.java | 12 ++++ .../apache/pulsar/proxy/server/WebServer.java | 19 +++++- .../server/AuthedAdminProxyHandlerTest.java | 28 +++++++++ .../proxy/server/ProxyConfigurationTest.java | 2 + .../proxy/server/ProxyIsAHttpProxyTest.java | 59 ++++++++++++++++++- 12 files changed, 150 insertions(+), 5 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 414756abcb7cb..292b1ef1683b7 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -903,6 +903,11 @@ saslJaasServerRoleTokenSignerSecretPath= # If >0, it will reject all HTTP requests with bodies larged than the configured limit httpMaxRequestSize=-1 +# The maximum size in bytes of the request header. Larger headers will allow for more and/or larger cookies plus larger +# form content encoded in a URL.However, larger headers consume more memory and can make a server more vulnerable to +# denial of service attacks. +httpMaxRequestHeaderSize = 8192 + # If true, the broker will reject all HTTP requests using the TRACE and TRACK verbs. # This setting may be necessary if the broker is deployed into an environment that uses http port # scanning and flags web servers allowing the TRACE method as insecure. diff --git a/conf/proxy.conf b/conf/proxy.conf index a5110ae57471a..a33db8bbf7a30 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -277,6 +277,11 @@ maxHttpServerConnections=2048 # Max concurrent web requests maxConcurrentHttpRequests=1024 +# The maximum size in bytes of the request header. Larger headers will allow for more and/or larger cookies plus larger +# form content encoded in a URL.However, larger headers consume more memory and can make a server more vulnerable to +# denial of service attacks. +httpMaxRequestHeaderSize = 8192 + ## Configure the datasource of basic authenticate, supports the file and Base64 format. # file: # basicAuthConf=/path/my/.htpasswd diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index cea159b74fd1d..b31c86560f848 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1584,6 +1584,17 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private long httpMaxRequestSize = -1; + @FieldContext( + category = CATEGORY_HTTP, + doc = """ + The maximum size in bytes of the request header. + Larger headers will allow for more and/or larger cookies plus larger form content encoded in a URL. + However, larger headers consume more memory and can make a server more vulnerable to denial of service + attacks. + """ + ) + private int httpMaxRequestHeaderSize = 8 * 1024; + @FieldContext( category = CATEGORY_HTTP, doc = "If true, the broker will reject all HTTP requests using the TRACE and TRACK verbs.\n" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java index 96e7c9d556b71..2d6a6af58477e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java @@ -33,6 +33,8 @@ import org.apache.pulsar.jetty.tls.JettySslContextFactory; import org.eclipse.jetty.server.ConnectionLimit; import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.ContextHandler; @@ -100,8 +102,10 @@ public WebService(PulsarService pulsar) throws PulsarServerException { List connectors = new ArrayList<>(); Optional port = config.getWebServicePort(); + HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setRequestHeaderSize(pulsar.getConfig().getHttpMaxRequestHeaderSize()); if (port.isPresent()) { - httpConnector = new ServerConnector(server); + httpConnector = new ServerConnector(server, new HttpConnectionFactory(httpConfig)); httpConnector.setPort(port.get()); httpConnector.setHost(pulsar.getBindAddress()); connectors.add(httpConnector); @@ -140,7 +144,7 @@ public WebService(PulsarService pulsar) throws PulsarServerException { config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec()); } - httpsConnector = new ServerConnector(server, sslCtxFactory); + httpsConnector = new ServerConnector(server, sslCtxFactory, new HttpConnectionFactory(httpConfig)); httpsConnector.setPort(tlsPort.get()); httpsConnector.setHost(pulsar.getBindAddress()); connectors.add(httpsConnector); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java index 2142900194731..55971c15adf68 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java @@ -72,6 +72,7 @@ public void testInit() throws Exception { assertEquals(config.getMaxMessagePublishBufferSizeInMB(), -1); assertEquals(config.getManagedLedgerDataReadPriority(), "bookkeeper-first"); assertEquals(config.getBacklogQuotaDefaultLimitGB(), 0.05); + assertEquals(config.getHttpMaxRequestHeaderSize(), 1234); OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create(config.getProperties()); assertEquals(offloadPolicies.getManagedLedgerOffloadedReadPriority().getValue(), "bookkeeper-first"); } diff --git a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf index 226b2f31a738b..bfbbfb7487c42 100644 --- a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf +++ b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf @@ -24,6 +24,7 @@ brokerServicePort=6650 brokerServicePortTls=6651 webServicePort=8080 webServicePortTls=4443 +httpMaxRequestHeaderSize=1234 bindAddress=0.0.0.0 advertisedAddress= clusterName="test_cluster" diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index e978f7c1ad5b1..d9dda9823ea89 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -65,6 +65,8 @@ class AdminProxyHandler extends ProxyServlet { private static final String ORIGINAL_PRINCIPAL_HEADER = "X-Original-Principal"; + public static final String INIT_PARAM_REQUEST_BUFFER_SIZE = "requestBufferSize"; + private static final Set functionRoutes = new HashSet<>(Arrays.asList( "/admin/v3/function", "/admin/v2/function", @@ -140,7 +142,7 @@ protected HttpClient createHttpClient() throws ServletException { } client.setIdleTimeout(Long.parseLong(value)); - value = config.getInitParameter("requestBufferSize"); + value = config.getInitParameter(INIT_PARAM_REQUEST_BUFFER_SIZE); if (value != null) { client.setRequestBufferSize(Integer.parseInt(value)); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index a91b6e70f5b8b..52e50a7e6b87b 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -661,6 +661,18 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private int httpOutputBufferSize = 32 * 1024; + @FieldContext( + minValue = 1, + category = CATEGORY_HTTP, + doc = """ + The maximum size in bytes of the request header. + Larger headers will allow for more and/or larger cookies plus larger form content encoded in a URL. + However, larger headers consume more memory and can make a server more vulnerable to denial of service + attacks. + """ + ) + private int httpMaxRequestHeaderSize = 8 * 1024; + @FieldContext( minValue = 1, category = CATEGORY_HTTP, diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java index 70799aa48f259..1ca8dc93ebf9e 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.proxy.server; +import static org.apache.pulsar.proxy.server.AdminProxyHandler.INIT_PARAM_REQUEST_BUFFER_SIZE; import io.prometheus.client.jetty.JettyStatisticsCollector; import java.io.IOException; import java.net.URI; @@ -93,6 +94,7 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication HttpConfiguration httpConfig = new HttpConfiguration(); httpConfig.setOutputBufferSize(config.getHttpOutputBufferSize()); + httpConfig.setRequestHeaderSize(config.getHttpMaxRequestHeaderSize()); if (config.getWebServicePort().isPresent()) { this.externalServicePort = config.getWebServicePort().get(); @@ -131,7 +133,7 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec()); } - connectorTls = new ServerConnector(server, sslCtxFactory); + connectorTls = new ServerConnector(server, sslCtxFactory, new HttpConnectionFactory(httpConfig)); connectorTls.setPort(config.getWebServicePortTls().get()); connectorTls.setHost(config.getBindAddress()); connectors.add(connectorTls); @@ -195,6 +197,8 @@ public void addServlet(String basePath, ServletHolder servletHolder, List> attributes, boolean requireAuthentication) { + popularServletParams(servletHolder, config); + Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst(); if (existingPath.isPresent()) { throw new IllegalArgumentException( @@ -214,6 +218,19 @@ public void addServlet(String basePath, ServletHolder servletHolder, handlers.add(context); } + private static void popularServletParams(ServletHolder servletHolder, ProxyConfiguration config) { + int requestBufferSize = -1; + try { + requestBufferSize = Integer.parseInt(servletHolder.getInitParameter(INIT_PARAM_REQUEST_BUFFER_SIZE)); + } catch (NumberFormatException nfe){ + log.warn("The init-param {} is invalidated, because it is not a number", INIT_PARAM_REQUEST_BUFFER_SIZE); + } + if (requestBufferSize > 0 || config.getHttpMaxRequestHeaderSize() > 0) { + int v = Math.max(requestBufferSize, config.getHttpMaxRequestHeaderSize()); + servletHolder.setInitParameter(INIT_PARAM_REQUEST_BUFFER_SIZE, String.valueOf(v)); + } + } + public void addRestResource(String basePath, String attribute, Object attributeValue, Class resourceClass) { ResourceConfig config = new ResourceConfig(); config.register(resourceClass); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java index 7abee46aea61d..de168284ab47f 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java @@ -75,6 +75,7 @@ protected void setup() throws Exception { conf.setProxyRoles(ImmutableSet.of("proxy")); conf.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName())); conf.setNumExecutorThreadPoolSize(5); + conf.setHttpMaxRequestHeaderSize(20000); super.internalSetup(); @@ -87,6 +88,7 @@ protected void setup() throws Exception { proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setHttpMaxRequestHeaderSize(20000); // enable tls and auth&auth at proxy proxyConfig.setTlsCertificateFilePath(getTlsFile("broker.cert")); @@ -185,4 +187,30 @@ public void testAuthenticatedProxyAsNonAdmin() throws Exception { Assert.assertEquals(ImmutableSet.of("tenant1/ns1"), user1Admin.namespaces().getNamespaces("tenant1")); } } + + @Test + public void testAuthenticatedRequestWithLongUri() throws Exception { + PulsarAdmin user1Admin = getAdminClient("user1"); + PulsarAdmin brokerAdmin = getDirectToBrokerAdminClient("admin"); + StringBuilder longTenant = new StringBuilder("tenant"); + for (int i = 10 * 1024; i > 0; i = i - 4){ + longTenant.append("_abc"); + } + try { + brokerAdmin.namespaces().getNamespaces(longTenant.toString()); + Assert.fail("expect error: Tenant not found"); + } catch (Exception ex){ + Assert.assertTrue(ex instanceof PulsarAdminException); + PulsarAdminException pulsarAdminException = (PulsarAdminException) ex; + Assert.assertEquals(pulsarAdminException.getStatusCode(), 404); + } + try { + user1Admin.namespaces().getNamespaces(longTenant.toString()); + Assert.fail("expect error: Tenant not found"); + } catch (Exception ex){ + Assert.assertTrue(ex instanceof PulsarAdminException); + PulsarAdminException pulsarAdminException = (PulsarAdminException) ex; + Assert.assertEquals(pulsarAdminException.getStatusCode(), 404); + } + } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java index ac1078a1a4596..97a73c20b60d0 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java @@ -68,6 +68,7 @@ public void testBackwardCompatibility() throws IOException { try (PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)))) { printWriter.println("zookeeperSessionTimeoutMs=60"); printWriter.println("zooKeeperCacheExpirySeconds=500"); + printWriter.println("httpMaxRequestHeaderSize=1234"); } testConfigFile.deleteOnExit(); InputStream stream = new FileInputStream(testConfigFile); @@ -75,6 +76,7 @@ public void testBackwardCompatibility() throws IOException { stream.close(); assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 60); assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 500); + assertEquals(serviceConfig.getHttpMaxRequestHeaderSize(), 1234); testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties"); if (testConfigFile.exists()) { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java index 4b69d72e46c92..246dd9f85e319 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java @@ -44,8 +44,12 @@ import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Result; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; @@ -66,6 +70,7 @@ public class ProxyIsAHttpProxyTest extends MockedPulsarServiceBaseTest { private Server backingServer1; private Server backingServer2; + private Server backingServer3; private PulsarResources resource; private Client client = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class)); @@ -85,6 +90,15 @@ protected void setup() throws Exception { backingServer2 = new Server(0); backingServer2.setHandler(newHandler("server2")); backingServer2.start(); + + backingServer3 = new Server(); + HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setRequestHeaderSize(20000); + ServerConnector connector = new ServerConnector(backingServer3, new HttpConnectionFactory(httpConfig)); + connector.setPort(0); + backingServer3.setConnectors(new Connector[]{connector}); + backingServer3.setHandler(newHandler("server3")); + backingServer3.start(); } private static AbstractHandler newHandler(String text) { @@ -96,7 +110,9 @@ public void handle(String target, Request baseRequest, response.setContentType("text/plain;charset=utf-8"); response.setStatus(HttpServletResponse.SC_OK); baseRequest.setHandled(true); - response.getWriter().println(String.format("%s,%s", text, request.getRequestURI())); + String uri = request.getRequestURI(); + response.getWriter().println(String.format("%s,%s", text, + uri.substring(0, uri.length() > 1024 ? 1024 : uri.length()))); } }; } @@ -331,6 +347,47 @@ public void testLongPath() throws Exception { } } + @Test + public void testLongUri() throws Exception { + Properties props = new Properties(); + props.setProperty("httpReverseProxy.3.path", "/service3"); + props.setProperty("httpReverseProxy.3.proxyTo", backingServer3.getURI().toString()); + props.setProperty("servicePort", "0"); + props.setProperty("webServicePort", "0"); + + ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); + AuthenticationService authService = new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)); + + StringBuilder longUri = new StringBuilder("/service3/tp"); + for (int i = 10 * 1024; i > 0; i = i - 11){ + longUri.append("_sub1_RETRY"); + } + + WebServer webServerMaxUriLen8k = new WebServer(proxyConfig, authService); + ProxyServiceStarter.addWebServerHandlers(webServerMaxUriLen8k, proxyConfig, null, + new BrokerDiscoveryProvider(proxyConfig, resource)); + webServerMaxUriLen8k.start(); + try { + Response r = client.target(webServerMaxUriLen8k.getServiceUri()).path(longUri.toString()).request().get(); + Assert.assertEquals(r.getStatus(), Response.Status.REQUEST_URI_TOO_LONG.getStatusCode()); + } finally { + webServerMaxUriLen8k.stop(); + } + + proxyConfig.setHttpMaxRequestHeaderSize(12 * 1024); + WebServer webServerMaxUriLen12k = new WebServer(proxyConfig, authService); + ProxyServiceStarter.addWebServerHandlers(webServerMaxUriLen12k, proxyConfig, null, + new BrokerDiscoveryProvider(proxyConfig, resource)); + webServerMaxUriLen12k.start(); + try { + Response r = client.target(webServerMaxUriLen12k.getServiceUri()).path(longUri.toString()).request().get(); + Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode()); + } finally { + webServerMaxUriLen12k.stop(); + } + } + @Test public void testPathEndsInSlash() throws Exception { Properties props = new Properties(); From 81971e2d91a18ac7aac245c36ce082cd3b8176ad Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 11 Apr 2023 14:50:08 +0800 Subject: [PATCH 087/280] [fix] [doc] Fix outdated java-doc of rate limiter (#19014) ### Motivation After PR #8611, the acquiring permits can be larger than configured msg-rate if used by subscribing. But doc was not updated in time. ### Modifications fix the outdated doc --- .../pulsar/broker/service/persistent/DispatchRateLimiter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java index 5b8987580c412..b1e4803548414 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java @@ -99,10 +99,8 @@ public long getAvailableDispatchRateLimitOnByte() { */ public boolean tryDispatchPermit(long msgPermits, long bytePermits) { boolean acquiredMsgPermit = msgPermits <= 0 || dispatchRateLimiterOnMessage == null - // acquiring permits must be < configured msg-rate; || dispatchRateLimiterOnMessage.tryAcquire(msgPermits); boolean acquiredBytePermit = bytePermits <= 0 || dispatchRateLimiterOnByte == null - // acquiring permits must be < configured msg-rate; || dispatchRateLimiterOnByte.tryAcquire(bytePermits); return acquiredMsgPermit && acquiredBytePermit; } From 36579dd0b656034a7a68341c181a52ccb042439a Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 11 Apr 2023 02:43:31 -0500 Subject: [PATCH 088/280] [improve][proxy] Only create ConnectionPool when needed (#20062) --- .../pulsar/proxy/server/ProxyConnection.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 36b3b4c6038fb..f03aa59619fd8 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -81,7 +81,7 @@ */ public class ProxyConnection extends PulsarHandler { private static final Logger LOG = LoggerFactory.getLogger(ProxyConnection.class); - // ConnectionPool is used by the proxy to issue lookup requests + // ConnectionPool is used by the proxy to issue lookup requests. It is null when doing direct broker proxying. private ConnectionPool connectionPool; private final AtomicLong requestIdGenerator = new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE / 2)); @@ -313,24 +313,7 @@ protected static boolean isTlsChannel(Channel channel) { } private synchronized void completeConnect() throws PulsarClientException { - Supplier clientCnxSupplier; - if (service.getConfiguration().isAuthenticationEnabled()) { - clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, - clientAuthData, clientAuthMethod, protocolVersionToAdvertise, - service.getConfiguration().isForwardAuthorizationCredentials(), this); - } else { - clientCnxSupplier = () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise); - } - - if (this.connectionPool == null) { - this.connectionPool = new ConnectionPool(clientConf, service.getWorkerGroup(), - clientCnxSupplier, - Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next()))); - } else { - LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", - remoteAddress, state, clientAuthRole); - } - + checkArgument(state == State.Connecting); LOG.info("[{}] complete connection, init proxy handler. authenticated with {} role {}, hasProxyToBrokerUrl: {}", remoteAddress, authMethod, clientAuthRole, hasProxyToBrokerUrl); if (hasProxyToBrokerUrl) { @@ -371,8 +354,26 @@ private synchronized void completeConnect() throws PulsarClientException { }); } else { // Client is doing a lookup, we can consider the handshake complete - // and we'll take care of just topics and - // partitions metadata lookups + // and we'll take care of just topics and partitions metadata lookups + Supplier clientCnxSupplier; + if (service.getConfiguration().isAuthenticationEnabled()) { + clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, + clientAuthData, clientAuthMethod, protocolVersionToAdvertise, + service.getConfiguration().isForwardAuthorizationCredentials(), this); + } else { + clientCnxSupplier = + () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise); + } + + if (this.connectionPool == null) { + this.connectionPool = new ConnectionPool(clientConf, service.getWorkerGroup(), + clientCnxSupplier, + Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next()))); + } else { + LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", + remoteAddress, state, clientAuthRole); + } + state = State.ProxyLookupRequests; lookupProxyHandler = service.newLookupProxyHandler(this); final ByteBuf msg = Commands.newConnected(protocolVersionToAdvertise, false); @@ -452,7 +453,7 @@ protected void authChallengeSuccessCallback(AuthData authChallenge) { } // First connection - if (this.connectionPool == null || state == State.Connecting) { + if (state == State.Connecting) { // authentication has completed, will send newConnected command. completeConnect(); } From 222e69a53ffc098115251253a2bc4fe8542ce55b Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Tue, 11 Apr 2023 01:59:04 -0700 Subject: [PATCH 089/280] [improve][broker] PIP-192 Improve TransferSheder for overload outlier for large clusters (#20059) --- .../extensions/scheduler/TransferShedder.java | 69 +++++++++++----- .../scheduler/TransferShedderTest.java | 78 +++++++++++++++++++ 2 files changed, 126 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 98e05296d605a..07d521a28afa7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -291,6 +291,10 @@ String peekMinBroker() { return brokersSortedByLoad.get(minBrokerIndex).getKey(); } + String peekMaxBroker() { + return brokersSortedByLoad.get(maxBrokerIndex).getKey(); + } + String pollMaxBroker() { return brokersSortedByLoad.get(maxBrokerIndex--).getKey(); } @@ -358,7 +362,9 @@ public Set findBundlesForUnloading(LoadManagerContext context, final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd(); boolean transfer = conf.isLoadBalancerTransferEnabled(); - if (stats.std() > targetStd || isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) { + if (stats.std() > targetStd + || isUnderLoaded(context, stats.peekMinBroker(), stats.avg) + || isOverLoaded(context, stats.peekMaxBroker(), stats.avg)) { unloadConditionHitCount++; } else { unloadConditionHitCount = 0; @@ -383,28 +389,36 @@ public Set findBundlesForUnloading(LoadManagerContext context, break; } UnloadDecision.Reason reason; - if (stats.std() <= targetStd) { - if (!isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) { - if (debugMode) { - log.info(CANNOT_CONTINUE_UNLOAD_MSG - + "The overall cluster load meets the target, std:{} <= targetStd:{}," - + " and minBroker:{} is not underloaded.", - stats.std(), targetStd, stats.peekMinBroker()); - } - break; - } else { - reason = Underloaded; - if (debugMode) { - log.info(String.format("broker:%s is underloaded:%s although " - + "load std:%.2f <= targetStd:%.2f. " - + "Continuing unload for this underloaded broker.", - stats.peekMinBroker(), - context.brokerLoadDataStore().get(stats.peekMinBroker()).get(), - stats.std(), targetStd)); - } + if (stats.std() > targetStd) { + reason = Overloaded; + } else if (isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) { + reason = Underloaded; + if (debugMode) { + log.info(String.format("broker:%s is underloaded:%s although " + + "load std:%.2f <= targetStd:%.2f. " + + "Continuing unload for this underloaded broker.", + stats.peekMinBroker(), + context.brokerLoadDataStore().get(stats.peekMinBroker()).get(), + stats.std(), targetStd)); } - } else { + } else if (isOverLoaded(context, stats.peekMaxBroker(), stats.avg)) { reason = Overloaded; + if (debugMode) { + log.info(String.format("broker:%s is overloaded:%s although " + + "load std:%.2f <= targetStd:%.2f. " + + "Continuing unload for this overloaded broker.", + stats.peekMaxBroker(), + context.brokerLoadDataStore().get(stats.peekMaxBroker()).get(), + stats.std(), targetStd)); + } + } else { + if (debugMode) { + log.info(CANNOT_CONTINUE_UNLOAD_MSG + + "The overall cluster load meets the target, std:{} <= targetStd:{}." + + "minBroker:{} is not underloaded. maxBroker:{} is not overloaded.", + stats.std(), targetStd, stats.peekMinBroker(), stats.peekMaxBroker()); + } + break; } String maxBroker = stats.pollMaxBroker(); @@ -671,6 +685,19 @@ private boolean isUnderLoaded(LoadManagerContext context, String broker, double context.brokerConfiguration().getLoadBalancerBrokerLoadTargetStd() / 2)); } + private boolean isOverLoaded(LoadManagerContext context, String broker, double avgLoad) { + var brokerLoadDataOptional = context.brokerLoadDataStore().get(broker); + if (brokerLoadDataOptional.isEmpty()) { + return false; + } + var conf = context.brokerConfiguration(); + var overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; + var targetStd = conf.getLoadBalancerBrokerLoadTargetStd(); + var brokerLoadData = brokerLoadDataOptional.get(); + var load = brokerLoadData.getWeightedMaxEMA(); + return load > overloadThreshold && load > avgLoad + targetStd; + } + private boolean isTransferable(LoadManagerContext context, Map availableBrokers, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 3279cb4e475d3..d673722d6fb44 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -200,6 +200,54 @@ public LoadManagerContext setupContext(int clusterSize) { return ctx; } + public LoadManagerContext setupContextLoadSkewedOverload(int clusterSize) { + var ctx = getContext(); + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + + int i = 0; + for (; i < clusterSize-1; i++) { + int brokerLoad = 1; + topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + 300_000, 700_000)); + brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + } + int brokerLoad = 100; + topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + 30_000_000, 70_000_000)); + brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + + return ctx; + } + + public LoadManagerContext setupContextLoadSkewedUnderload(int clusterSize) { + var ctx = getContext(); + + var brokerLoadDataStore = ctx.brokerLoadDataStore(); + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + + int i = 0; + for (; i < clusterSize-2; i++) { + int brokerLoad = 98; + topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + 30_000_000, 70_000_000)); + brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + } + + int brokerLoad = 99; + topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + 30_000_000, 70_000_000)); + brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + i++; + + brokerLoad = 1; + topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + 300_000, 700_000)); + brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + return ctx; + } + public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load, String broker) { var loadData = new BrokerLoadData(); SystemResourceUsage usage1 = new SystemResourceUsage(); @@ -1176,6 +1224,36 @@ public void testRandomLoad() throws IllegalAccessException { } } + @Test + public void testOverloadOutlier() { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = setupContextLoadSkewedOverload(100); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + var expected = new HashSet(); + expected.add(new UnloadDecision( + new Unload("broker99", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF", + Optional.of("broker52")), Success, Overloaded)); + assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), 0.019900000000000008); + assertEquals(counter.getLoadStd(), 0.09850375627355534); + } + + @Test + public void testUnderloadOutlier() { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = setupContextLoadSkewedUnderload(100); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + var expected = new HashSet(); + expected.add(new UnloadDecision( + new Unload("broker98", "my-tenant/my-namespace98/0x00000000_0x0FFFFFFF", + Optional.of("broker99")), Success, Underloaded)); + assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), 0.9704000000000005); + assertEquals(counter.getLoadStd(), 0.09652895938523735); + } + @Test public void testRandomLoadStats() { int numBrokers = 10; From dc1cdacab8eadbd3f9d207e66d4aefbca8bca920 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 11 Apr 2023 12:17:11 +0300 Subject: [PATCH 090/280] [fix][misc] Support .nar files without explicit directory entries (#19304) Co-authored-by: tison --- .../src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java index a71589e6d6427..9bd5bc48df819 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java @@ -122,6 +122,8 @@ private static void unpack(final File nar, final File workingDirectory) throws I if (jarEntry.isDirectory()) { FileUtils.ensureDirectoryExistAndCanReadAndWrite(f); } else { + // The directory entry might appear after the file entry + FileUtils.ensureDirectoryExistAndCanReadAndWrite(f.getParentFile()); makeFile(jarFile.getInputStream(jarEntry), f); } } From cd2aa550d0fe4e72b5ff88c4f6c1c2795b3ff2cd Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 11 Apr 2023 21:38:32 +0800 Subject: [PATCH 091/280] [improve] [broker] PIP-240: new public method unloadSubscription in PersistentTopic (#19737) part-1 of PIP-240: add a new method unloadSubscription( String subName ) for PersistentTopic --- .../service/BrokerServiceException.java | 12 + .../service/persistent/PersistentTopic.java | 48 ++++ .../client/api/UnloadSubscriptionTest.java | 265 ++++++++++++++++++ 3 files changed, 325 insertions(+) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java index fd3a391bca34a..3e77588b2459f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java @@ -160,6 +160,18 @@ public SubscriptionNotFoundException(String msg) { } } + public static class UnsupportedSubscriptionException extends BrokerServiceException { + public UnsupportedSubscriptionException(String msg) { + super(msg); + } + } + + public static class SubscriptionConflictUnloadException extends BrokerServiceException { + public SubscriptionConflictUnloadException(String msg) { + super(msg); + } + } + public static class SubscriptionBusyException extends BrokerServiceException { public SubscriptionBusyException(String msg) { super(msg); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 0374fc98212f2..0f5e60439810f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -49,6 +49,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import lombok.Getter; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -94,12 +95,14 @@ import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; +import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionConflictUnloadException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; +import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedSubscriptionException; import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Dispatcher; @@ -430,6 +433,51 @@ public AtomicLong getPendingWriteOps() { return pendingWriteOps; } + /** + * Unload a subscriber. + * @throws SubscriptionNotFoundException If subscription not founded. + * @throws UnsupportedSubscriptionException If the subscription is typed compaction. + * @throws SubscriptionConflictUnloadException Conflict topic-close, topic-delete, another-subscribe-unload, + * cannot unload subscription now + */ + public CompletableFuture unloadSubscription(@Nonnull String subName) { + final PersistentSubscription sub = subscriptions.get(subName); + if (sub == null) { + return CompletableFuture.failedFuture( + new SubscriptionNotFoundException(String.format("Subscription %s not found", subName))); + } + if (Compactor.COMPACTION_SUBSCRIPTION.equals(sub.getName())){ + return CompletableFuture.failedFuture( + new UnsupportedSubscriptionException(String.format("Unsupported subscription: %s", subName))); + } + // Fence old subscription -> Rewind cursor -> Replace with a new subscription. + return sub.disconnect().thenCompose(ignore -> { + if (!lock.writeLock().tryLock()) { + return CompletableFuture.failedFuture(new SubscriptionConflictUnloadException(String.format("Conflict" + + " topic-close, topic-delete, another-subscribe-unload, cannot unload subscription %s now", + topic, subName))); + } + try { + if (isFenced) { + return CompletableFuture.failedFuture(new TopicFencedException(String.format( + "Topic[%s] is fenced, can not unload subscription %s now", topic, subName))); + } + if (sub != subscriptions.get(subName)) { + // Another task already finished. + return CompletableFuture.failedFuture(new SubscriptionConflictUnloadException(String.format( + "Another unload subscriber[%s] has been finished, do not repeat call.", subName))); + } + sub.getCursor().rewind(); + PersistentSubscription subNew = PersistentTopic.this.createPersistentSubscription(sub.getName(), + sub.getCursor(), sub.isReplicated(), sub.getSubscriptionProperties()); + subscriptions.put(subName, subNew); + return CompletableFuture.completedFuture(null); + } finally { + lock.writeLock().unlock(); + } + }); + } + private PersistentSubscription createPersistentSubscription(String subscriptionName, ManagedCursor cursor, boolean replicated, Map subscriptionProperties) { Objects.requireNonNull(compactedTopic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java new file mode 100644 index 0000000000000..93d5bf30ec6b1 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.apache.pulsar.client.api.SubscriptionType.Shared; +import static org.apache.pulsar.client.api.SubscriptionType.Key_Shared; +import static org.apache.pulsar.client.api.SubscriptionType.Failover; +import static org.apache.pulsar.client.api.SubscriptionType.Exclusive; +import static org.testng.Assert.assertEquals; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.TopicMessageIdImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class UnloadSubscriptionTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setSystemTopicEnabled(false); + conf.setTransactionCoordinatorEnabled(false); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider(name = "unloadCases") + public Object[][] unloadCases (){ + // [msgCount, enabledBatch, maxMsgPerBatch, subType, ackMsgCount] + return new Object[][]{ + {100, false, 1, Exclusive, 0}, + {100, false, 1, Failover, 0}, + {100, false, 1, Shared, 0}, + {100, false, 1, Key_Shared, 0}, + {100, true, 5, Exclusive, 0}, + {100, true, 5, Failover, 0}, + {100, true, 5, Shared, 0}, + {100, true, 5, Key_Shared, 0}, + {100, false, 1, Exclusive, 50}, + {100, false, 1, Failover, 50}, + {100, false, 1, Shared, 50}, + {100, false, 1, Key_Shared, 50}, + {100, true, 5, Exclusive, 50}, + {100, true, 5, Failover, 50}, + {100, true, 5, Shared, 50}, + {100, true, 5, Key_Shared, 50}, + }; + } + + @Test(dataProvider = "unloadCases") + public void testSingleConsumer(int msgCount, boolean enabledBatch, int maxMsgPerBatch, SubscriptionType subType, + int ackMsgCount) throws Exception { + final String topicName = "persistent://my-property/my-ns/tp-" + UUID.randomUUID(); + final String subName = "sub"; + Consumer consumer = createConsumer(topicName, subName, subType); + ProducerAndMessageIds producerAndMessageIds = + createProducerAndSendMessages(topicName, msgCount, enabledBatch, maxMsgPerBatch); + log.info("send message-ids:{}-{}", producerAndMessageIds.messageIds.size(), + toString(producerAndMessageIds.messageIds)); + + // Receive all messages and ack some. + MessagesEntry messagesEntry = receiveAllMessages(consumer); + assertEquals(messagesEntry.messageSet.size(), msgCount); + if (ackMsgCount > 0){ + LinkedHashSet ackedMessageIds = new LinkedHashSet<>(); + Iterator messageIdIterator = messagesEntry.messageIdSet.iterator(); + for (int i = ackMsgCount; i > 0; i--){ + ackedMessageIds.add(messageIdIterator.next()); + } + consumer.acknowledge(ackedMessageIds.stream().toList()); + log.info("ack message-ids: {}", toString(ackedMessageIds.stream().toList())); + } + + + // Unload subscriber. + PersistentTopic persistentTopic = getPersistentTopic(topicName); + persistentTopic.unloadSubscription(subName); + // Receive all messages for the second time. + MessagesEntry messagesEntryForTheSecondTime = receiveAllMessages(consumer); + log.info("received message-ids for the second time: {}", + toString(messagesEntryForTheSecondTime.messageIdSet.stream().toList())); + assertEquals(messagesEntryForTheSecondTime.messageSet.size(), msgCount - ackMsgCount); + + // cleanup. + producerAndMessageIds.producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } + + @Test(dataProvider = "unloadCases") + public void testMultiConsumer(int msgCount, boolean enabledBatch, int maxMsgPerBatch, SubscriptionType subType, + int ackMsgCount) throws Exception { + if (subType == Exclusive){ + return; + } + final String topicName = "persistent://my-property/my-ns/tp-" + UUID.randomUUID(); + final String subName = "sub"; + Consumer consumer1 = createConsumer(topicName, subName, subType); + Consumer consumer2 = createConsumer(topicName, subName, subType); + ProducerAndMessageIds producerAndMessageIds = + createProducerAndSendMessages(topicName, msgCount, enabledBatch, maxMsgPerBatch); + log.info("send message-ids:{}-{}", producerAndMessageIds.messageIds.size(), + toString(producerAndMessageIds.messageIds)); + + // Receive all messages and ack some. + MessagesEntry messagesEntry1 = receiveAllMessages(consumer1); + MessagesEntry messagesEntry2 = receiveAllMessages(consumer2); + LinkedHashSet allMessages = new LinkedHashSet<>(); + allMessages.addAll(messagesEntry1.messageSet); + allMessages.addAll(messagesEntry2.messageSet); + assertEquals(allMessages.size(), msgCount); + if (ackMsgCount > 0){ + LinkedHashSet allMessageIds = new LinkedHashSet<>(); + LinkedHashSet ackedMessageIds = new LinkedHashSet<>(); + allMessageIds.addAll(messagesEntry1.messageIdSet); + allMessageIds.addAll(messagesEntry2.messageIdSet); + Iterator messageIdIterator = allMessageIds.iterator(); + for (int i = ackMsgCount; i > 0; i--){ + ackedMessageIds.add(messageIdIterator.next()); + } + consumer1.acknowledge(ackedMessageIds.stream().toList()); + log.info("ack message-ids: {}", toString(ackedMessageIds.stream().toList())); + } + + // Unload subscriber. + PersistentTopic persistentTopic = getPersistentTopic(topicName); + persistentTopic.unloadSubscription(subName); + + // Receive all messages for the second time. + MessagesEntry messagesEntryForTheSecondTime1 = receiveAllMessages(consumer1); + MessagesEntry messagesEntryForTheSecondTime2 = receiveAllMessages(consumer2); + LinkedHashSet allMessagesForTheSecondTime = new LinkedHashSet<>(); + allMessagesForTheSecondTime.addAll(messagesEntryForTheSecondTime1.messageSet); + allMessagesForTheSecondTime.addAll(messagesEntryForTheSecondTime2.messageSet); + LinkedHashSet allMessageIdsForTheSecondTime = new LinkedHashSet<>(); + allMessageIdsForTheSecondTime.addAll(messagesEntry1.messageIdSet); + allMessageIdsForTheSecondTime.addAll(messagesEntry2.messageIdSet); + log.info("received message-ids for the second time: {}", + toString(allMessageIdsForTheSecondTime.stream().toList())); + assertEquals(allMessagesForTheSecondTime.size(), msgCount - ackMsgCount); + + // cleanup. + producerAndMessageIds.producer.close(); + consumer1.close(); + consumer2.close(); + admin.topics().delete(topicName); + } + + private static String toString(List messageIds){ + List messageIdStrings = new ArrayList<>(messageIds.size()); + for (MessageId messageId : messageIds){ + MessageIdImpl messageIdImpl; + if (messageId instanceof TopicMessageIdImpl) { + TopicMessageIdImpl topicMessageId = (TopicMessageIdImpl) messageId; + messageIdImpl = (MessageIdImpl) topicMessageId.getInnerMessageId(); + } else { + messageIdImpl = (MessageIdImpl) messageId; + } + StringBuilder stringBuilder = new StringBuilder(String.valueOf(messageIdImpl.getEntryId())); + if (messageIdImpl instanceof BatchMessageIdImpl){ + BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageIdImpl; + stringBuilder.append("_") + .append(batchMessageId.getBatchIndex()) + .append("/") + .append(batchMessageId.getBatchSize()); + } + messageIdStrings.add(stringBuilder.toString()); + } + return messageIdStrings.toString(); + } + + private PersistentTopic getPersistentTopic(String topicName) { + return (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + } + + private ProducerAndMessageIds createProducerAndSendMessages(String topicName, int msgCount, boolean enabledBatch, + int maxMsgPerBatch) throws Exception { + final Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(enabledBatch) + .batchingMaxMessages(maxMsgPerBatch) + .create(); + ArrayList> messageIds = new ArrayList<>(); + for (int i = 0; i < msgCount; i++) { + messageIds.add(producer.newMessage().key(String.valueOf(i % 10)).value(String.valueOf(i)).sendAsync()); + } + FutureUtil.waitForAll(messageIds).join(); + return new ProducerAndMessageIds(producer, + messageIds.stream().map(CompletableFuture::join).collect(Collectors.toList())); + } + + private record ProducerAndMessageIds(Producer producer, List messageIds) {} + + private Consumer createConsumer(String topicName, String subName, SubscriptionType subType) + throws Exception { + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(subType) + .isAckReceiptEnabled(true) + .subscribe(); + return consumer; + } + + private MessagesEntry receiveAllMessages(Consumer consumer) throws Exception { + final Set messageSet = Collections.synchronizedSet(new LinkedHashSet<>()); + final Set messageIdSet = Collections.synchronizedSet(new LinkedHashSet<>()); + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg == null){ + break; + } + messageIdSet.add(msg.getMessageId()); + messageSet.add(msg.getValue()); + } + return new MessagesEntry(messageSet, messageIdSet); + } + + private record MessagesEntry(Set messageSet, Set messageIdSet) {} + +} \ No newline at end of file From 4f503fd886cd25670b6344defb8feb1ff2a6e816 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 11 Apr 2023 21:47:02 +0800 Subject: [PATCH 092/280] [fix] [cli] Fix Broker crashed by too much memory usage of pulsar tools (#20031) ### Motivation After #15868, we allow `PULSAR_MEM` & `PULSAR_GC` to be overridden in `pulsar_tool_env.sh`. Many users set `-Xms` to `2G` or larger in `PULSAR_MEM`, this will make the tools(such as `pulsar-admin`) cost a lot of memory, and if users execute `pulsar-admin` or another tool on the machine where the Broker is deployed, the current device will not have enough memory to allocate, resulting in a broker crash. ### Modifications When `PULSAR_MEM` is overridden in `pulsar_tool_env.sh`, delete parameter `-Xms` --- conf/pulsar_tools_env.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh index a356dbb9a28df..9d22b73905df3 100755 --- a/conf/pulsar_tools_env.sh +++ b/conf/pulsar_tools_env.sh @@ -42,6 +42,19 @@ # PULSAR_GLOBAL_ZK_CONF= # Extra options to be passed to the jvm +# Discard parameter "-Xms" of $PULSAR_MEM, which tends to be the Broker's minimum memory, to avoid using too much +# memory by tools. +if [ -n "$PULSAR_MEM" ]; then + PULSAR_MEM_ARR=("${PULSAR_MEM}") + PULSAR_MEM_REWRITE="" + for i in ${PULSAR_MEM_ARR} + do + if [ "${i:0:4}" != "-Xms" ]; then + PULSAR_MEM_REWRITE="$PULSAR_MEM_REWRITE $i"; + fi + done + PULSAR_MEM=${PULSAR_MEM_REWRITE} +fi PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"} # Garbage collection options From f2076b44764ffbbf39e01b3873a6735a90bc3d47 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 11 Apr 2023 22:50:04 +0800 Subject: [PATCH 093/280] [fix][test] Fix flaky testCreateTopicWithZombieReplicatorCursor (#20037) --- .../persistent/PersistentTopicTest.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index c63be7aad01cd..412b8207e34c7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -38,7 +38,9 @@ import java.io.ByteArrayOutputStream; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -52,6 +54,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -83,6 +86,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +@Slf4j @Test(groups = "broker") public class PersistentTopicTest extends BrokerTestBase { @@ -558,10 +562,10 @@ public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy) admin.tenants().updateTenant("prop", tenantInfo); if (topicLevelPolicy) { - admin.topics().setReplicationClusters(topicName, Collections.singletonList(remoteCluster)); + admin.topics().setReplicationClusters(topicName, Arrays.asList("test", remoteCluster)); } else { admin.namespaces().setNamespaceReplicationClustersAsync( - namespace, Collections.singleton(remoteCluster)).get(); + namespace, Sets.newHashSet("test", remoteCluster)).get(); } final PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false) @@ -576,16 +580,27 @@ public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy) }; assertEquals(getCursors.get(), Collections.singleton(conf.getReplicatorPrefix() + "." + remoteCluster)); + // PersistentTopics#onPoliciesUpdate might happen in different threads, so there might be a race between two + // updates of the replication clusters. So here we sleep for a while to reduce the flakiness. + Thread.sleep(100); + + // Configure the local cluster to avoid the topic being deleted in PersistentTopics#checkReplication if (topicLevelPolicy) { - admin.topics().setReplicationClusters(topicName, Collections.emptyList()); + admin.topics().setReplicationClusters(topicName, Collections.singletonList("test")); } else { - admin.namespaces().setNamespaceReplicationClustersAsync(namespace, Collections.emptySet()).get(); + admin.namespaces().setNamespaceReplicationClustersAsync(namespace, Collections.singleton("test")).get(); } admin.clusters().deleteCluster(remoteCluster); // Now the cluster and its related policy has been removed but the replicator cursor still exists - topic.initialize().get(3, TimeUnit.SECONDS); - Awaitility.await().atMost(3, TimeUnit.SECONDS) - .until(() -> !topic.getManagedLedger().getCursors().iterator().hasNext()); + Awaitility.await().atMost(Duration.ofSeconds(10)).until(() -> { + log.info("Before initialize..."); + try { + topic.initialize().get(3, TimeUnit.SECONDS); + } catch (ExecutionException e) { + log.warn("Failed to initialize: {}", e.getCause().getMessage()); + } + return !topic.getManagedLedger().getCursors().iterator().hasNext(); + }); } } From b136cab02756e71369b9f9e7d0c0f295566624da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 11 Apr 2023 18:43:42 +0200 Subject: [PATCH 094/280] [fix][broker] Fix leader broker log (#19987) --- .../java/org/apache/pulsar/broker/PulsarService.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index b2eefe298b53c..aad7d32f732c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -81,6 +81,7 @@ import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.intercept.BrokerInterceptors; +import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils; import org.apache.pulsar.broker.loadbalance.LoadManager; @@ -1133,8 +1134,14 @@ protected void startLeaderElectionService() { } } else { if (leaderElectionService != null) { - LOG.info("This broker is a follower. Current leader is {}", - leaderElectionService.getCurrentLeader()); + final Optional currentLeader = leaderElectionService.getCurrentLeader(); + if (currentLeader.isPresent()) { + LOG.info("This broker is a follower. Current leader is {}", + currentLeader); + } else { + LOG.info("This broker is a follower. No leader has been elected yet"); + } + } if (loadSheddingTask != null) { loadSheddingTask.cancel(); From 15453964f6e6086b0c5e34736958cf749f16f5e1 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 11 Apr 2023 13:19:32 -0500 Subject: [PATCH 095/280] [feat][proxy] PIP-250: Add proxyVersion to CommandConnect (#19618) PIP #19623 Relates to https://github.com/apache/pulsar/pull/19540 ### Motivation In order to get more information about connections, it is helpful for the proxy to supply its version to the broker. ### Modifications * Add `proxy_version` field to the `CommandConnect` protobuf message * Update proxy and broker to handle this new field ### Verifying this change New tests are added with this PR. ### Does this pull request potentially affect one of the following parts: - [x] The binary protocol This will be submitted as part of a PIP. ### Documentation - [x] `doc-not-needed` --- .../authorization/AuthorizationService.java | 2 +- .../pulsar/broker/service/ServerCnx.java | 40 +++++++++++++++--- .../pulsar/broker/service/TransportCnx.java | 1 + .../pulsar/broker/service/ServerCnxTest.java | 41 +++++++++++++++++++ .../pulsar/common/protocol/Commands.java | 11 +++++ pulsar-common/src/main/proto/PulsarApi.proto | 6 ++- .../proxy/server/DirectProxyHandler.java | 2 +- .../pulsar/proxy/server/ProxyClientCnx.java | 3 +- .../pulsar/proxy/server/ProxyConnection.java | 9 ++++ 9 files changed, 105 insertions(+), 10 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 706eadf0ec2dc..0c61219b57a50 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -464,7 +464,7 @@ public boolean isValidOriginalPrincipal(String authenticatedPrincipal, } } - private boolean isProxyRole(String role) { + public boolean isProxyRole(String role) { return role != null && conf.getProxyRoles().contains(role); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index bfd7c001e4cfb..01274ab4460cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -210,6 +210,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private int pendingSendRequest = 0; private final String replicatorPrefix; private String clientVersion = null; + private String proxyVersion = null; private int nonPersistentPendingMessages = 0; private final int maxNonPersistentPendingMessages; private String originalPrincipal = null; @@ -320,7 +321,10 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg); return; } - log.info("New connection from {}", remoteAddress); + if (log.isDebugEnabled()) { + // Connection information is logged after a successful Connect command is processed. + log.debug("New connection from {}", remoteAddress); + } this.ctx = ctx; this.commandSender = new PulsarCommandSenderImpl(brokerInterceptor, this); this.service.getPulsarStats().recordConnectionCreate(); @@ -690,6 +694,15 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg); return; } + if (proxyVersion != null && !service.getAuthorizationService().isProxyRole(authRole)) { + // Only allow proxyVersion to be set when connecting with a proxy + state = State.Failed; + service.getPulsarStats().recordConnectionCreateFail(); + final ByteBuf msg = Commands.newError(-1, ServerError.AuthorizationError, + "Must not set proxyVersion without connecting as a ProxyRole."); + NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg); + return; + } } maybeScheduleAuthenticationCredentialsRefresh(); } @@ -703,6 +716,18 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { if (isNotBlank(clientVersion)) { this.clientVersion = clientVersion.intern(); } + if (!service.isAuthenticationEnabled()) { + log.info("[{}] connected with clientVersion={}, clientProtocolVersion={}, proxyVersion={}", remoteAddress, + clientVersion, clientProtoVersion, proxyVersion); + } else if (originalPrincipal != null) { + log.info("[{}] connected role={} and originalAuthRole={} using authMethod={}, clientVersion={}, " + + "clientProtocolVersion={}, proxyVersion={}", remoteAddress, authRole, originalPrincipal, + authMethod, clientVersion, clientProtoVersion, proxyVersion); + } else { + log.info("[{}] connected with role={} using authMethod={}, clientVersion={}, clientProtocolVersion={}, " + + "proxyVersion={}", remoteAddress, authRole, authMethod, clientVersion, clientProtoVersion, + proxyVersion); + } if (brokerInterceptor != null) { brokerInterceptor.onConnectionCreated(this); } @@ -761,10 +786,6 @@ public void authChallengeSuccessCallback(AuthData authChallenge, authenticateOriginalData(clientProtocolVersion, clientVersion); } else { completeConnect(clientProtocolVersion, clientVersion); - if (log.isDebugEnabled()) { - log.debug("[{}] Client successfully authenticated with {} role {} and originalPrincipal {}", - remoteAddress, authMethod, this.authRole, originalPrincipal); - } } } else { // Refresh the auth data @@ -948,6 +969,10 @@ protected void handleConnect(CommandConnect connect) { features.copyFrom(connect.getFeatureFlags()); } + if (connect.hasProxyVersion()) { + proxyVersion = connect.getProxyVersion(); + } + if (!service.isAuthenticationEnabled()) { completeConnect(clientProtocolVersion, clientVersion); return; @@ -3266,6 +3291,11 @@ public String getClientVersion() { return clientVersion; } + @Override + public String getProxyVersion() { + return proxyVersion; + } + @VisibleForTesting void setAutoReadDisabledRateLimiting(boolean isLimiting) { this.autoReadDisabledRateLimiting = isLimiting; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java index f1aaca2b290fa..d267160652ae4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java @@ -27,6 +27,7 @@ public interface TransportCnx { String getClientVersion(); + String getProxyVersion(); SocketAddress clientAddress(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 874d43896223c..c3bab634a42c1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -297,6 +297,21 @@ public void testConnectCommandWithProtocolVersion() throws Exception { channel.finish(); } + @Test(timeOut = 30000) + public void testConnectCommandWithProxyVersion() throws Exception { + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + ByteBuf clientCommand = Commands.newConnect("none", null, 1, null, null, null, null, null, + "my-pulsar-proxy"); + channel.writeInbound(clientCommand); + + assertEquals(serverCnx.getState(), State.Connected); + assertEquals(serverCnx.getProxyVersion(), "my-pulsar-proxy"); + channel.finish(); + } + @DataProvider(name = "clientVersions") public Object[][] clientVersions() { return new Object[][]{ @@ -512,6 +527,32 @@ public void testConnectCommandWithPassingOriginalPrincipal() throws Exception { channel.finish(); } + @Test + public void testConnectWithNonProxyRoleAndProxyVersion() throws Exception { + AuthenticationService authenticationService = mock(AuthenticationService.class); + AuthenticationProvider authenticationProvider = new MockAuthenticationProvider(); + String authMethodName = authenticationProvider.getAuthMethodName(); + + when(brokerService.getAuthenticationService()).thenReturn(authenticationService); + when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); + svcConfig.setAuthenticationEnabled(true); + svcConfig.setAuthorizationEnabled(true); + + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + ByteBuf clientCommand = Commands.newConnect(authMethodName, AuthData.of("pass.pass".getBytes()), + 1, null, null, null, null, null, "my-pulsar-proxy"); + channel.writeInbound(clientCommand); + Object response = getResponse(); + assertTrue(response instanceof CommandError); + assertEquals(((CommandError) response).getError(), ServerError.AuthorizationError); + assertEquals(serverCnx.getState(), State.Failed); + channel.finish(); + } + + @Test public void testAuthChallengePrincipalChangeFails() throws Exception { AuthenticationService authenticationService = mock(AuthenticationService.class); AuthenticationProvider authenticationProvider = new MockAlwaysExpiredAuthenticationProvider(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 85c4d021fdf22..cf0cd820a6d10 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -234,11 +234,22 @@ public static ByteBuf newConnect(String authMethodName, String authData, int pro public static ByteBuf newConnect(String authMethodName, AuthData authData, int protocolVersion, String libVersion, String targetBroker, String originalPrincipal, AuthData originalAuthData, String originalAuthMethod) { + return newConnect(authMethodName, authData, protocolVersion, libVersion, targetBroker, originalPrincipal, + originalAuthData, originalAuthMethod, null); + } + + public static ByteBuf newConnect(String authMethodName, AuthData authData, int protocolVersion, String libVersion, + String targetBroker, String originalPrincipal, AuthData originalAuthData, + String originalAuthMethod, String proxyVersion) { BaseCommand cmd = localCmd(Type.CONNECT); CommandConnect connect = cmd.setConnect() .setClientVersion(libVersion != null ? libVersion : "Pulsar Client") .setAuthMethodName(authMethodName); + if (proxyVersion != null) { + connect.setProxyVersion(proxyVersion); + } + if (targetBroker != null) { // When connecting through a proxy, we need to specify which broker do we want to be proxied through connect.setProxyToBrokerUrl(targetBroker); diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index d9c41eeec9740..afe193eeb7e9d 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -268,7 +268,7 @@ enum ProtocolVersion { } message CommandConnect { - required string client_version = 1; + required string client_version = 1; // The version of the client. Proxy should forward client's client_version. optional AuthMethod auth_method = 2; // Deprecated. Use "auth_method_name" instead. optional string auth_method_name = 5; optional bytes auth_data = 3; @@ -291,6 +291,8 @@ message CommandConnect { // Feature flags optional FeatureFlags feature_flags = 10; + + optional string proxy_version = 11; // Version of the proxy. Should only be forwarded by a proxy. } message FeatureFlags { @@ -308,7 +310,7 @@ message CommandConnected { } message CommandAuthResponse { - optional string client_version = 1; + optional string client_version = 1; // The version of the client. Proxy should forward client's client_version. optional AuthData response = 2; optional int32 protocol_version = 3 [default = 0]; } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index cb93f885a1d68..d63b04b6734de 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -327,7 +327,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { ByteBuf command = Commands.newConnect( authentication.getAuthMethodName(), authData, protocolVersion, proxyConnection.clientVersion, null /* target broker */, - originalPrincipal, clientAuthData, clientAuthMethod); + originalPrincipal, clientAuthData, clientAuthMethod, PulsarVersion.getVersion()); writeAndFlush(command); isTlsOutboundChannel = ProxyConnection.isTlsChannel(inboundChannel); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java index a1994fb5af4b0..00ec0b592e463 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java @@ -23,6 +23,7 @@ import io.netty.channel.EventLoopGroup; import java.util.Arrays; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.api.AuthData; @@ -66,7 +67,7 @@ protected ByteBuf newConnectCommand() throws Exception { AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); return Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion, proxyConnection.clientVersion, proxyToTargetBrokerAddress, clientAuthRole, clientAuthData, - clientAuthMethod); + clientAuthMethod, PulsarVersion.getVersion()); } @Override diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index f03aa59619fd8..4d32c9dce2d49 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -497,6 +497,15 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), return; } + if (connect.hasProxyVersion()) { + if (LOG.isDebugEnabled()) { + LOG.debug("[{}] Client illegally provided proxyVersion.", remoteAddress); + } + state = State.Closing; + writeAndFlushAndClose(Commands.newError(-1, ServerError.NotAllowedError, "Must not provide proxyVersion")); + return; + } + try { // init authn this.clientConf = createClientConfiguration(); From 075b625993746a6c2d72343cb969845aed2cd28a Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 11 Apr 2023 15:39:26 -0500 Subject: [PATCH 096/280] [fix][proxy] Refresh auth data if ProxyLookupRequests (#20067) Fixes: https://github.com/apache/pulsar/issues/10816 PIP: #19771 Supersedes: https://github.com/apache/pulsar/pull/19026 Depends on: https://github.com/apache/pulsar/pull/20062 ### Motivation The Pulsar Proxy does not properly handle authentication data refresh when in state `ProxyLookupRequests`. The consequence is described in https://github.com/apache/pulsar/issues/10816. Essentially, the problem is that the proxy caches stale authentication data and sends it to the broker leading to connection failures. https://github.com/apache/pulsar/pull/17831 attempted to fix the underlying problem, but it missed an important edge cases. Specifically, it missed the case that the `ConnectionPool` will have multiple connections when a lookup gets redirected. As such, the following problem exists (and is fixed by this PR): 1. Client opens connection to perform lookups. 2. Proxy connects to broker 1 to get the topic ownership info. 3. Time passes. 4. Client does an additional lookup, and this topic is on a newly created broker 2. In this case, the proxy opens a new connection with the stale client auth data. 5. Broker 2 rejects the connection because it fails with expired authentication. ### Modifications * Remove some of the implementation from https://github.com/apache/pulsar/pull/17831. This new implementation still allows a broker to challenge the client through the proxy, but notably, it limits the number of challenges sent to the client. Further, the proxy does not challenge the client when the auth data is not expired. * Introduce authentication refresh in the proxy so that the proxy challenges the client any time the auth data is expired. * Update the `ProxyClientCnx` to get the `clientAuthData` from the `ProxyConnection` to ensure that it gets new authentication data. * Add clock skew to the `AuthenticationProviderToken`. This is necessary to make some of the testing not flaky and it will also be necessary for users to configure in their clusters. ### Verifying this change The `ProxyRefreshAuthTest` covers the existing behavior and I expanded it to cover the edge case described above. Additionally, testing this part of the code will be much easier to test once we implement #19624. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: the relevant tests pass locally, so I am going to skip the forked tests. --- .../AuthenticationProviderToken.java | 22 ++- .../pulsar/proxy/server/ProxyClientCnx.java | 67 +++---- .../proxy/server/ProxyConfiguration.java | 7 + .../pulsar/proxy/server/ProxyConnection.java | 185 +++++++++++++----- .../proxy/server/ProxyRefreshAuthTest.java | 31 ++- 5 files changed, 216 insertions(+), 96 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java index fed5ba063fd44..67e1f39a38924 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java @@ -73,6 +73,9 @@ public class AuthenticationProviderToken implements AuthenticationProvider { // The token audience stands for this broker. The field `tokenAudienceClaim` of a valid token, need contains this. static final String CONF_TOKEN_AUDIENCE = "tokenAudience"; + // The amount of time in seconds that a token is allowed to be out of sync with the server's time when performing + // token validation. + static final String CONF_TOKEN_ALLOWED_CLOCK_SKEW_SECONDS = "tokenAllowedClockSkewSeconds"; static final String TOKEN = "token"; @@ -101,6 +104,7 @@ public class AuthenticationProviderToken implements AuthenticationProvider { private String confTokenPublicAlgSettingName; private String confTokenAudienceClaimSettingName; private String confTokenAudienceSettingName; + private String confTokenAllowedClockSkewSecondsSettingName; @Override public void close() throws IOException { @@ -125,6 +129,7 @@ public void initialize(ServiceConfiguration config) throws IOException, IllegalA this.confTokenPublicAlgSettingName = prefix + CONF_TOKEN_PUBLIC_ALG; this.confTokenAudienceClaimSettingName = prefix + CONF_TOKEN_AUDIENCE_CLAIM; this.confTokenAudienceSettingName = prefix + CONF_TOKEN_AUDIENCE; + this.confTokenAllowedClockSkewSecondsSettingName = prefix + CONF_TOKEN_ALLOWED_CLOCK_SKEW_SECONDS; // we need to fetch the algorithm before we fetch the key this.publicKeyAlg = getPublicKeyAlgType(config); @@ -133,7 +138,12 @@ public void initialize(ServiceConfiguration config) throws IOException, IllegalA this.audienceClaim = getTokenAudienceClaim(config); this.audience = getTokenAudience(config); - this.parser = Jwts.parserBuilder().setSigningKey(this.validationKey).build(); + long allowedSkew = getConfTokenAllowedClockSkewSeconds(config); + + this.parser = Jwts.parserBuilder() + .setAllowedClockSkewSeconds(allowedSkew) + .setSigningKey(this.validationKey) + .build(); if (audienceClaim != null && audience == null) { throw new IllegalArgumentException("Token Audience Claim [" + audienceClaim @@ -329,6 +339,16 @@ private String getTokenAudience(ServiceConfiguration conf) throws IllegalArgumen } } + // get Token's allowed clock skew in seconds. If not configured, defaults to 0. + private long getConfTokenAllowedClockSkewSeconds(ServiceConfiguration conf) throws IllegalArgumentException { + String allowedSkewStr = (String) conf.getProperty(confTokenAllowedClockSkewSecondsSettingName); + if (StringUtils.isNotBlank(allowedSkewStr)) { + return Long.parseLong(allowedSkewStr); + } else { + return 0; + } + } + private static final class TokenAuthenticationState implements AuthenticationState { private final AuthenticationProviderToken provider; private final SocketAddress remoteAddress; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java index 00ec0b592e463..782454022b1ed 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java @@ -29,10 +29,11 @@ import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.api.proto.CommandAuthChallenge; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.netty.NettyChannelUtil; @Slf4j /** - * Channel handler for Pulsar proxy's Pulsar broker client connections. + * Channel handler for Pulsar proxy's Pulsar broker client connections for lookup requests. *

* Please see {@link org.apache.pulsar.common.protocol.PulsarDecoder} javadoc for important details about handle* * method parameter instance lifecycle. @@ -41,15 +42,13 @@ public class ProxyClientCnx extends ClientCnx { private final boolean forwardClientAuthData; private final String clientAuthMethod; private final String clientAuthRole; - private final AuthData clientAuthData; private final ProxyConnection proxyConnection; public ProxyClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, String clientAuthRole, - AuthData clientAuthData, String clientAuthMethod, int protocolVersion, + String clientAuthMethod, int protocolVersion, boolean forwardClientAuthData, ProxyConnection proxyConnection) { super(conf, eventLoopGroup, protocolVersion); this.clientAuthRole = clientAuthRole; - this.clientAuthData = clientAuthData; this.clientAuthMethod = clientAuthMethod; this.forwardClientAuthData = forwardClientAuthData; this.proxyConnection = proxyConnection; @@ -60,9 +59,15 @@ protected ByteBuf newConnectCommand() throws Exception { if (log.isDebugEnabled()) { log.debug("New Connection opened via ProxyClientCnx with params clientAuthRole = {}," + " clientAuthData = {}, clientAuthMethod = {}", - clientAuthRole, clientAuthData, clientAuthMethod); + clientAuthRole, proxyConnection.getClientAuthData(), clientAuthMethod); + } + AuthData clientAuthData = null; + if (forwardClientAuthData) { + // There is a chance this auth data is expired because the ProxyConnection does not do early token refresh. + // Based on the current design, the best option is to configure the broker to accept slightly stale + // authentication data. + clientAuthData = proxyConnection.getClientAuthData(); } - authenticationDataProvider = authentication.getAuthData(remoteHostName); AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); return Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion, @@ -76,43 +81,21 @@ protected void handleAuthChallenge(CommandAuthChallenge authChallenge) { checkArgument(authChallenge.getChallenge().hasAuthData()); boolean isRefresh = Arrays.equals(AuthData.REFRESH_AUTH_DATA_BYTES, authChallenge.getChallenge().getAuthData()); - if (!forwardClientAuthData || !isRefresh) { - super.handleAuthChallenge(authChallenge); - return; - } - - try { - if (log.isDebugEnabled()) { - log.debug("Proxy {} request to refresh the original client authentication data for " - + "the proxy client {}", proxyConnection.ctx().channel(), ctx.channel()); - } - - proxyConnection.ctx().writeAndFlush(Commands.newAuthChallenge(clientAuthMethod, AuthData.REFRESH_AUTH_DATA, - protocolVersion)) - .addListener(writeFuture -> { - if (writeFuture.isSuccess()) { - if (log.isDebugEnabled()) { - log.debug("Proxy {} sent the auth challenge to original client to refresh credentials " - + "with method {} for the proxy client {}", - proxyConnection.ctx().channel(), clientAuthMethod, ctx.channel()); - } - } else { - log.error("Failed to send the auth challenge to original client by the proxy {} " - + "for the proxy client {}", - proxyConnection.ctx().channel(), - ctx.channel(), - writeFuture.cause()); - closeWithException(writeFuture.cause()); - } + if (forwardClientAuthData && isRefresh) { + proxyConnection.getValidClientAuthData() + .thenApplyAsync(authData -> { + NettyChannelUtil.writeAndFlushWithVoidPromise(ctx, + Commands.newAuthResponse(clientAuthMethod, authData, this.protocolVersion, + String.format("Pulsar-Java-v%s", PulsarVersion.getVersion()))); + return null; + }, ctx.executor()) + .exceptionally(ex -> { + log.warn("Failed to get valid client auth data. Closing connection.", ex); + ctx.close(); + return null; }); - - if (state == State.SentConnectFrame) { - state = State.Connecting; - } - } catch (Exception e) { - log.error("Failed to send the auth challenge to origin client by the proxy {} for the proxy client {}", - proxyConnection.ctx().channel(), ctx.channel(), e); - closeWithException(e); + } else { + super.handleAuthChallenge(authChallenge); } } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 52e50a7e6b87b..3ecd670cbbf7a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -364,6 +364,13 @@ public class ProxyConfiguration implements PulsarConfiguration { + "to take effect" ) private boolean forwardAuthorizationCredentials = false; + + @FieldContext( + category = CATEGORY_AUTHENTICATION, + doc = "Interval of time for checking for expired authentication credentials. Disable by setting to 0." + ) + private int authenticationRefreshCheckSeconds = 60; + @FieldContext( category = CATEGORY_AUTHENTICATION, doc = "Whether the '/metrics' endpoint requires authentication. Defaults to true." diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 4d32c9dce2d49..ba9247a085dff 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -29,13 +29,17 @@ import io.netty.handler.codec.haproxy.HAProxyMessage; import io.netty.handler.ssl.SslHandler; import io.netty.resolver.dns.DnsAddressResolverGroup; +import io.netty.util.concurrent.ScheduledFuture; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.channels.ClosedChannelException; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -64,10 +68,12 @@ import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.api.proto.CommandLookupTopic; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata; +import org.apache.pulsar.common.api.proto.FeatureFlags; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarHandler; +import org.apache.pulsar.common.util.Runnables; import org.apache.pulsar.common.util.netty.NettyChannelUtil; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.slf4j.Logger; @@ -92,10 +98,16 @@ public class ProxyConnection extends PulsarHandler { private LookupProxyHandler lookupProxyHandler = null; @Getter private DirectProxyHandler directProxyHandler = null; + private ScheduledFuture authRefreshTask; + // When authChallengeSentTime is not Long.MAX_VALUE, it means the proxy is waiting for the client to respond + // to an auth challenge. When authChallengeSentTime is Long.MAX_VALUE, there are no pending auth challenges. + private long authChallengeSentTime = Long.MAX_VALUE; + private FeatureFlags features; + private Set> pendingBrokerAuthChallenges = null; private final BrokerProxyValidator brokerProxyValidator; private final ConnectionController connectionController; String clientAuthRole; - AuthData clientAuthData; + volatile AuthData clientAuthData; String clientAuthMethod; String clientVersion; @@ -191,6 +203,15 @@ public synchronized void channelInactive(ChannelHandlerContext ctx) throws Excep directProxyHandler = null; } + if (authRefreshTask != null) { + authRefreshTask.cancel(false); + } + + if (pendingBrokerAuthChallenges != null) { + pendingBrokerAuthChallenges.forEach(future -> future.cancel(true)); + pendingBrokerAuthChallenges = null; + } + service.getClientCnxs().remove(this); LOG.info("[{}] Connection closed", remoteAddress); @@ -358,7 +379,7 @@ private synchronized void completeConnect() throws PulsarClientException { Supplier clientCnxSupplier; if (service.getConfiguration().isAuthenticationEnabled()) { clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, - clientAuthData, clientAuthMethod, protocolVersionToAdvertise, + clientAuthMethod, protocolVersionToAdvertise, service.getConfiguration().isForwardAuthorizationCredentials(), this); } else { clientCnxSupplier = @@ -376,6 +397,15 @@ private synchronized void completeConnect() throws PulsarClientException { state = State.ProxyLookupRequests; lookupProxyHandler = service.newLookupProxyHandler(this); + if (service.getConfiguration().isAuthenticationEnabled() + && service.getConfiguration().getAuthenticationRefreshCheckSeconds() > 0) { + authRefreshTask = ctx.executor().scheduleAtFixedRate( + Runnables.catchingAndLoggingThrowables( + this::refreshAuthenticationCredentialsAndCloseIfTooExpired), + service.getConfiguration().getAuthenticationRefreshCheckSeconds(), + service.getConfiguration().getAuthenticationRefreshCheckSeconds(), + TimeUnit.SECONDS); + } final ByteBuf msg = Commands.newConnected(protocolVersionToAdvertise, false); writeAndFlush(msg); } @@ -472,6 +502,61 @@ protected void authChallengeSuccessCallback(AuthData authChallenge) { } } + private void refreshAuthenticationCredentialsAndCloseIfTooExpired() { + assert ctx.executor().inEventLoop(); + if (state != State.ProxyLookupRequests) { + // Happens when an exception is thrown that causes this connection to close. + return; + } else if (!authState.isExpired()) { + // Credentials are still valid. Nothing to do at this point + return; + } + + if (System.nanoTime() - authChallengeSentTime + > TimeUnit.SECONDS.toNanos(service.getConfiguration().getAuthenticationRefreshCheckSeconds())) { + LOG.warn("[{}] Closing connection after timeout on refreshing auth credentials", remoteAddress); + ctx.close(); + } + + maybeSendAuthChallenge(); + } + + private void maybeSendAuthChallenge() { + assert ctx.executor().inEventLoop(); + + if (!supportsAuthenticationRefresh()) { + LOG.warn("[{}] Closing connection because client doesn't support auth credentials refresh", remoteAddress); + ctx.close(); + return; + } else if (authChallengeSentTime != Long.MAX_VALUE) { + // If the proxy sent a refresh but hasn't yet heard back, do not send another challenge. + return; + } else if (service.getConfiguration().getAuthenticationRefreshCheckSeconds() < 1) { + // Without the refresh check enabled, there is no way to guarantee the ProxyConnection will close + // this connection if the client fails to respond to the auth challenge with valid auth data. + // The cost is minimal since the client can recreate the connection. This logic prevents a leak. + LOG.warn("[{}] Closing connection because auth credentials refresh is disabled", remoteAddress); + ctx.close(); + return; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("[{}] Refreshing authentication credentials", remoteAddress); + } + try { + AuthData challenge = authState.refreshAuthentication(); + writeAndFlush(Commands.newAuthChallenge(authMethod, challenge, protocolVersionToAdvertise)); + if (LOG.isDebugEnabled()) { + LOG.debug("[{}] Sent auth challenge to client to refresh credentials with method: {}.", + remoteAddress, authMethod); + } + authChallengeSentTime = System.nanoTime(); + } catch (AuthenticationException e) { + LOG.warn("[{}] Failed to refresh authentication: {}", remoteAddress, e); + ctx.close(); + } + } + @Override protected void handleConnect(CommandConnect connect) { checkArgument(state == State.Init); @@ -481,6 +566,10 @@ protected void handleConnect(CommandConnect connect) { this.protocolVersionToAdvertise = getProtocolVersionToAdvertise(connect); this.proxyToBrokerUrl = connect.hasProxyToBrokerUrl() ? connect.getProxyToBrokerUrl() : "null"; this.clientVersion = connect.getClientVersion(); + features = new FeatureFlags(); + if (connect.hasFeatureFlags()) { + features.copyFrom(connect.getFeatureFlags()); + } if (LOG.isDebugEnabled()) { LOG.debug("Received CONNECT from {} proxyToBroker={}", remoteAddress, proxyToBrokerUrl); @@ -574,53 +663,25 @@ protected void handleAuthResponse(CommandAuthResponse authResponse) { } try { + // Reset the auth challenge sent time to indicate we are not waiting on a client response. + authChallengeSentTime = Long.MAX_VALUE; AuthData clientData = AuthData.of(authResponse.getResponse().getAuthData()); + // Authenticate the client's auth data and send to the broker concurrently + // Note: this implementation relies on the current weakness that prevents multi-stage authentication + // from working when forwardAuthorizationCredentials is enabled. Here is an issue to fix the protocol: + // https://github.com/apache/pulsar/issues/19291. doAuthentication(clientData); - if (service.getConfiguration().isForwardAuthorizationCredentials() - && connectionPool != null && state == State.ProxyLookupRequests) { - connectionPool.getConnections().forEach(toBrokerCnxFuture -> { - String clientVersion; - if (authResponse.hasClientVersion()) { - clientVersion = authResponse.getClientVersion(); - } else { - clientVersion = this.clientVersion; - } - int protocolVersion; - if (authResponse.hasProtocolVersion()) { - protocolVersion = authResponse.getProtocolVersion(); - } else { - protocolVersion = Commands.getCurrentProtocolVersion(); + if (service.getConfiguration().isForwardAuthorizationCredentials()) { + // Update the clientAuthData to be able to initialize future ProxyClientCnx. + this.clientAuthData = clientData; + // We only have pendingBrokerAuthChallenges when forwardAuthorizationCredentials is enabled. + if (pendingBrokerAuthChallenges != null && !pendingBrokerAuthChallenges.isEmpty()) { + // Send auth data to pending challenges from the broker + for (CompletableFuture challenge : pendingBrokerAuthChallenges) { + challenge.complete(clientData); } - - ByteBuf cmd = - Commands.newAuthResponse(clientAuthMethod, clientData, protocolVersion, clientVersion); - toBrokerCnxFuture.thenAccept(toBrokerCnx -> toBrokerCnx.ctx().writeAndFlush(cmd) - .addListener(writeFuture -> { - if (writeFuture.isSuccess()) { - if (LOG.isDebugEnabled()) { - LOG.debug("{} authentication is refreshed successfully by {}, " - + "auth method: {} ", - toBrokerCnx.ctx().channel(), ctx.channel(), clientAuthMethod); - } - } else { - LOG.error("Failed to forward the auth response " - + "from the proxy to the broker through the proxy client, " - + "proxy: {}, proxy client: {}", - ctx.channel(), - toBrokerCnx.ctx().channel(), - writeFuture.cause()); - toBrokerCnx.ctx().channel().pipeline() - .fireExceptionCaught(writeFuture.cause()); - } - })) - .whenComplete((__, ex) -> { - if (ex != null) { - LOG.error("Failed to forward the auth response from the proxy to " - + "the broker through the proxy client, proxy: {}", - ctx().channel(), ex); - } - }); - }); + pendingBrokerAuthChallenges.clear(); + } } } catch (Exception e) { String errorMsg = "Unable to handleAuthResponse"; @@ -769,4 +830,36 @@ private void writeAndFlush(ByteBuf cmd) { private void writeAndFlushAndClose(ByteBuf cmd) { NettyChannelUtil.writeAndFlushWithClosePromise(ctx, cmd); } + + boolean supportsAuthenticationRefresh() { + return features != null && features.isSupportsAuthRefresh(); + } + + AuthData getClientAuthData() { + return clientAuthData; + } + + /** + * Thread-safe method to retrieve unexpired client auth data. Due to inherent race conditions, + * the auth data may expire before it is used. + */ + CompletableFuture getValidClientAuthData() { + final CompletableFuture clientAuthDataFuture = new CompletableFuture<>(); + ctx().executor().execute(Runnables.catchingAndLoggingThrowables(() -> { + // authState is not thread safe, so this must run on the ProxyConnection's event loop. + if (!authState.isExpired()) { + clientAuthDataFuture.complete(clientAuthData); + } else if (state == State.ProxyLookupRequests) { + maybeSendAuthChallenge(); + if (pendingBrokerAuthChallenges == null) { + pendingBrokerAuthChallenges = new HashSet<>(); + } + pendingBrokerAuthChallenges.add(clientAuthDataFuture); + } else { + clientAuthDataFuture.completeExceptionally(new PulsarClientException.AlreadyClosedException( + "ProxyConnection is not in a valid state to get client auth data for " + remoteAddress)); + } + })); + return clientAuthDataFuture; + } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java index d14105b0b43c2..bde989fc432f9 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.proxy.server; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.mockito.Mockito.spy; import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import io.jsonwebtoken.SignatureAlgorithm; @@ -31,13 +30,11 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import javax.crypto.SecretKey; -import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.ClientCnx; @@ -81,6 +78,9 @@ protected void doInitConf() throws Exception { conf.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); Properties properties = new Properties(); properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); + // The skew should be double the proxy's refresh interval to ensure the broker accepts auth data + // that the proxy might forward. + properties.setProperty("tokenAllowedClockSkewSeconds", "2"); conf.setProperties(properties); conf.setClusterName("proxy-authorization"); @@ -108,6 +108,7 @@ protected void setup() throws Exception { proxyConfig.setAuthenticationEnabled(true); proxyConfig.setAuthorizationEnabled(false); proxyConfig.setForwardAuthorizationCredentials(true); + proxyConfig.setAuthenticationRefreshCheckSeconds(1); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setAdvertisedAddress(null); @@ -162,14 +163,30 @@ public void testAuthDataRefresh(boolean forwardAuthData) throws Exception { .authentication(authenticationToken)); String topic = "persistent://my-tenant/my-ns/my-topic1"; - @Cleanup - Producer ignored = spy(pulsarClient.newProducer() - .topic(topic).create()); PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + pulsarClient.getPartitionsForTopic(topic).get(); Set> connections = pulsarClientImpl.getCnxPool().getConnections(); - Awaitility.await().during(4, SECONDS).untilAsserted(() -> { + Awaitility.await().during(5, SECONDS).untilAsserted(() -> { + pulsarClient.getPartitionsForTopic(topic).get(); + assertTrue(connections.stream().allMatch(n -> { + try { + ClientCnx clientCnx = n.get(); + long timestamp = clientCnx.getLastDisconnectedTimestamp(); + return timestamp == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + })); + }); + + // Force all connections from proxy to broker to close and therefore require the proxy to re-authenticate with + // the broker. (The client doesn't lose this connection.) + restartBroker(); + + // Rerun assertion to ensure that it still works + Awaitility.await().during(5, SECONDS).untilAsserted(() -> { pulsarClient.getPartitionsForTopic(topic).get(); assertTrue(connections.stream().allMatch(n -> { try { From 6514cddee5ef1f1ecf8a63fc78429d65cd5ffd36 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 12 Apr 2023 09:23:34 +0800 Subject: [PATCH 097/280] [improve][broker] PIP-192: Redirect the request to current load manager (#20058) --- .../loadbalance/LeaderElectionService.java | 9 +- .../broker/loadbalance/NoopLoadManager.java | 1 + .../extensions/BrokerRegistryImpl.java | 15 +-- .../extensions/ExtensibleLoadManagerImpl.java | 6 +- .../extensions/data/BrokerLookupData.java | 12 ++ .../filter/BrokerLoadManagerClassFilter.java | 50 ++++++++ .../extensions/manager/RedirectManager.java | 116 ++++++++++++++++++ .../impl/BrokerLoadManagerClassFilter.java | 41 +++++++ .../impl/ModularLoadManagerImpl.java | 2 + .../impl/SimpleLoadManagerImpl.java | 2 + .../broker/namespace/NamespaceService.java | 57 ++++++--- .../extensions/BrokerRegistryTest.java | 11 +- .../ExtensibleLoadManagerImplTest.java | 95 +++++++++++--- .../extensions/data/BrokerLookupDataTest.java | 4 +- .../filter/BrokerFilterTestBase.java | 8 +- .../BrokerIsolationPoliciesFilterTest.java | 4 +- .../BrokerLoadManagerClassFilterTest.java | 65 ++++++++++ .../scheduler/TransferShedderTest.java | 3 +- .../BrokerLoadManagerClassFilterTest.java | 73 +++++++++++ .../data/loadbalancer/ServiceLookupData.java | 4 + .../data/loadbalancer/LoadReport.java | 20 +++ .../data/loadbalancer/LocalBrokerData.java | 16 +++ 22 files changed, 568 insertions(+), 46 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java index 5ef95761e4ec6..05fe4353f3e76 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java @@ -37,7 +37,14 @@ public class LeaderElectionService implements AutoCloseable { public LeaderElectionService(CoordinationService cs, String localWebServiceAddress, Consumer listener) { - this.leaderElection = cs.getLeaderElection(LeaderBroker.class, ELECTION_ROOT, listener); + this(cs, localWebServiceAddress, ELECTION_ROOT, listener); + } + + public LeaderElectionService(CoordinationService cs, + String localWebServiceAddress, + String electionRoot, + Consumer listener) { + this.leaderElection = cs.getLeaderElection(LeaderBroker.class, electionRoot, listener); this.localValue = new LeaderBroker(localWebServiceAddress); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java index e8c0567fd0c8c..0de2ae92db61a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java @@ -65,6 +65,7 @@ public void start() throws PulsarServerException { pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), pulsar.getAdvertisedListeners()); localData.setProtocols(pulsar.getProtocolDataToAdvertise()); + localData.setLoadManagerClassName(this.pulsar.getConfig().getLoadManagerClassName()); String brokerReportPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; try { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java index 4c2beba5d30b3..921ce35b5c65e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; +import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -51,8 +51,6 @@ @Slf4j public class BrokerRegistryImpl implements BrokerRegistry { - protected static final String LOOKUP_DATA_PATH = "/loadbalance/extension/brokers"; - private final PulsarService pulsar; private final ServiceConfiguration conf; @@ -94,6 +92,8 @@ public BrokerRegistryImpl(PulsarService pulsar) { pulsar.getProtocolDataToAdvertise(), pulsar.getConfiguration().isEnablePersistentTopics(), pulsar.getConfiguration().isEnableNonPersistentTopics(), + conf.getLoadManagerClassName(), + System.currentTimeMillis(), pulsar.getBrokerVersion()); this.state = State.Init; } @@ -151,7 +151,7 @@ public String getBrokerId() { @Override public CompletableFuture> getAvailableBrokersAsync() { this.checkState(); - return brokerLookupDataLockManager.listLocks(LOOKUP_DATA_PATH).thenApply(Lists::newArrayList); + return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenApply(ArrayList::new); } @Override @@ -215,7 +215,7 @@ private void handleMetadataStoreNotification(Notification t) { return; } this.scheduler.submit(() -> { - String brokerId = t.getPath().substring(LOOKUP_DATA_PATH.length() + 1); + String brokerId = t.getPath().substring(LOADBALANCE_BROKERS_ROOT.length() + 1); for (BiConsumer listener : listeners) { listener.accept(brokerId, t.getType()); } @@ -227,12 +227,13 @@ private void handleMetadataStoreNotification(Notification t) { @VisibleForTesting protected static boolean isVerifiedNotification(Notification t) { - return t.getPath().startsWith(LOOKUP_DATA_PATH + "/") && t.getPath().length() > LOOKUP_DATA_PATH.length() + 1; + return t.getPath().startsWith(LOADBALANCE_BROKERS_ROOT + "/") + && t.getPath().length() > LOADBALANCE_BROKERS_ROOT.length() + 1; } @VisibleForTesting protected static String keyPath(String brokerId) { - return String.format("%s/%s", LOOKUP_DATA_PATH, brokerId); + return String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, brokerId); } private void checkState() throws IllegalStateException { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index e054a1bc662c7..4ebf537f7a8a8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -52,6 +52,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerLoadManagerClassFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager; @@ -106,6 +107,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000; + private static final String ELECTION_ROOT = "/loadbalance/extension/leader"; + private PulsarService pulsar; private ServiceConfiguration conf; @@ -187,6 +190,7 @@ public enum Role { */ public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline = new ArrayList<>(); + this.brokerFilterPipeline.add(new BrokerLoadManagerClassFilter()); this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); // TODO: Make brokerSelectionStrategy configurable. @@ -215,7 +219,7 @@ public void start() throws PulsarServerException { } this.brokerRegistry = new BrokerRegistryImpl(pulsar); this.leaderElectionService = new LeaderElectionService( - pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), + pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), ELECTION_ROOT, state -> { pulsar.getLoadManagerExecutor().execute(() -> { if (state == LeaderElectionState.Leading) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java index 4c9e503129e66..41f5b18e321e8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java @@ -36,6 +36,8 @@ public record BrokerLookupData (String webServiceUrl, Map protocols, boolean persistentTopicsEnabled, boolean nonPersistentTopicsEnabled, + String loadManagerClassName, + long startTimestamp, String brokerVersion) implements ServiceLookupData { @Override public String getWebServiceUrl() { @@ -67,6 +69,16 @@ public Optional getProtocol(String protocol) { return Optional.ofNullable(this.protocols().get(protocol)); } + @Override + public String getLoadManagerClassName() { + return this.loadManagerClassName; + } + + @Override + public long getStartTimestamp() { + return this.startTimestamp; + } + public LookupResult toLookupResult() { return new LookupResult(webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, LookupResult.Type.BrokerUrl, false); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java new file mode 100644 index 0000000000000..4ee28a5225a0d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import java.util.Map; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.naming.ServiceUnitId; + +public class BrokerLoadManagerClassFilter implements BrokerFilter { + + public static final String FILTER_NAME = "broker_load_manager_class_filter"; + @Override + public String name() { + return FILTER_NAME; + } + + @Override + public Map filter( + Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) + throws BrokerFilterException { + if (brokers.isEmpty()) { + return brokers; + } + brokers.entrySet().removeIf(entry -> { + BrokerLookupData v = entry.getValue(); + return !v.getLoadManagerClassName().equals(context.brokerConfiguration().getLoadManagerClassName()); + }); + return brokers; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java new file mode 100644 index 0000000000000..4aff77937a5b4 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.coordination.LockManager; +import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; + +@Slf4j +public class RedirectManager { + private final PulsarService pulsar; + + private final LockManager brokerLookupDataLockManager; + + + public RedirectManager(PulsarService pulsar) { + this.pulsar = pulsar; + this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class); + } + + public CompletableFuture> getAvailableBrokerLookupDataAsync() { + return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenCompose(availableBrokers -> { + Map map = new ConcurrentHashMap<>(); + List> futures = new ArrayList<>(); + for (String brokerId : availableBrokers) { + futures.add(this.brokerLookupDataLockManager.readLock( + String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, brokerId)).thenAccept(lookupDataOpt -> { + if (lookupDataOpt.isPresent()) { + map.put(brokerId, lookupDataOpt.get()); + } else { + log.warn("Got an empty lookup data, brokerId: {}", brokerId); + } + })); + } + + return FutureUtil.waitForAll(futures).thenApply(__ -> map); + }); + } + + public CompletableFuture> findRedirectLookupResultAsync() { + String currentLMClassName = pulsar.getConfiguration().getLoadManagerClassName(); + boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfig(), log); + return getAvailableBrokerLookupDataAsync().thenApply(lookupDataMap -> { + if (lookupDataMap.isEmpty()) { + String errorMsg = "No available broker found."; + log.warn(errorMsg); + throw new IllegalStateException(errorMsg); + } + AtomicReference latestServiceLookupData = new AtomicReference<>(); + AtomicLong lastStartTimestamp = new AtomicLong(0L); + lookupDataMap.forEach((key, value) -> { + if (lastStartTimestamp.get() <= value.getStartTimestamp()) { + lastStartTimestamp.set(value.getStartTimestamp()); + latestServiceLookupData.set(value); + } + }); + if (latestServiceLookupData.get() == null) { + String errorMsg = "No latest service lookup data found."; + log.warn(errorMsg); + throw new IllegalStateException(errorMsg); + } + if (latestServiceLookupData.get().getLoadManagerClassName().equals(currentLMClassName)) { + if (debug) { + log.info("We don't need to redirect, current load manager class name: {}", + currentLMClassName); + } + return Optional.empty(); + } + var serviceLookupDataObj = latestServiceLookupData.get(); + var candidateBrokers = new ArrayList(); + lookupDataMap.forEach((key, value) -> { + if (value.getLoadManagerClassName().equals(serviceLookupDataObj.getLoadManagerClassName())) { + candidateBrokers.add(value); + } + }); + var selectedBroker = candidateBrokers.get((int) (Math.random() * candidateBrokers.size())); + + return Optional.of(new LookupResult(selectedBroker.getWebServiceUrl(), + selectedBroker.getWebServiceUrlTls(), + selectedBroker.getPulsarServiceUrl(), + selectedBroker.getPulsarServiceUrlTls(), + true)); + }); + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java new file mode 100644 index 0000000000000..5d6a56ba86960 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.impl; + +import java.util.Set; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilter; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.policies.data.loadbalancer.BundleData; + +public class BrokerLoadManagerClassFilter implements BrokerFilter { + + @Override + public void filter(Set brokers, BundleData bundleToAssign, + LoadData loadData, + ServiceConfiguration conf) throws BrokerFilterException { + loadData.getBrokerData().forEach((key, value) -> { + if (!value.getLocalData().getLoadManagerClassName() + .equals(conf.getLoadManagerClassName())) { + brokers.remove(key); + } + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 4756b885ff217..c6f406a7c8a5c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -267,6 +267,7 @@ public void initialize(final PulsarService pulsar) { placementStrategy = ModularLoadManagerStrategy.create(conf); policies = new SimpleResourceAllocationPolicies(pulsar); + filterPipeline.add(new BrokerLoadManagerClassFilter()); filterPipeline.add(new BrokerVersionFilter()); LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap); @@ -960,6 +961,7 @@ public void start() throws PulsarServerException { // configure broker-topic mode localData.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics()); localData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics()); + localData.setLoadManagerClassName(conf.getLoadManagerClassName()); String lookupServiceAddress = pulsar.getLookupServiceAddress(); brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index c98857b2fc44c..5e99456971147 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -1090,6 +1090,8 @@ private LoadReport generateLoadReportForcefully() throws Exception { loadReport.setSystemResourceUsage(systemResourceUsage); loadReport.setBundleStats(pulsar.getBrokerService().getBundleStats()); loadReport.setTimestamp(System.currentTimeMillis()); + loadReport.setLoadManagerClassName(pulsar.getConfig().getLoadManagerClassName()); + loadReport.setStartTimestamp(System.currentTimeMillis()); final Set oldBundles = lastLoadReport.getBundles(); final Set newBundles = loadReport.getBundles(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index c9cd33d6d87c8..a0e2bf7534c02 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -56,6 +56,7 @@ import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.ResourceUnit; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.manager.RedirectManager; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; @@ -136,6 +137,7 @@ public class NamespaceService implements AutoCloseable { private final ConcurrentOpenHashMap namespaceClients; private final List bundleOwnershipListeners; + private final RedirectManager redirectManager; private static final Counter lookupRedirects = Counter.build("pulsar_broker_lookup_redirects", "-").register(); @@ -164,6 +166,7 @@ public NamespaceService(PulsarService pulsar) { ConcurrentOpenHashMap.newBuilder().build(); this.bundleOwnershipListeners = new CopyOnWriteArrayList<>(); this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class); + this.redirectManager = new RedirectManager(pulsar); } public void initialize() { @@ -177,12 +180,20 @@ public CompletableFuture> getBrokerServiceUrlAsync(TopicN CompletableFuture> future = getBundleAsync(topic) .thenCompose(bundle -> { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { - return loadManager.get().findBrokerServiceUrl(Optional.of(topic), bundle); - } else { - // TODO: Add unit tests cover it. - return findBrokerServiceUrl(bundle, options); - } + // Do redirection if the cluster is in rollback or deploying. + return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> { + if (optResult.isPresent()) { + LOG.info("[{}] Redirect lookup request to {} for topic {}", + pulsar.getSafeWebServiceAddress(), optResult.get(), topic); + return CompletableFuture.completedFuture(optResult); + } + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + return loadManager.get().findBrokerServiceUrl(Optional.of(topic), bundle); + } else { + // TODO: Add unit tests cover it. + return findBrokerServiceUrl(bundle, options); + } + }); }); future.thenAccept(optResult -> { @@ -271,22 +282,40 @@ private CompletableFuture> internalGetWebServiceUrl(Optional { - if (lookupResult.isPresent()) { + return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> { + if (optResult.isPresent()) { + LOG.info("[{}] Redirect lookup request to {} for topic {}", + pulsar.getSafeWebServiceAddress(), optResult.get(), topic); try { - LookupData lookupData = lookupResult.get().getLookupData(); + LookupData lookupData = optResult.get().getLookupData(); final String redirectUrl = options.isRequestHttps() ? lookupData.getHttpUrlTls() : lookupData.getHttpUrl(); - return Optional.of(new URL(redirectUrl)); + return CompletableFuture.completedFuture(Optional.of(new URL(redirectUrl))); } catch (Exception e) { // just log the exception, nothing else to do LOG.warn("internalGetWebServiceUrl [{}]", e.getMessage(), e); } + return CompletableFuture.completedFuture(Optional.empty()); } - return Optional.empty(); + CompletableFuture> future = + ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config) + ? loadManager.get().findBrokerServiceUrl(topic, bundle) : + findBrokerServiceUrl(bundle, options); + + return future.thenApply(lookupResult -> { + if (lookupResult.isPresent()) { + try { + LookupData lookupData = lookupResult.get().getLookupData(); + final String redirectUrl = options.isRequestHttps() + ? lookupData.getHttpUrlTls() : lookupData.getHttpUrl(); + return Optional.of(new URL(redirectUrl)); + } catch (Exception e) { + // just log the exception, nothing else to do + LOG.warn("internalGetWebServiceUrl [{}]", e.getMessage(), e); + } + } + return Optional.empty(); + }); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java index 05a156f87de11..26986a494f0be 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; +import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; @@ -379,19 +380,19 @@ public void testCloseRegister() throws Exception { public void testIsVerifiedNotification() { assertFalse(BrokerRegistryImpl.isVerifiedNotification(new Notification(NotificationType.Created, "/"))); assertFalse(BrokerRegistryImpl.isVerifiedNotification(new Notification(NotificationType.Created, - BrokerRegistryImpl.LOOKUP_DATA_PATH + "xyz"))); + LOADBALANCE_BROKERS_ROOT + "xyz"))); assertFalse(BrokerRegistryImpl.isVerifiedNotification(new Notification(NotificationType.Created, - BrokerRegistryImpl.LOOKUP_DATA_PATH))); + LOADBALANCE_BROKERS_ROOT))); assertTrue(BrokerRegistryImpl.isVerifiedNotification( - new Notification(NotificationType.Created, BrokerRegistryImpl.LOOKUP_DATA_PATH + "/brokerId"))); + new Notification(NotificationType.Created, LOADBALANCE_BROKERS_ROOT + "/brokerId"))); assertTrue(BrokerRegistryImpl.isVerifiedNotification( - new Notification(NotificationType.Created, BrokerRegistryImpl.LOOKUP_DATA_PATH + "/brokerId/xyz"))); + new Notification(NotificationType.Created, LOADBALANCE_BROKERS_ROOT + "/brokerId/xyz"))); } @Test public void testKeyPath() { String keyPath = BrokerRegistryImpl.keyPath("brokerId"); - assertEquals(keyPath, BrokerRegistryImpl.LOOKUP_DATA_PATH + "/brokerId"); + assertEquals(keyPath, LOADBALANCE_BROKERS_ROOT + "/brokerId"); } public BrokerRegistryImpl.State getState(BrokerRegistryImpl brokerRegistry) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index ae7ceeed92838..9ab2467d0ef1b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -473,7 +473,8 @@ public Map filter(Map broker } @Test - public void testStartOldLoadManager() throws Exception { + public void testDeployAndRollbackLoadManager() throws Exception { + // Test rollback to modular load manager. ServiceConfiguration defaultConf = getDefaultConf(); defaultConf.setAllowAutoTopicCreation(true); defaultConf.setForceDeleteNamespaceAllowed(true); @@ -482,20 +483,86 @@ public void testStartOldLoadManager() throws Exception { try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { // start pulsar3 with old load manager var pulsar3 = additionalPulsarTestContext.getPulsarService(); + String topic = "persistent://public/default/test"; + + String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl()); + + String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult1, lookupResult2); + assertEquals(lookupResult1, lookupResult3); + + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get("test")).get(); + LookupOptions options = LookupOptions.builder() + .authoritative(false) + .requestHttps(false) + .readOnly(false) + .loadTopicsInBundle(false).build(); + Optional webServiceUrl1 = + pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl1.isPresent()); + assertEquals(webServiceUrl1.get().toString(), pulsar3.getWebServiceAddress()); + + Optional webServiceUrl2 = + pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl2.isPresent()); + assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); + + Optional webServiceUrl3 = + pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl3.isPresent()); + assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString()); + + // Test deploy new broker with new load manager + ServiceConfiguration conf = getDefaultConf(); + conf.setAllowAutoTopicCreation(true); + conf.setForceDeleteNamespaceAllowed(true); + conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + try (var additionPulsarTestContext = createAdditionalPulsarTestContext(conf)) { + var pulsar4 = additionPulsarTestContext.getPulsarService(); + + Set availableCandidates = Sets.newHashSet(pulsar1.getBrokerServiceUrl(), + pulsar2.getBrokerServiceUrl(), + pulsar4.getBrokerServiceUrl()); + String lookupResult4 = pulsar4.getAdminClient().lookups().lookupTopic(topic); + assertTrue(availableCandidates.contains(lookupResult4)); + + String lookupResult5 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResult6 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + String lookupResult7 = pulsar3.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult4, lookupResult5); + assertEquals(lookupResult4, lookupResult6); + assertEquals(lookupResult4, lookupResult7); + + Set availableWebUrlCandidates = Sets.newHashSet(pulsar1.getWebServiceAddress(), + pulsar2.getWebServiceAddress(), + pulsar4.getWebServiceAddress()); + + webServiceUrl1 = + pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl1.isPresent()); + assertTrue(availableWebUrlCandidates.contains(webServiceUrl1.get().toString())); + + webServiceUrl2 = + pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl2.isPresent()); + assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); + + // The pulsar3 will redirect to pulsar4 + webServiceUrl3 = + pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl3.isPresent()); + // It will redirect to pulsar4 + assertTrue(availableWebUrlCandidates.contains(webServiceUrl3.get().toString())); + + var webServiceUrl4 = + pulsar4.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl4.isPresent()); + assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString()); - var availableBrokers = pulsar3.getLoadManager().get().getAvailableBrokers(); - assertEquals(availableBrokers.size(), 1); - assertEquals(availableBrokers.iterator().next(), pulsar3.getLookupServiceAddress()); - - availableBrokers = pulsar1.getLoadManager().get().getAvailableBrokers(); - assertEquals(availableBrokers.size(), 2); - assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress())); - assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress())); - - availableBrokers = pulsar2.getLoadManager().get().getAvailableBrokers(); - assertEquals(availableBrokers.size(), 2); - assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress())); - assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress())); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java index b5d2d63a31156..7bcd0687f0008 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.data; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.junit.Assert; @@ -41,7 +42,8 @@ public void testConstructors() { }}; BrokerLookupData lookupData = new BrokerLookupData( webServiceUrl, webServiceUrlTls, pulsarServiceUrl, - pulsarServiceUrlTls, advertisedListeners, protocols, true, true, "3.0"); + pulsarServiceUrlTls, advertisedListeners, protocols, true, true, + ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(),"3.0"); Assert.assertEquals(webServiceUrl, lookupData.webServiceUrl()); Assert.assertEquals(webServiceUrlTls, lookupData.webServiceUrlTls()); Assert.assertEquals(pulsarServiceUrl, lookupData.pulsarServiceUrl()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java index c521861471c9a..68bd7b29094cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java @@ -30,6 +30,7 @@ import java.util.function.BiConsumer; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -105,6 +106,10 @@ public BrokerLookupData getLookupData() { } public BrokerLookupData getLookupData(String version) { + return getLookupData(version, ExtensibleLoadManagerImpl.class.getName()); + } + + public BrokerLookupData getLookupData(String version, String loadManagerClassName) { String webServiceUrl = "http://localhost:8080"; String webServiceUrlTls = "https://localhoss:8081"; String pulsarServiceUrl = "pulsar://localhost:6650"; @@ -115,6 +120,7 @@ public BrokerLookupData getLookupData(String version) { }}; return new BrokerLookupData( webServiceUrl, webServiceUrlTls, pulsarServiceUrl, - pulsarServiceUrlTls, advertisedListeners, protocols, true, true, version); + pulsarServiceUrlTls, advertisedListeners, protocols, true, true, + loadManagerClassName, -1, version); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index 5bc1e436bae25..c2c534f72e9db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -33,6 +33,7 @@ import java.util.Set; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; @@ -211,7 +212,8 @@ public BrokerLookupData getLookupData(boolean persistentTopicsEnabled, return new BrokerLookupData( webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, advertisedListeners, protocols, - persistentTopicsEnabled, nonPersistentTopicsEnabled, "3.0.0"); + persistentTopicsEnabled, nonPersistentTopicsEnabled, + ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(), "3.0.0"); } public LoadManagerContext getContext() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java new file mode 100644 index 0000000000000..0169b57fe993e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import static org.testng.Assert.assertEquals; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.testng.annotations.Test; +import java.util.HashMap; +import java.util.Map; + + +/** + * Unit test for {@link BrokerLoadManagerClassFilter}. + */ +public class BrokerLoadManagerClassFilterTest extends BrokerFilterTestBase { + + @Test + public void test() throws BrokerFilterException { + LoadManagerContext context = getContext(); + context.brokerConfiguration().setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter(); + + Map originalBrokers = Map.of( + "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), + "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), + "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), + "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()) + ); + + Map result = filter.filter(new HashMap<>(originalBrokers), null, context); + assertEquals(result, Map.of( + "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), + "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()) + )); + + context.brokerConfiguration().setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + result = filter.filter(new HashMap<>(originalBrokers), null, context); + + assertEquals(result, Map.of( + "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), + "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()) + )); + + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index d673722d6fb44..af5890fcacbb2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -664,7 +664,8 @@ public BrokerLookupData getLookupData() { return new BrokerLookupData( webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, advertisedListeners, protocols, - true, true, "3.0.0"); + true, true, + conf.getLoadManagerClassName(), System.currentTimeMillis(), "3.0.0"); } private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java new file mode 100644 index 0000000000000..856bbac029226 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.policies.data.loadbalancer.BrokerData; +import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; +import org.testng.annotations.Test; +import java.util.HashSet; +import java.util.Set; + +/** + * Unit test for {@link BrokerLoadManagerClassFilter}. + */ +public class BrokerLoadManagerClassFilterTest { + + @Test + public void test() throws BrokerFilterException { + BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter(); + + LoadData loadData = new LoadData(); + LocalBrokerData localBrokerData = new LocalBrokerData(); + localBrokerData.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + + LocalBrokerData localBrokerData1 = new LocalBrokerData(); + localBrokerData1.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + loadData.getBrokerData().put("broker1", new BrokerData(localBrokerData)); + loadData.getBrokerData().put("broker2", new BrokerData(localBrokerData1)); + + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + + Set brokers = new HashSet<>(){{ + add("broker1"); + add("broker2"); + }}; + filter.filter(brokers, null, loadData, conf); + + assertEquals(brokers.size(), 1); + assertTrue(brokers.contains("broker1")); + + brokers = new HashSet<>(){{ + add("broker1"); + add("broker2"); + }}; + conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + filter.filter(brokers, null, loadData, conf); + assertEquals(brokers.size(), 1); + assertTrue(brokers.contains("broker2")); + } +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java index 2d138edd924bc..bf4a11c8f875d 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java @@ -48,4 +48,8 @@ public interface ServiceLookupData { */ Optional getProtocol(String protocol); + String getLoadManagerClassName(); + + long getStartTimestamp(); + } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java index 75fddc21bf91b..6e519a3f0735f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java @@ -55,6 +55,8 @@ public class LoadReport implements LoadManagerReport { private int numProducers; private int numBundles; private Map protocols; + private String loadManagerClassName; + private long startTimestamp; // This place-holder requires to identify correct LoadManagerReport type while deserializing @SuppressWarnings("checkstyle:ConstantName") public static final String loadReportType = LoadReport.class.getSimpleName(); @@ -474,4 +476,22 @@ public void setProtocols(Map protocols) { public Optional getProtocol(String protocol) { return Optional.ofNullable(protocols.get(protocol)); } + + @Override + public String getLoadManagerClassName() { + return this.loadManagerClassName; + } + + public void setLoadManagerClassName(String loadManagerClassName) { + this.loadManagerClassName = loadManagerClassName; + } + + @Override + public long getStartTimestamp() { + return this.startTimestamp; + } + + public void setStartTimestamp(long startTimestamp) { + this.startTimestamp = startTimestamp; + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java index 8c0c008e0a555..df85a4d989f99 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java @@ -91,6 +91,9 @@ public class LocalBrokerData implements LoadManagerReport { // private Map advertisedListeners; + private String loadManagerClassName; + private long startTimestamp; + // For JSON only. public LocalBrokerData() { this(null, null, null, null); @@ -113,6 +116,7 @@ public LocalBrokerData(final String webServiceUrl, final String webServiceUrlTls this.pulsarServiceUrlTls = pulsarServiceUrlTls; lastStats = new ConcurrentHashMap<>(); lastUpdate = System.currentTimeMillis(); + startTimestamp = System.currentTimeMillis(); cpu = new ResourceUsage(); memory = new ResourceUsage(); directMemory = new ResourceUsage(); @@ -529,4 +533,16 @@ public Map getAdvertisedListeners() { public void setAdvertisedListeners(Map advertisedListeners) { this.advertisedListeners = advertisedListeners; } + + public String getLoadManagerClassName() { + return this.loadManagerClassName; + } + + public void setLoadManagerClassName(String loadManagerClassName) { + this.loadManagerClassName = loadManagerClassName; + } + + public long getStartTimestamp() { + return this.startTimestamp; + } } From 18ccd9ef22bb44d952875460fbacad45c8d8d26a Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Tue, 11 Apr 2023 19:28:25 -0700 Subject: [PATCH 098/280] [fix] Upgrade Vertx to match BK's version (otherwise BK does not work) (#20070) ### Motivation BK deployed with Pulsar fail on start with ``` ERROR org.apache.bookkeeper.common.component.AbstractLifecycleComponent - Failed to start Component: http-service java.lang.NoSuchMethodError: 'io.vertx.core.Future io.vertx.core.Vertx.deployVerticle(io.vertx.core.Verticle)' at org.apache.bookkeeper.http.vertx.VertxHttpServer.startServer(VertxHttpServer.java:93) ~[org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar:4.16.0] at org.apache.bookkeeper.server.service.HttpService.doStart(HttpService.java:62) ~[org.apache.bookkeeper-bookkeeper-server-4.16.0.jar:4.16.0] at org.apache.bookkeeper.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:83) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0] at org.apache.bookkeeper.common.component.LifecycleComponentStack.lambda$start$4(LifecycleComponentStack.java:144) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0] at com.google.common.collect.ImmutableList.forEach(ImmutableList.java:422) ~[com.google.guava-guava-31.0.1-jre.jar:?] at org.apache.bookkeeper.common.component.LifecycleComponentStack.start(LifecycleComponentStack.java:144) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0] at org.apache.bookkeeper.common.component.ComponentStarter.startComponent(ComponentStarter.java:84) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0] at org.apache.bookkeeper.server.Main.doMain(Main.java:224) ~[org.apache.bookkeeper-bookkeeper-server-4.16.0.jar:4.16.0] at org.apache.bookkeeper.server.Main.main(Main.java:199) ~[org.apache.bookkeeper-bookkeeper-server-4.16.0.jar:4.16.0] ``` The root cause is that BK upgraded to 4.16.0 and needs newer version of vertx that introduced API changes https://github.com/apache/bookkeeper/pull/3435 Pulsar forces older version of dependency. ### Modifications Upgrade Vertx to the same version as BK --- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 4cb1d1544a63e..0d5cd7c02a23a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -470,11 +470,11 @@ The Apache Software License, Version 2.0 * JCTools - Java Concurrency Tools for the JVM - org.jctools-jctools-core-2.1.2.jar * Vertx - - io.vertx-vertx-auth-common-3.9.8.jar - - io.vertx-vertx-bridge-common-3.9.8.jar - - io.vertx-vertx-core-3.9.8.jar - - io.vertx-vertx-web-3.9.8.jar - - io.vertx-vertx-web-common-3.9.8.jar + - io.vertx-vertx-auth-common-4.3.8.jar + - io.vertx-vertx-bridge-common-4.3.8.jar + - io.vertx-vertx-core-4.3.8.jar + - io.vertx-vertx-web-4.3.8.jar + - io.vertx-vertx-web-common-4.3.8.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-3.8.1.jar - org.apache.zookeeper-zookeeper-jute-3.8.1.jar diff --git a/pom.xml b/pom.xml index 389d9f6f83487..404807bc4ad7c 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ flexible messaging model and an intuitive client API. 2.34 1.10.50 0.16.0 - 3.9.8 + 4.3.8 6.29.4.1 1.7.32 4.4 From cff3f9baa03f166697d31ff517f234ecf0fc702d Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Wed, 12 Apr 2023 11:00:14 +0800 Subject: [PATCH 099/280] [fix][client] Cache empty schema version in ProducerImpl schemaCache. (#19929) Co-authored-by: wangjinlong --- .../pulsar/client/api/MockBrokerService.java | 21 ++++ .../client/api/MockBrokerServiceHooks.java | 5 + .../impl/ProducerEmptySchemaCacheTest.java | 100 ++++++++++++++++++ .../pulsar/client/impl/ProducerImpl.java | 23 +++- 4 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java index 3e6ee4f2a969d..9ca0bafe00b3b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java @@ -44,6 +44,7 @@ import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandCloseProducerHook; import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandConnectHook; import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandFlowHook; +import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandGetOrCreateSchemaHook; import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandPartitionLookupHook; import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandProducerHook; import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandSendHook; @@ -55,6 +56,7 @@ import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.api.proto.CommandConnect; import org.apache.pulsar.common.api.proto.CommandFlow; +import org.apache.pulsar.common.api.proto.CommandGetOrCreateSchema; import org.apache.pulsar.common.api.proto.CommandLookupTopic; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata; @@ -77,6 +79,7 @@ import org.slf4j.LoggerFactory; /** + * */ public class MockBrokerService { private LookupData lookupData; @@ -244,6 +247,19 @@ protected void handleCloseConsumer(CommandCloseConsumer closeConsumer) { ctx.writeAndFlush(Commands.newSuccess(closeConsumer.getRequestId())); } + @Override + protected void handleGetOrCreateSchema(CommandGetOrCreateSchema commandGetOrCreateSchema) { + if (handleGetOrCreateSchema != null) { + handleGetOrCreateSchema.apply(ctx, commandGetOrCreateSchema); + return; + } + + // default + ctx.writeAndFlush( + Commands.newGetOrCreateSchemaResponse(commandGetOrCreateSchema.getRequestId(), + SchemaVersion.Empty)); + } + @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { log.warn("Got exception", cause); @@ -276,6 +292,7 @@ final protected void handlePong(CommandPong pong) { private CommandUnsubscribeHook handleUnsubscribe = null; private CommandCloseProducerHook handleCloseProducer = null; private CommandCloseConsumerHook handleCloseConsumer = null; + private CommandGetOrCreateSchemaHook handleGetOrCreateSchema = null; public MockBrokerService() { server = new Server(0); @@ -416,6 +433,10 @@ public void setHandleCloseConsumer(CommandCloseConsumerHook hook) { handleCloseConsumer = hook; } + public void setHandleGetOrCreateSchema(CommandGetOrCreateSchemaHook hook) { + handleGetOrCreateSchema = hook; + } + public void resetHandleCloseConsumer() { handleCloseConsumer = null; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java index 06289e123c8b9..dd2058ce5a485 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java @@ -26,6 +26,7 @@ import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.api.proto.CommandConnect; import org.apache.pulsar.common.api.proto.CommandFlow; +import org.apache.pulsar.common.api.proto.CommandGetOrCreateSchema; import org.apache.pulsar.common.api.proto.CommandLookupTopic; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata; import org.apache.pulsar.common.api.proto.CommandProducer; @@ -77,4 +78,8 @@ interface CommandCloseProducerHook { interface CommandCloseConsumerHook { void apply(ChannelHandlerContext ctx, CommandCloseConsumer closeConsumer); } + + interface CommandGetOrCreateSchemaHook { + void apply(ChannelHandlerContext ctx, CommandGetOrCreateSchema closeConsumer); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java new file mode 100644 index 0000000000000..079b8bd8eccbd --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import lombok.Cleanup; +import org.apache.pulsar.client.api.MockBrokerService; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.testng.Assert.assertEquals; + +@Test(groups = "broker-impl") +public class ProducerEmptySchemaCacheTest { + + MockBrokerService mockBrokerService; + + @BeforeClass(alwaysRun = true) + public void setup() { + mockBrokerService = new MockBrokerService(); + mockBrokerService.start(); + } + + @AfterClass(alwaysRun = true) + public void teardown() { + mockBrokerService.stop(); + } + + @Test + public void testProducerShouldCacheEmptySchema() throws Exception { + @Cleanup + PulsarClientImpl client = (PulsarClientImpl) PulsarClient.builder() + .serviceUrl(mockBrokerService.getBrokerAddress()) + .build(); + + AtomicLong counter = new AtomicLong(0); + + mockBrokerService.setHandleGetOrCreateSchema((ctx, commandGetOrCreateSchema) -> { + counter.incrementAndGet(); + ctx.writeAndFlush( + Commands.newGetOrCreateSchemaResponse(commandGetOrCreateSchema.getRequestId(), + SchemaVersion.Empty)); + }); + + // this schema mode is used in consumer retry and dlq Producer + // when the origin consumer has Schema.BYTES schema + // and when retry message or dlq message is send + // will use typed message builder set Schema.Bytes to send message. + + Schema schema = Schema.BYTES; + Schema readerSchema = Schema.BYTES; + + @Cleanup + Producer dlqProducer = client.newProducer(Schema.AUTO_PRODUCE_BYTES(schema)) + .topic("testAutoProduceBytesSchemaShouldCache") + .sendTimeout(5, TimeUnit.SECONDS) + .maxPendingMessages(0) + .enableBatching(false) + .create(); + + for (int i = 10; i > 0; i--) { + TypedMessageBuilder typedMessageBuilderNew = + dlqProducer.newMessage(Schema.AUTO_PRODUCE_BYTES(readerSchema)) + .value("hello".getBytes()); + + typedMessageBuilderNew.send(); + } + + // schema should only be requested once. + // and if the schemaVersion is empty (e.g. Schema.BYTES) + // it should be cached by the client + // to avoid continuously send `CommandGetOrCreateSchema` rpc + assertEquals(counter.get(), 1); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index b8f249dbc0e4b..2192ebfb64e75 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -88,6 +88,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Commands.ChecksumType; import org.apache.pulsar.common.protocol.schema.SchemaHash; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.DateFormatter; @@ -733,11 +734,16 @@ boolean populateMessageSchema(MessageImpl msg, SendCallback callback) { completeCallbackAndReleaseSemaphore(msg.getUncompressedSize(), callback, e); return false; } + byte[] schemaVersion = schemaCache.get(msg.getSchemaHash()); if (schemaVersion != null) { - msgMetadataBuilder.setSchemaVersion(schemaVersion); + if (schemaVersion != SchemaVersion.Empty.bytes()) { + msgMetadataBuilder.setSchemaVersion(schemaVersion); + } + msg.setSchemaState(MessageImpl.SchemaState.Ready); } + return true; } @@ -746,7 +752,11 @@ private boolean rePopulateMessageSchema(MessageImpl msg) { if (schemaVersion == null) { return false; } - msg.getMessageBuilder().setSchemaVersion(schemaVersion); + + if (schemaVersion != SchemaVersion.Empty.bytes()) { + msg.getMessageBuilder().setSchemaVersion(schemaVersion); + } + msg.setSchemaState(MessageImpl.SchemaState.Ready); return true; } @@ -769,12 +779,15 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call } } else { log.info("[{}] [{}] GetOrCreateSchema succeed", topic, producerName); - // In broker, if schema version is an empty byte array, it means the topic doesn't have schema. In this - // case, we should not cache the schema version so that the schema version of the message metadata will - // be null, instead of an empty array. + // In broker, if schema version is an empty byte array, it means the topic doesn't have schema. + // In this case, we cache the schema version to `SchemaVersion.Empty.bytes()`. + // When we need to set the schema version of the message metadata, + // we should check if the cached schema version is `SchemaVersion.Empty.bytes()` if (v.length != 0) { schemaCache.putIfAbsent(msg.getSchemaHash(), v); msg.getMessageBuilder().setSchemaVersion(v); + } else { + schemaCache.putIfAbsent(msg.getSchemaHash(), SchemaVersion.Empty.bytes()); } msg.setSchemaState(MessageImpl.SchemaState.Ready); } From 1e9c7f5236866d07105421c566241e3798f63097 Mon Sep 17 00:00:00 2001 From: Apurva007 Date: Tue, 11 Apr 2023 20:07:45 -0700 Subject: [PATCH 100/280] [Fix][Doc] Update docs for Partition increase command (#20035) Co-authored-by: Apurva Telang --- .../pulsar/broker/admin/v1/PersistentTopics.java | 4 ++-- .../pulsar/broker/admin/v2/PersistentTopics.java | 4 ++-- .../java/org/apache/pulsar/client/admin/Topics.java | 12 ++++++------ .../apache/pulsar/admin/cli/CmdPersistentTopics.java | 2 +- .../java/org/apache/pulsar/admin/cli/CmdTopics.java | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 16f1b357dcd38..e9bb1a4054764 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -256,14 +256,14 @@ public void createNonPartitionedTopic( } /** - * It updates number of partitions of an existing non-global partitioned topic. It requires partitioned-topic to be + * It updates number of partitions of an existing partitioned topic. It requires partitioned-topic to be * already exist and number of new partitions must be greater than existing number of partitions. Decrementing * number of partitions requires deletion of topic which is not supported. */ @POST @Path("/{property}/{cluster}/{namespace}/{topic}/partitions") @ApiOperation(hidden = true, value = "Increment partitions of an existing partitioned topic.", - notes = "It only increments partitions of existing non-global partitioned-topic") + notes = "It increments partitions of existing partitioned-topic") @ApiResponses(value = { @ApiResponse(code = 204, message = "Update topic partition successful."), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 477cc8b57129d..eee363aeed86b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -775,14 +775,14 @@ public void deleteDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncRe } /** - * It updates number of partitions of an existing non-global partitioned topic. It requires partitioned-topic to be + * It updates number of partitions of an existing partitioned topic. It requires partitioned-topic to be * already exist and number of new partitions must be greater than existing number of partitions. Decrementing * number of partitions requires deletion of topic which is not supported. */ @POST @Path("/{tenant}/{namespace}/{topic}/partitions") @ApiOperation(value = "Increment partitions of an existing partitioned topic.", - notes = "It only increments partitions of existing non-global partitioned-topic") + notes = "It increments partitions of existing partitioned-topic") @ApiResponses(value = { @ApiResponse(code = 204, message = "Update topic partition successful."), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java index 0d5d14eb73463..f599e2566bffc 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java @@ -589,7 +589,7 @@ default CompletableFuture createNonPartitionedTopicAsync(String topic) { CompletableFuture createMissedPartitionsAsync(String topic); /** - * Update number of partitions of a non-global partitioned topic. + * Update number of partitions of a partitioned topic. *

* It requires partitioned-topic to be already exist and number of new partitions must be greater than existing * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. @@ -605,7 +605,7 @@ default CompletableFuture createNonPartitionedTopicAsync(String topic) { void updatePartitionedTopic(String topic, int numPartitions) throws PulsarAdminException; /** - * Update number of partitions of a non-global partitioned topic asynchronously. + * Update number of partitions of a partitioned topic asynchronously. *

* It requires partitioned-topic to be already exist and number of new partitions must be greater than existing * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. @@ -621,7 +621,7 @@ default CompletableFuture createNonPartitionedTopicAsync(String topic) { CompletableFuture updatePartitionedTopicAsync(String topic, int numPartitions); /** - * Update number of partitions of a non-global partitioned topic. + * Update number of partitions of a partitioned topic. *

* It requires partitioned-topic to be already exist and number of new partitions must be greater than existing * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. @@ -641,7 +641,7 @@ void updatePartitionedTopic(String topic, int numPartitions, boolean updateLocal throws PulsarAdminException; /** - * Update number of partitions of a non-global partitioned topic asynchronously. + * Update number of partitions of a partitioned topic asynchronously. *

* It requires partitioned-topic to be already exist and number of new partitions must be greater than existing * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. @@ -661,7 +661,7 @@ CompletableFuture updatePartitionedTopicAsync(String topic, int numPartiti boolean force); /** - * Update number of partitions of a non-global partitioned topic. + * Update number of partitions of a partitioned topic. *

* It requires partitioned-topic to be already exist and number of new partitions must be greater than existing * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. @@ -679,7 +679,7 @@ void updatePartitionedTopic(String topic, int numPartitions, boolean updateLocal throws PulsarAdminException; /** - * Update number of partitions of a non-global partitioned topic asynchronously. + * Update number of partitions of a partitioned topic asynchronously. *

* It requires partitioned-topic to be already exist and number of new partitions must be greater than existing * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java index 6e59be39c018b..3c1662d00a034 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java @@ -211,7 +211,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Update existing non-global partitioned topic. " + @Parameters(commandDescription = "Update existing partitioned topic. " + "New updating number of partitions must be greater than existing number of partitions.") private class UpdatePartitionedCmd extends CliCommand { diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index 8b11990dc531b..f96410749aea6 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -585,7 +585,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Update existing non-global partitioned topic. " + @Parameters(commandDescription = "Update existing partitioned topic. " + "New updating number of partitions must be greater than existing number of partitions.") private class UpdatePartitionedCmd extends CliCommand { From a332fead21390b3fbbb86c2d17a161717a170161 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Wed, 12 Apr 2023 13:12:46 +0800 Subject: [PATCH 101/280] Upgrade the RocksDB version to 7.9.2 to keep sync with BookKeeper's RocksDB dependency (#20072) ### Motivation BookKeeper has upgraded the RocksDB dependency to 7.9.2, related discussion: https://lists.apache.org/thread/8j90y4vrvgz1nvt5pb0xdjjy3o8z57z7 https://github.com/apache/bookkeeper/issues/3734 However, Pulsar also has the RocksDB dependency and it will override the BookKeeper's RocksDB dependency version. It will lead to the release package still using the old RocksDB version (6.29.4.1) ### Modifications Upgrade the Pulsar's RocksDB dependency to 7.9.2 to keep sync with the BookKeeper's RocksDB dependency. --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 0d5cd7c02a23a..c321e54de2e01 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -403,7 +403,7 @@ The Apache Software License, Version 2.0 - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar * SnakeYaml -- org.yaml-snakeyaml-1.32.jar - * RocksDB - org.rocksdb-rocksdbjni-6.29.4.1.jar + * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar * OkHttp3 diff --git a/pom.xml b/pom.xml index 404807bc4ad7c..094a35b5ed13a 100644 --- a/pom.xml +++ b/pom.xml @@ -141,7 +141,7 @@ flexible messaging model and an intuitive client API. 1.10.50 0.16.0 4.3.8 - 6.29.4.1 + 7.9.2 1.7.32 4.4 2.18.0 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index e101191bbf536..214a928a74941 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -399,7 +399,7 @@ The Apache Software License, Version 2.0 - trino-spi-368.jar - trino-record-decoder-368.jar * RocksDB JNI - - rocksdbjni-6.29.4.1.jar + - rocksdbjni-7.9.2.jar * SnakeYAML - snakeyaml-1.32.jar * Bean Validation API From 3efdc9af3bede77b018146caf1cd11730701ba4f Mon Sep 17 00:00:00 2001 From: Ayman Khalil <13960949+aymkhalil@users.noreply.github.com> Date: Wed, 12 Apr 2023 06:09:23 -0700 Subject: [PATCH 102/280] [improve][pulsar-shell] Use singleton custom command factories (#20064) --- .../pulsar/admin/cli/PulsarAdminToolTest.java | 31 +++++++++++++++++++ .../pulsar/admin/cli/PulsarAdminTool.java | 10 ++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index ccae1b1176527..2fe7b0236f0e8 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -29,8 +29,11 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import static org.testng.AssertJUnit.assertNotNull; + import com.beust.jcommander.JCommander; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; @@ -45,12 +48,14 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.admin.cli.extensions.CustomCommandFactory; import org.apache.pulsar.admin.cli.utils.SchemaExtractor; import org.apache.pulsar.client.admin.Bookies; import org.apache.pulsar.client.admin.BrokerStats; @@ -2438,6 +2443,32 @@ public void customCommands() throws Exception { } + @Test + public void customCommandsFactoryImmutable() throws Exception { + File narFile = new File(PulsarAdminTool.class.getClassLoader() + .getResource("cliextensions/customCommands-nar.nar").getFile()); + log.info("NAR FILE is {}", narFile); + + PulsarAdminBuilder builder = mock(PulsarAdminBuilder.class); + PulsarAdmin admin = mock(PulsarAdmin.class); + when(builder.build()).thenReturn(admin); + Topics topics = mock(Topics.class); + when(admin.topics()).thenReturn(topics); + TopicStats topicStats = mock(TopicStats.class); + when(topics.getStats(anyString())).thenReturn(topicStats); + when(topicStats.toString()).thenReturn("MOCK-TOPIC-STATS"); + + Properties properties = new Properties(); + properties.put("webServiceUrl", "http://localhost:2181"); + properties.put("cliExtensionsDirectory", narFile.getParentFile().getAbsolutePath()); + properties.put("customCommandFactories", "dummy"); + PulsarAdminTool tool = new PulsarAdminTool(properties); + List customCommandFactories = tool.customCommandFactories; + assertNotNull(customCommandFactories); + tool.run(split("-h")); + assertSame(tool.customCommandFactories, customCommandFactories); + } + @Test public void testHelpFlag() { PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index ca0a8a055cfc9..c06016be43883 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -25,7 +25,6 @@ import com.google.common.annotations.VisibleForTesting; import java.io.FileInputStream; import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -49,7 +48,7 @@ public class PulsarAdminTool { private static int lastExitCode = Integer.MIN_VALUE; - protected List customCommandFactories = new ArrayList(); + protected final List customCommandFactories; protected Map> commandMap; protected JCommander jcommander; protected RootParams rootParams; @@ -101,6 +100,7 @@ public static class RootParams { public PulsarAdminTool(Properties properties) throws Exception { this.properties = properties; + customCommandFactories = CustomCommandFactoryProvider.createCustomCommandFactories(properties); rootParams = new RootParams(); // fallback to previous-version serviceUrl property to maintain backward-compatibility initRootParamsFromProperties(properties); @@ -169,7 +169,6 @@ public Properties getConfiguration() { return properties; } }; - loadCustomCommandFactories(); for (CustomCommandFactory factory : customCommandFactories) { List customCommandGroups = factory.commandGroups(context); @@ -191,11 +190,6 @@ public Properties getConfiguration() { } } - private void loadCustomCommandFactories() throws Exception { - customCommandFactories = CustomCommandFactoryProvider.createCustomCommandFactories(properties); - } - - private void addCommand(Map.Entry> c, Supplier admin) throws Exception { // To remain backwards compatibility for "source" and "sink" commands // TODO eventually remove this From 117fad54bc67e8c9342cd220080eaf52930414ac Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 12 Apr 2023 10:22:05 -0500 Subject: [PATCH 103/280] [improve] Protect branch-3.0 (#20080) ### Motivation Add branch protection to the new `branch-3.0`. ### Modifications * Updated the `.asf.yaml` to include `branch-3.0`. ### Documentation - [x] `doc-not-needed` --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index bec521f1a8c71..459dcb6e2fe57 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -82,6 +82,7 @@ github: branch-2.9: {} branch-2.10: {} branch-2.11: {} + branch-3.0: {} notifications: commits: commits@pulsar.apache.org From 547d792439e7b8bfefb0236b078ed145731da658 Mon Sep 17 00:00:00 2001 From: ken <1647023764@qq.com> Date: Thu, 13 Apr 2023 01:22:02 +0800 Subject: [PATCH 104/280] [fix] [broker] error TimeUnit to record publish latency (#20074) Co-authored-by: fanjianye Co-authored-by: tison --- .../apache/pulsar/broker/rest/RestMessagePublishContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java index d7cee8b600b31..3c9adbd3e4fe4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java @@ -53,7 +53,7 @@ public void completed(Exception exception, long ledgerId, long entryId) { + "triggered send callback.", topic.getName(), ledgerId, entryId); } - topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.MICROSECONDS); + topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.NANOSECONDS); positionFuture.complete(PositionImpl.get(ledgerId, entryId)); } recycle(); From 52e8144587548a692e550a8538f4d2667b5499d6 Mon Sep 17 00:00:00 2001 From: Andy Walker Date: Wed, 12 Apr 2023 20:28:42 -0400 Subject: [PATCH 105/280] [fix][fn] check user metric len before iterating (#20021) Co-authored-by: Andy Walker --- pulsar-function-go/pf/instance.go | 3 ++ .../pf/instanceControlServicer_test.go | 53 +++++++++++++++++++ pulsar-function-go/pf/stats_test.go | 27 ++++++++++ 3 files changed, 83 insertions(+) diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 5d17cfe0c333a..a82273031ecfb 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -669,6 +669,9 @@ func (gi *goInstance) getTotalReceived1min() float32 { func (gi *goInstance) getUserMetricsMap() map[string]float64 { userMetricMap := map[string]float64{} filteredMetricFamilies := gi.getFilteredMetricFamilies(PulsarFunctionMetricsPrefix + UserMetric) + if len(filteredMetricFamilies) == 0 { + return userMetricMap + } for _, m := range filteredMetricFamilies[0].GetMetric() { var isFuncMetric bool var userLabelName string diff --git a/pulsar-function-go/pf/instanceControlServicer_test.go b/pulsar-function-go/pf/instanceControlServicer_test.go index 836ec6e5c7940..9344d0a591547 100644 --- a/pulsar-function-go/pf/instanceControlServicer_test.go +++ b/pulsar-function-go/pf/instanceControlServicer_test.go @@ -21,6 +21,7 @@ package pf import ( "context" + "fmt" "log" "net" "testing" @@ -76,3 +77,55 @@ func TestInstanceControlServicer_serve_creates_valid_instance(t *testing.T) { log.Printf("Response: %+v", resp.Success) assert.Equal(t, resp.Success, true) } + +func instanceCommunicationClient(t *testing.T, instance *goInstance) pb.InstanceControlClient { + t.Helper() + + if instance == nil { + t.Fatalf("cannot create communication client for nil instance") + } + + var ( + ctx context.Context = context.Background() + cf context.CancelFunc + ) + + if testDeadline, ok := t.Deadline(); ok { + ctx, cf = context.WithDeadline(context.Background(), testDeadline) + t.Cleanup(cf) + } + + lis = bufconn.Listen(bufSize) + t.Cleanup(func() { + lis.Close() + }) + // create a gRPC server object + grpcServer := grpc.NewServer() + t.Cleanup(func() { + grpcServer.Stop() + }) + + servicer := InstanceControlServicer{instance} + // must register before we start the service. + pb.RegisterInstanceControlServer(grpcServer, &servicer) + + // start the server + t.Logf("Serving InstanceCommunication on port %d", instance.context.GetPort()) + + go func() { + if err := grpcServer.Serve(lis); err != nil { + panic(fmt.Sprintf("grpc server exited with error: %v", err)) + } + }() + + // Now we can setup the client: + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(getBufDialer(lis)), grpc.WithInsecure()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) + } + t.Cleanup(func() { + conn.Close() + }) + client := pb.NewInstanceControlClient(conn) + return client +} diff --git a/pulsar-function-go/pf/stats_test.go b/pulsar-function-go/pf/stats_test.go index d52b08b717377..7b415ef5eff0b 100644 --- a/pulsar-function-go/pf/stats_test.go +++ b/pulsar-function-go/pf/stats_test.go @@ -20,6 +20,7 @@ package pf import ( + "context" "fmt" "io/ioutil" "math" @@ -28,6 +29,7 @@ import ( "time" "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes/empty" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" @@ -257,3 +259,28 @@ func TestUserMetrics(t *testing.T) { gi.close() metricsServicer.close() } + +func TestInstanceControlMetrics(t *testing.T) { + instance := newGoInstance() + t.Cleanup(instance.close) + instanceClient := instanceCommunicationClient(t, instance) + _, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{}) + assert.NoError(t, err, "err communicating with instance control: %v", err) + + testLabels := []string{"userMetricControlTest1", "userMetricControlTest2"} + for _, label := range testLabels { + assert.NotContainsf(t, label, "user metrics should not yet contain %s", label) + } + + for value, label := range testLabels { + instance.context.RecordMetric(label, float64(value+1)) + } + time.Sleep(time.Second) + + metrics, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{}) + assert.NoError(t, err, "err communicating with instance control: %v", err) + for value, label := range testLabels { + assert.Containsf(t, metrics.UserMetrics, label, "user metrics should contain metric %s", label) + assert.EqualValuesf(t, value+1, metrics.UserMetrics[label], "user metric %s != %d", label, value+1) + } +} From 4d8156fca0582a350d30557d9243c851c4a45830 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 13 Apr 2023 08:51:52 +0200 Subject: [PATCH 106/280] [improve][ci]Add branch-3.0 to owasp checks (#20081) --- .github/workflows/ci-owasp-dependency-check.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index dba1c92ca28dd..8aec501f8d868 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -39,6 +39,7 @@ jobs: matrix: include: - branch: master + - branch: branch-3.0 - branch: branch-2.11 - branch: branch-2.10 jdk: 11 From 653271e8d8aeb361ec260780ba88ccf3a880f403 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 13 Apr 2023 23:09:10 +0800 Subject: [PATCH 107/280] [fix][test]Fix flaky test produceCommitTest (#20006) Fixes https://github.com/apache/pulsar/issues/18466 ### Motivation There are two main goals in solving this issue: 1. Fix the unstable tests in `produceCommitTest`. 2. Prevent transaction timeouts created by other tests from affecting the `testTxnTimeoutAtTransactionMetadataStore` test during its execution. ### Modification 1. Change the message-sending method to synchronous. (fix `produceCommitTest`) 2. Increase the transaction timeout to 10 minutes (fix `testTxnTimeoutAtTransactionMetadataStore`). --- .../apache/pulsar/client/impl/TransactionEndToEndTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 83feaa3ac1158..34cc3bc1ca526 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -280,9 +280,9 @@ private void produceCommitTest(boolean enableBatch) throws Exception { int messageCnt = 1000; for (int i = 0; i < messageCnt; i++) { if (i % 5 == 0) { - producer.newMessage(txn1).value(("Hello Txn - " + i).getBytes(UTF_8)).send(); + producer.newMessage(txn1).value(("Hello Txn - " + i).getBytes(UTF_8)).sendAsync(); } else { - producer.newMessage(txn2).value(("Hello Txn - " + i).getBytes(UTF_8)).send(); + producer.newMessage(txn2).value(("Hello Txn - " + i).getBytes(UTF_8)).sendAsync(); } txnMessageCnt++; } @@ -811,7 +811,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri public Transaction getTxn() throws Exception { return pulsarClient .newTransaction() - .withTransactionTimeout(10, TimeUnit.SECONDS) + .withTransactionTimeout(10, TimeUnit.MINUTES) .build() .get(); } From 091ee2504ffbe6ec98e354b76e7f4c045e1914aa Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 13 Apr 2023 10:56:22 -0700 Subject: [PATCH 108/280] [improve][broker] PIP-192 Fix getLastMessageId for compressed payload(And add compression and maxBatchSize for the load balance system topic) (#20087) --- .../channel/ServiceUnitStateChannelImpl.java | 4 ++ .../pulsar/broker/service/BrokerService.java | 6 +- .../pulsar/broker/service/ServerCnx.java | 11 ++++ .../impl/RawBatchMessageContainerImpl.java | 12 ++-- .../StrategicTwoPhaseCompactor.java | 17 ++++- .../RawBatchMessageContainerImplTest.java | 63 +++++++++++++------ .../GetLastMessageIdCompactedTest.java | 43 +++++++++++++ .../ServiceUnitStateCompactionTest.java | 24 +++++-- 8 files changed, 145 insertions(+), 35 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index bd62c53be6083..68c6440e68e4b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -81,6 +81,7 @@ import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; @@ -106,6 +107,8 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { TopicDomain.persistent.value(), SYSTEM_NAMESPACE, "loadbalancer-service-unit-state").toString(); + + public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD; private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec public static final long VERSION_ID_INIT = 1; // initial versionId private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; @@ -285,6 +288,7 @@ public synchronized void start() throws PulsarServerException { producer = pulsar.getClient().newProducer(schema) .enableBatching(true) + .compressionType(MSG_COMPRESSION_TYPE) .maxPendingMessages(MAX_OUTSTANDING_PUB_MESSAGES) .blockIfQueueFull(true) .topic(TOPIC) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index bb08734298f04..33e5500d623d2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2144,9 +2144,9 @@ public CompletableFuture checkTopicNsOwnership(final String topic) { if (ownedByThisInstance) { return CompletableFuture.completedFuture(null); } else { - String msg = String.format("Namespace bundle for topic (%s) not served by this instance. " - + "Please redo the lookup. Request is denied: namespace=%s", topic, - topicName.getNamespace()); + String msg = String.format("Namespace bundle for topic (%s) not served by this instance:%s. " + + "Please redo the lookup. Request is denied: namespace=%s", + topic, pulsar.getLookupServiceAddress(), topicName.getNamespace()); log.warn(msg); return FutureUtil.failedFuture(new ServiceUnitNotReadyException(msg)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 01274ab4460cb..888668e15b167 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -129,6 +129,7 @@ import org.apache.pulsar.common.api.proto.CommandUnsubscribe; import org.apache.pulsar.common.api.proto.CommandWatchTopicList; import org.apache.pulsar.common.api.proto.CommandWatchTopicListClose; +import org.apache.pulsar.common.api.proto.CompressionType; import org.apache.pulsar.common.api.proto.FeatureFlags; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; @@ -141,6 +142,8 @@ import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.api.proto.SingleMessageMetadata; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.compression.CompressionCodec; +import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.intercept.InterceptException; import org.apache.pulsar.common.naming.Metadata; import org.apache.pulsar.common.naming.NamespaceName; @@ -2167,6 +2170,14 @@ private int calculateTheLastBatchIndexInBatch(MessageMetadata metadata, ByteBuf if (batchSize <= 1){ return -1; } + if (metadata.hasCompression()) { + var tmp = payload; + CompressionType compressionType = metadata.getCompression(); + CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); + int uncompressedSize = metadata.getUncompressedSize(); + payload = codec.decode(payload, uncompressedSize); + tmp.release(); + } SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata(); int lastBatchIndexInBatch = -1; for (int i = 0; i < batchSize; i++){ diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java index cf6b213155cd7..7e1c2cd5e3fe3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java @@ -47,13 +47,13 @@ public class RawBatchMessageContainerImpl extends BatchMessageContainerImpl { MessageCrypto msgCrypto; Set encryptionKeys; CryptoKeyReader cryptoKeyReader; - public RawBatchMessageContainerImpl(int maxNumMessagesInBatch) { + + public RawBatchMessageContainerImpl(int maxNumMessagesInBatch, int maxBytesInBatch) { super(); - compressionType = CompressionType.NONE; - compressor = new CompressionCodecNone(); - if (maxNumMessagesInBatch > 0) { - this.maxNumMessagesInBatch = maxNumMessagesInBatch; - } + this.compressionType = CompressionType.NONE; + this.compressor = new CompressionCodecNone(); + this.maxNumMessagesInBatch = maxNumMessagesInBatch; + this.maxBytesInBatch = maxBytesInBatch; } private ByteBuf encrypt(ByteBuf compressedPayload) { if (msgCrypto == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index 37b03e275d6bf..557d4a6580161 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.compaction; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import java.time.Duration; import java.util.Iterator; @@ -62,17 +63,29 @@ public class StrategicTwoPhaseCompactor extends TwoPhaseCompactor { private static final Logger log = LoggerFactory.getLogger(StrategicTwoPhaseCompactor.class); private static final int MAX_OUTSTANDING = 500; + private static final int MAX_NUM_MESSAGES_IN_BATCH = 1000; + private static final int MAX_BYTES_IN_BATCH = 128 * 1024; private static final int MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS = 20 * 1000; private final Duration phaseOneLoopReadTimeout; private final RawBatchMessageContainerImpl batchMessageContainer; + @VisibleForTesting public StrategicTwoPhaseCompactor(ServiceConfiguration conf, PulsarClient pulsar, BookKeeper bk, ScheduledExecutorService scheduler, int maxNumMessagesInBatch) { + this(conf, pulsar, bk, scheduler, maxNumMessagesInBatch, MAX_BYTES_IN_BATCH); + } + + private StrategicTwoPhaseCompactor(ServiceConfiguration conf, + PulsarClient pulsar, + BookKeeper bk, + ScheduledExecutorService scheduler, + int maxNumMessagesInBatch, + int maxBytesInBatch) { super(conf, pulsar, bk, scheduler); - batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch); + batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch, maxBytesInBatch); phaseOneLoopReadTimeout = Duration.ofSeconds(conf.getBrokerServiceCompactionPhaseOneLoopTimeInSeconds()); } @@ -80,7 +93,7 @@ public StrategicTwoPhaseCompactor(ServiceConfiguration conf, PulsarClient pulsar, BookKeeper bk, ScheduledExecutorService scheduler) { - this(conf, pulsar, bk, scheduler, -1); + this(conf, pulsar, bk, scheduler, MAX_NUM_MESSAGES_IN_BATCH, MAX_BYTES_IN_BATCH); } public CompletableFuture compact(String topic) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java index 9fa834a166cda..9b8b1e5efb99c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java @@ -19,8 +19,10 @@ package org.apache.pulsar.client.impl; -import static org.apache.pulsar.common.api.proto.CompressionType.LZ4; import static org.apache.pulsar.common.api.proto.CompressionType.NONE; +import static org.apache.pulsar.common.api.proto.CompressionType.ZSTD; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertTrue; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.io.IOException; @@ -45,6 +47,7 @@ import org.apache.pulsar.compaction.CompactionTest; import org.testng.Assert; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class RawBatchMessageContainerImplTest { @@ -53,9 +56,11 @@ public class RawBatchMessageContainerImplTest { CryptoKeyReader cryptoKeyReader; Map encryptKeys; + int maxBytesInBatch = 5 * 1024 * 1024; + public void setEncryptionAndCompression(boolean encrypt, boolean compress) { if (compress) { - compressionType = LZ4; + compressionType = ZSTD; } else { compressionType = NONE; } @@ -100,14 +105,24 @@ public MessageImpl createMessage(String topic, String value, int entryId) { @BeforeMethod public void setup() throws Exception { - setEncryptionAndCompression(false, false); + setEncryptionAndCompression(false, true); } - @Test - public void testToByteBuf() throws IOException { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2); + @DataProvider(name = "testBatchLimitByMessageCount") + public static Object[][] testBatchLimitByMessageCount() { + return new Object[][] {{true}, {false}}; + } + + @Test(timeOut = 20000, dataProvider = "testBatchLimitByMessageCount") + public void testToByteBufWithBatchLimit(boolean testBatchLimitByMessageCount) throws IOException { + RawBatchMessageContainerImpl container = testBatchLimitByMessageCount ? + new RawBatchMessageContainerImpl(2, Integer.MAX_VALUE) : + new RawBatchMessageContainerImpl(Integer.MAX_VALUE, 5); + String topic = "my-topic"; - container.add(createMessage(topic, "hi-1", 0), null); - container.add(createMessage(topic, "hi-2", 1), null); + var full1 = container.add(createMessage(topic, "hi-1", 0), null); + var full2 = container.add(createMessage(topic, "hi-2", 1), null); + assertFalse(full1); + assertTrue(full2); ByteBuf buf = container.toByteBuf(); @@ -126,18 +141,23 @@ public void testToByteBuf() throws IOException { MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder(); Assert.assertEquals(metadata.getNumMessagesInBatch(), 2); Assert.assertEquals(metadata.getHighestSequenceId(), 1); - Assert.assertEquals(metadata.getCompression(), NONE); + Assert.assertEquals(metadata.getCompression(), ZSTD); + + CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); + ByteBuf payload = codec.decode(metadataAndPayload, metadata.getUncompressedSize()); SingleMessageMetadata messageMetadata = new SingleMessageMetadata(); + messageMetadata.setCompactedOut(true); ByteBuf payload1 = Commands.deSerializeSingleMessageInBatch( - singleMessageMetadataAndPayload.getPayload(), messageMetadata, 0, 2); + payload, messageMetadata, 0, 2); ByteBuf payload2 = Commands.deSerializeSingleMessageInBatch( - singleMessageMetadataAndPayload.getPayload(), messageMetadata, 1, 2); + payload, messageMetadata, 1, 2); Assert.assertEquals(payload1.toString(Charset.defaultCharset()), "hi-1"); Assert.assertEquals(payload2.toString(Charset.defaultCharset()), "hi-2"); payload1.release(); payload2.release(); + payload.release(); singleMessageMetadataAndPayload.release(); metadataAndPayload.release(); buf.release(); @@ -147,7 +167,7 @@ public void testToByteBuf() throws IOException { public void testToByteBufWithCompressionAndEncryption() throws IOException { setEncryptionAndCompression(true, true); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); container.setCryptoKeyReader(cryptoKeyReader); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); @@ -169,7 +189,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException { MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder(); Assert.assertEquals(metadata.getNumMessagesInBatch(), 2); Assert.assertEquals(metadata.getHighestSequenceId(), 1); - Assert.assertEquals(metadata.getCompression(), compressionType); + Assert.assertEquals(metadata.getCompression(), ZSTD); ByteBuf payload = singleMessageMetadataAndPayload.getPayload(); int maxDecryptedSize = msgCrypto.getMaxOutputSize(payload.readableBytes()); @@ -197,7 +217,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException { @Test public void testToByteBufWithSingleMessage() throws IOException { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); ByteBuf buf = container.toByteBuf(); @@ -218,9 +238,12 @@ public void testToByteBufWithSingleMessage() throws IOException { MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder(); Assert.assertEquals(metadata.getNumMessagesInBatch(), 1); Assert.assertEquals(metadata.getHighestSequenceId(), 0); - Assert.assertEquals(metadata.getCompression(), NONE); + Assert.assertEquals(metadata.getCompression(), ZSTD); + + CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); + ByteBuf payload = codec.decode(metadataAndPayload, metadata.getUncompressedSize()); - Assert.assertEquals(singleMessageMetadataAndPayload.getPayload().toString(Charset.defaultCharset()), "hi-1"); + Assert.assertEquals(payload.toString(Charset.defaultCharset()), "hi-1"); singleMessageMetadataAndPayload.release(); metadataAndPayload.release(); buf.release(); @@ -228,7 +251,7 @@ public void testToByteBufWithSingleMessage() throws IOException { @Test public void testMaxNumMessagesInBatch() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); String topic = "my-topic"; boolean isFull = container.add(createMessage(topic, "hi", 0), null); @@ -238,14 +261,14 @@ public void testMaxNumMessagesInBatch() { @Test(expectedExceptions = UnsupportedOperationException.class) public void testCreateOpSendMsg() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); container.createOpSendMsg(); } @Test public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); Assert.assertEquals(container.getNumMessagesInBatch(), 1); @@ -263,7 +286,7 @@ public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { @Test public void testToByteBufWithEncryptionWithInvalidEncryptKeys() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); container.setCryptoKeyReader(cryptoKeyReader); encryptKeys = new HashMap<>(); encryptKeys.put(null, null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java index 0be9fa407542f..317b1a227e585 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; @@ -263,6 +264,48 @@ public void testGetLastMessageIdAfterCompaction(boolean enabledBatch) throws Exc admin.topics().delete(topicName, false); } + @Test(dataProvider = "enabledBatch") + public void testGetLastMessageIdAfterCompactionWithCompression(boolean enabledBatch) throws Exception { + String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp"); + String subName = "sub"; + Consumer consumer = createConsumer(topicName, subName); + var producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .batchingMaxPublishDelay(3, TimeUnit.HOURS) + .batchingMaxBytes(Integer.MAX_VALUE) + .compressionType(CompressionType.ZSTD) + .enableBatching(enabledBatch).create(); + + List> sendFutures = new ArrayList<>(); + sendFutures.add(producer.newMessage().key("k0").value("v0").sendAsync()); + sendFutures.add(producer.newMessage().key("k0").value("v1").sendAsync()); + sendFutures.add(producer.newMessage().key("k0").value("v2").sendAsync()); + producer.flush(); + sendFutures.add(producer.newMessage().key("k1").value("v0").sendAsync()); + sendFutures.add(producer.newMessage().key("k1").value("v1").sendAsync()); + sendFutures.add(producer.newMessage().key("k1").value("v2").sendAsync()); + producer.flush(); + FutureUtil.waitForAll(sendFutures).join(); + + triggerCompactionAndWait(topicName); + + MessageIdImpl lastMessageIdByTopic = getLastMessageIdByTopic(topicName); + MessageIdImpl messageId = (MessageIdImpl) consumer.getLastMessageId(); + assertEquals(messageId.getLedgerId(), lastMessageIdByTopic.getLedgerId()); + assertEquals(messageId.getEntryId(), lastMessageIdByTopic.getEntryId()); + if (enabledBatch){ + BatchMessageIdImpl lastBatchMessageIdByTopic = (BatchMessageIdImpl) lastMessageIdByTopic; + BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; + assertEquals(batchMessageId.getBatchSize(), lastBatchMessageIdByTopic.getBatchSize()); + assertEquals(batchMessageId.getBatchIndex(), lastBatchMessageIdByTopic.getBatchIndex()); + } + + // cleanup. + consumer.close(); + producer.close(); + admin.topics().delete(topicName, false); + } + @Test(dataProvider = "enabledBatch") public void testGetLastMessageIdAfterCompactionEndWithNullMsg(boolean enabledBatch) throws Exception { String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 02812898dc49a..1a69a86f7c6b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -25,6 +25,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -185,6 +186,7 @@ TestData generateTestData() throws PulsarAdminException, PulsarClientException { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); @@ -352,12 +354,13 @@ public void testCompactionWithTableview() throws Exception { compactor.compact(topic, strategy).get(); // consumer with readCompacted enabled only get compacted entries - var tableview = pulsar.getClient().newTableViewBuilder(schema) + var tableview = pulsar.getClient().newTableView(schema) .topic(topic) .loadConf(Map.of( "topicCompactionStrategyClassName", ServiceUnitStateCompactionStrategy.class.getName())) .create(); + for(var etr : tableview.entrySet()){ Assert.assertEquals(expected.remove(etr.getKey()), etr.getValue()); if (expected.isEmpty()) { @@ -376,6 +379,7 @@ public void testReadCompactedBeforeCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -420,6 +424,7 @@ public void testReadEntriesAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -460,6 +465,7 @@ public void testSeekEarliestAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -557,6 +563,7 @@ public void testSlowTableviewAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); @@ -627,6 +634,7 @@ public void testBrokerRestartAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); String key = "key0"; @@ -672,6 +680,7 @@ public void testCompactEmptyTopic() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -733,7 +742,9 @@ public void testWholeBatchCompactedOut() throws Exception { public void testCompactionWithLastDeletedKey() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; - Producer producer = pulsarClient.newProducer(schema).topic(topic).enableBatching(true) + Producer producer = pulsarClient.newProducer(schema).topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) + .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition).create(); pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); @@ -761,7 +772,9 @@ public void testCompactionWithLastDeletedKey() throws Exception { public void testEmptyCompactionLedger() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; - Producer producer = pulsarClient.newProducer(schema).topic(topic).enableBatching(true) + Producer producer = pulsarClient.newProducer(schema).topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) + .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition).create(); pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); @@ -791,7 +804,8 @@ public void testAllEmptyCompactionLedger() throws Exception { final int messages = 10; // 1.create producer and publish message to the topic. - ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic); + ProducerBuilder builder = pulsarClient.newProducer(schema) + .compressionType(MSG_COMPRESSION_TYPE).topic(topic); builder.batchingMaxMessages(messages / 5); Producer producer = builder.create(); @@ -828,6 +842,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() // 1.create producer and publish message to the topic. ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic); + builder.compressionType(MSG_COMPRESSION_TYPE); builder.enableBatching(true); @@ -876,6 +891,7 @@ public void testReadUnCompacted() // 1.create producer and publish message to the topic. ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic); + builder.compressionType(MSG_COMPRESSION_TYPE); builder.batchingMaxMessages(messages / 5); Producer producer = builder.create(); From 6152cc82ac8a344cfc2539c56f15cec6f4a1cff7 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 15 Apr 2023 13:42:07 +0800 Subject: [PATCH 109/280] [fix][broker] Fix avoid future of clear delayed message can't complete (#20075) --- .../BucketDelayedDeliveryTrackerFactory.java | 28 +++++++++++ .../pulsar/broker/delayed/bucket/Bucket.java | 3 +- .../bucket/BucketDelayedDeliveryTracker.java | 4 +- ...PersistentDispatcherMultipleConsumers.java | 17 +++---- .../service/persistent/PersistentTopic.java | 26 +++++----- .../persistent/BucketDelayedDeliveryTest.java | 48 +++++++++++++++++++ 6 files changed, 102 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java index f0feb8b27d6a1..6a00bfd199584 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java @@ -21,13 +21,19 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.common.util.FutureUtil; public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrackerFactory { @@ -72,6 +78,28 @@ public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers d delayedDeliveryMaxIndexesPerBucketSnapshotSegment, delayedDeliveryMaxNumBuckets); } + /** + * Clean up residual snapshot data. + * If tracker has not been created or has been closed, then we can't clean up the snapshot with `tracker.clear`, + * this method can clean up the residual snapshots without creating a tracker. + */ + public CompletableFuture cleanResidualSnapshots(ManagedCursor cursor) { + Map cursorProperties = cursor.getCursorProperties(); + List> futures = new ArrayList<>(); + FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create(); + cursorProperties.forEach((k, v) -> { + if (k != null && v != null && k.startsWith(BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX)) { + CompletableFuture future = sequencer.sequential(() -> { + return cursor.removeCursorProperty(k) + .thenCompose(__ -> bucketSnapshotStorage.deleteBucketSnapshot(Long.parseLong(v))); + }); + futures.add(future); + } + }); + + return FutureUtil.waitForAll(futures); + } + @Override public void close() throws Exception { if (bucketSnapshotStorage != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 5b7023be5034e..4d7d3aa512be6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.broker.delayed.bucket; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; +import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,7 +41,6 @@ @AllArgsConstructor abstract class Bucket { - static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket"; static final String DELIMITER = "_"; static final int MaxRetryTimes = 3; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index a34bd51af98e4..6ead1e207b0ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELAYED_BUCKET_KEY_PREFIX; +import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes; import com.google.common.annotations.VisibleForTesting; @@ -66,6 +66,8 @@ @ThreadSafe public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker { + public static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket"; + static final CompletableFuture NULL_LONG_PROMISE = CompletableFuture.completedFuture(null); static final int AsyncOperationTimeoutSeconds = 60; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 4ac755860fc7b..c60b4562bf111 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -49,6 +49,7 @@ import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers; @@ -1098,19 +1099,15 @@ public CompletableFuture clearDelayedMessages() { return CompletableFuture.completedFuture(null); } - if (delayedDeliveryTracker.isEmpty() && topic.getBrokerService() - .getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory) { - synchronized (this) { - if (delayedDeliveryTracker.isEmpty()) { - delayedDeliveryTracker = Optional - .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this)); - } - } - } - if (delayedDeliveryTracker.isPresent()) { return this.delayedDeliveryTracker.get().clear(); } else { + DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory = + topic.getBrokerService().getDelayedDeliveryTrackerFactory(); + if (delayedDeliveryTrackerFactory instanceof BucketDelayedDeliveryTrackerFactory + bucketDelayedDeliveryTrackerFactory) { + return bucketDelayedDeliveryTrackerFactory.cleanResidualSnapshots(cursor); + } return CompletableFuture.completedFuture(null); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 0f5e60439810f..fe181bb1c01f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -81,6 +81,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; +import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -1169,15 +1170,21 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, } Dispatcher dispatcher = persistentSubscription.getDispatcher(); - final Dispatcher temporaryDispatcher; if (dispatcher == null) { - log.info("[{}][{}] Dispatcher is null, try to create temporary dispatcher to clear delayed message", topic, - subscriptionName); - dispatcher = temporaryDispatcher = - new PersistentDispatcherMultipleConsumers(this, persistentSubscription.cursor, - persistentSubscription); - } else { - temporaryDispatcher = null; + DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory = + brokerService.getDelayedDeliveryTrackerFactory(); + if (delayedDeliveryTrackerFactory instanceof BucketDelayedDeliveryTrackerFactory + bucketDelayedDeliveryTrackerFactory) { + ManagedCursor cursor = persistentSubscription.getCursor(); + bucketDelayedDeliveryTrackerFactory.cleanResidualSnapshots(cursor).whenComplete((__, ex) -> { + if (ex != null) { + unsubscribeFuture.completeExceptionally(ex); + } else { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + } + }); + } + return; } dispatcher.clearDelayedMessages().whenComplete((__, ex) -> { @@ -1186,9 +1193,6 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, } else { asyncDeleteCursor(subscriptionName, unsubscribeFuture); } - if (temporaryDispatcher != null) { - temporaryDispatcher.close(); - } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index 5480a2e7a704a..0a82b2b4c3cb0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -305,4 +305,52 @@ public void testBucketDelayedIndexMetrics() throws Exception { assertTrue(namespaceMetric.isPresent()); assertEquals(6, namespaceMetric.get().value); } + + @Test + public void testDelete() throws Exception { + String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testDelete"); + + @Cleanup + Consumer c1 = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + for (int i = 0; i < 1000; i++) { + producer.newMessage() + .value("msg") + .deliverAfter(1, TimeUnit.HOURS) + .send(); + } + + Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher(); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000)); + + Map cursorProperties = + ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties(); + List bucketIds = cursorProperties.entrySet().stream() + .filter(x -> x.getKey().startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket")).map( + x -> Long.valueOf(x.getValue())).toList(); + + assertTrue(bucketIds.size() > 0); + + admin.topics().delete(topic, true); + + for (Long bucketId : bucketIds) { + try { + LedgerHandle ledgerHandle = + pulsarTestContext.getBookKeeperClient() + .openLedger(bucketId, BookKeeper.DigestType.CRC32C, new byte[]{}); + Assert.fail("Should fail"); + } catch (BKException.BKNoSuchLedgerExistsException e) { + // ignore it + } + } + } } From c4aec6661e795c46181dc1fa79282065fa875768 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sun, 16 Apr 2023 16:05:44 +0800 Subject: [PATCH 110/280] [fix][broker] Ensure previous delayed index be removed from snapshotSegmentLastIndexTable & Make load operate asynchronous (#20086) --- .../bucket/BucketDelayedDeliveryTracker.java | 106 ++++++++++-------- ...PersistentDispatcherMultipleConsumers.java | 13 +-- .../BucketDelayedDeliveryTrackerTest.java | 30 +++-- 3 files changed, 85 insertions(+), 64 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 6ead1e207b0ce..6678c6df254b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; -import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Range; @@ -84,7 +83,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final int maxNumBuckets; - private long numberDelayedMessages; + private volatile long numberDelayedMessages; @Getter @VisibleForTesting @@ -102,6 +101,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final BucketDelayedMessageIndexStats stats; + private CompletableFuture pendingLoad = null; + public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -269,7 +270,7 @@ private void afterCreateImmutableBucket(Pair immu if (ex == null) { immutableBucket.setSnapshotSegments(null); immutableBucket.asyncUpdateSnapshotLength(); - log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(), + log.info("[{}] Create bucket snapshot finish, bucketKey: {}", dispatcher.getName(), immutableBucket.bucketKey()); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.create, @@ -529,17 +530,25 @@ protected long nextDeliveryTime() { } @Override - public synchronized long getNumberOfDelayedMessages() { + public long getNumberOfDelayedMessages() { return numberDelayedMessages; } @Override - public synchronized long getBufferMemoryUsage() { + public long getBufferMemoryUsage() { return this.lastMutableBucket.getBufferMemoryUsage() + sharedBucketPriorityQueue.bytesCapacity(); } @Override public synchronized NavigableSet getScheduledMessages(int maxMessages) { + if (!checkPendingOpDone()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", + dispatcher.getName()); + } + return Collections.emptyNavigableSet(); + } + long cutoffTime = getCutoffTime(); lastMutableBucket.moveScheduledMessageToSharedQueue(cutoffTime, sharedBucketPriorityQueue); @@ -558,6 +567,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId); if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) { + // All message of current snapshot segment are scheduled, try load next snapshot segment if (bucket.merging) { log.info("[{}] Skip load to wait for bucket snapshot merge finish, bucketKey:{}", dispatcher.getName(), bucket.bucketKey()); @@ -569,26 +579,19 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}", dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); } - // All message of current snapshot segment are scheduled, load next snapshot segment - // TODO make it asynchronous and not blocking this process - try { - boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone(); - - if (!createFutureDone) { - log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}", - dispatcher.getName(), bucket.bucketKey()); - break; - } - - if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) { - immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); - bucket.asyncDeleteBucketSnapshot(stats); - continue; - } + boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone(); + if (!createFutureDone) { + log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}", + dispatcher.getName(), bucket.bucketKey()); + break; + } - long loadStartTime = System.currentTimeMillis(); - stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load); - bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> { + long loadStartTime = System.currentTimeMillis(); + stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load); + CompletableFuture loadFuture = pendingLoad = bucket.asyncLoadNextBucketSnapshotEntry() + .thenAccept(indexList -> { + synchronized (BucketDelayedDeliveryTracker.this) { + this.snapshotSegmentLastIndexTable.remove(ledgerId, entryId); if (CollectionUtils.isEmpty(indexList)) { immutableBuckets.asMapOfRanges() .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); @@ -603,31 +606,36 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } - }).whenComplete((__, ex) -> { - if (ex != null) { - // Back bucket state - bucket.setCurrentSegmentEntryId(preSegmentEntryId); - - log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); - - stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load); - } else { - log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); - - stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, - System.currentTimeMillis() - loadStartTime); + } + }).whenComplete((__, ex) -> { + if (ex != null) { + // Back bucket state + bucket.setCurrentSegmentEntryId(preSegmentEntryId); + + log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); + + stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load); + } else { + log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), + (preSegmentEntryId == bucket.lastSegmentEntryId) ? "-1" : preSegmentEntryId + 1); + + stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, + System.currentTimeMillis() - loadStartTime); + } + synchronized (this) { + if (timeout != null) { + timeout.cancel(); } - }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS); - } catch (Exception e) { - // Ignore exception to reload this segment on the next schedule. - log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}", - dispatcher.getName(), bucket.bucketKey(), e); + timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS); + } + }); + + if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) { break; } } - snapshotSegmentLastIndexTable.remove(ledgerId, entryId); positions.add(new PositionImpl(ledgerId, entryId)); @@ -643,6 +651,14 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa return positions; } + private synchronized boolean checkPendingOpDone() { + if (pendingLoad == null || pendingLoad.isDone()) { + pendingLoad = null; + return true; + } + return false; + } + @Override public boolean shouldPauseAllDeliveries() { return false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index c60b4562bf111..81adda053e8cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -46,7 +46,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; @@ -1089,7 +1088,7 @@ protected synchronized boolean shouldPauseDeliveryForDelayTracker() { } @Override - public synchronized long getNumberOfDelayedMessages() { + public long getNumberOfDelayedMessages() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L); } @@ -1169,15 +1168,7 @@ public PersistentTopic getTopic() { public long getDelayedTrackerMemoryUsage() { - if (delayedDeliveryTracker.isEmpty()) { - return 0; - } - - if (delayedDeliveryTracker.get() instanceof AbstractDelayedDeliveryTracker) { - return delayedDeliveryTracker.get().getBufferMemoryUsage(); - } - - return 0; + return delayedDeliveryTracker.map(DelayedDeliveryTracker::getBufferMemoryUsage).orElse(0L); } public Map getBucketDelayedIndexStats() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 95234d688f6a0..39b3992fbd195 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -39,6 +39,7 @@ import java.util.NavigableSet; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -197,9 +198,11 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { }); assertTrue(tracker.hasMessageAvailable()); - Set scheduledMessages = tracker.getScheduledMessages(100); - - assertEquals(scheduledMessages.size(), 1); + Set scheduledMessages = new TreeSet<>(); + Awaitility.await().untilAsserted(() -> { + scheduledMessages.addAll(tracker.getScheduledMessages(100)); + assertEquals(scheduledMessages.size(), 1); + }); tracker.addMessage(101, 101, 101 * 10); @@ -216,12 +219,15 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { clockTime.set(100 * 10); assertTrue(tracker2.hasMessageAvailable()); - scheduledMessages = tracker2.getScheduledMessages(70); + Set scheduledMessages2 = new TreeSet<>(); - assertEquals(scheduledMessages.size(), 70); + Awaitility.await().untilAsserted(() -> { + scheduledMessages2.addAll(tracker2.getScheduledMessages(70)); + assertEquals(scheduledMessages2.size(), 70); + }); int i = 31; - for (PositionImpl scheduledMessage : scheduledMessages) { + for (PositionImpl scheduledMessage : scheduledMessages2) { assertEquals(scheduledMessage, PositionImpl.get(i, i)); i++; } @@ -298,7 +304,11 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) { clockTime.set(110 * 10); - NavigableSet scheduledMessages = tracker2.getScheduledMessages(110); + NavigableSet scheduledMessages = new TreeSet<>(); + Awaitility.await().untilAsserted(() -> { + scheduledMessages.addAll(tracker2.getScheduledMessages(110)); + assertEquals(scheduledMessages.size(), 110); + }); for (int i = 1; i <= 110; i++) { PositionImpl position = scheduledMessages.pollFirst(); assertEquals(position, PositionImpl.get(i, i)); @@ -370,7 +380,11 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) { assertEquals(tracker2.getScheduledMessages(100).size(), 0); - assertEquals(tracker2.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue); + Set scheduledMessages = new TreeSet<>(); + Awaitility.await().untilAsserted(() -> { + scheduledMessages.addAll(tracker2.getScheduledMessages(100)); + assertEquals(scheduledMessages.size(), delayedMessagesInSnapshotValue); + }); assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty()); assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty()); From 47b24b071353f9285704e29e397b851e31df4037 Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Sun, 16 Apr 2023 17:05:46 +0300 Subject: [PATCH 111/280] [fix][doc] Add link to PIP proposal (#19911) ### Motivation Add a link to see existing PIP issues so you can select a number --- wiki/proposals/PIP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index e10452b107fd8..159c9199895d1 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -80,7 +80,7 @@ The process works in the following way: 1. The author(s) of the proposal will create a GitHub issue ticket choosing the template for PIP proposals. The issue title should be "PIP-xxx: title", where the "xxx" number should be chosen to be the next number from the existing PIP - issues, listed [here]([url](https://github.com/apache/pulsar/labels/PIP)). + issues, listed [here](https://github.com/apache/pulsar/issues?q=is%3Aissue+label%3APIP+) 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion need to happen in the mailing list. Please avoid discussing it using From b7eab9469177eda2c56e36bb9871aab48a17d4ec Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 17 Apr 2023 10:59:32 +0800 Subject: [PATCH 112/280] [fix][doc] ConnectorDocGenerator support Java 9+ (#20100) Signed-off-by: tison --- .../pulsar/io/docs/ConnectorDocGenerator.java | 69 +++++++------------ .../io/elasticsearch/ElasticSearchConfig.java | 5 ++ 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java index fe12b2b11ce84..fec7b12087977 100644 --- a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java +++ b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java @@ -21,6 +21,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.google.common.base.Strings; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; @@ -28,11 +29,9 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.net.URL; -import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Set; import lombok.extern.slf4j.Slf4j; @@ -47,21 +46,12 @@ public class ConnectorDocGenerator { private static final String INDENT = " "; private static Reflections newReflections() throws Exception { - List urls = new ArrayList<>(); - ClassLoader[] classLoaders = new ClassLoader[] { - ConnectorDocGenerator.class.getClassLoader(), - Thread.currentThread().getContextClassLoader() - }; - for (int i = 0; i < classLoaders.length; i++) { - if (classLoaders[i] instanceof URLClassLoader) { - urls.addAll(Arrays.asList(((URLClassLoader) classLoaders[i]).getURLs())); - } else { - throw new RuntimeException("ClassLoader '" + classLoaders[i] + " is not an instance of URLClassLoader"); - } + final String[] classpathList = System.getProperty("java.class.path").split(":"); + final List urlList = new ArrayList<>(); + for (String file : classpathList) { + urlList.add(new File(file).toURI().toURL()); } - ConfigurationBuilder confBuilder = new ConfigurationBuilder(); - confBuilder.setUrls(urls); - return new Reflections(confBuilder); + return new Reflections(new ConfigurationBuilder().setUrls(urlList)); } private final Reflections reflections; @@ -70,7 +60,7 @@ public ConnectorDocGenerator() throws Exception { this.reflections = newReflections(); } - private void generateConnectorYaml(Class configClass, PrintWriter writer) { + private void generateConnectorYamlFile(Class configClass, PrintWriter writer) { log.info("Processing connector config class : {}", configClass); writer.println("configs:"); @@ -82,7 +72,9 @@ private void generateConnectorYaml(Class configClass, PrintWriter writer) { } FieldDoc fieldDoc = field.getDeclaredAnnotation(FieldDoc.class); if (null == fieldDoc) { - throw new RuntimeException("Missing `FieldDoc` for field '" + field.getName() + "'"); + final String message = "Missing FieldDoc for field '%s' in class '%s'." + .formatted(field.getName(), configClass.getCanonicalName()); + throw new RuntimeException(message); } writer.println(INDENT + "# " + fieldDoc.help()); String fieldPrefix = ""; @@ -99,28 +91,28 @@ private void generateConnectorYaml(Class configClass, PrintWriter writer) { writer.flush(); } - private void generateConnectorYaml(Class connectorClass, Connector connectorDef, PrintWriter writer) { + private void generateConnectorYamlFile(Class connectorClass, Connector connectorDef, PrintWriter writer) { log.info("Processing connector definition : {}", connectorDef); writer.println("# " + connectorDef.type() + " connector : " + connectorClass.getName()); writer.println(); writer.println("# " + connectorDef.help()); writer.println(); - generateConnectorYaml(connectorDef.configClass(), writer); + generateConnectorYamlFile(connectorDef.configClass(), writer); } - private void generatorConnectorYamls(String outputDir) throws IOException { + private void generatorConnectorYamlFiles(String outputDir) throws IOException { Set> connectorClasses = reflections.getTypesAnnotatedWith(Connector.class); log.info("Retrieve all `Connector` annotated classes : {}", connectorClasses); for (Class connectorClass : connectorClasses) { - Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class); - try (FileOutputStream fos = new FileOutputStream( - Paths.get( - outputDir, - "pulsar-io-" + connectorDef.name() - + "-" + connectorDef.type().name().toLowerCase()).toString() + ".yml")) { + final Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class); + final String name = connectorDef.name().toLowerCase(); + final String type = connectorDef.type().name().toLowerCase(); + final String filename = "pulsar-io-%s-%s.yml".formatted(name, type); + final Path outputPath = Path.of(outputDir, filename); + try (FileOutputStream fos = new FileOutputStream(outputPath.toFile())) { PrintWriter pw = new PrintWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8)); - generateConnectorYaml(connectorClass, connectorDef, pw); + generateConnectorYamlFile(connectorClass, connectorDef, pw); pw.flush(); } } @@ -130,23 +122,14 @@ private void generatorConnectorYamls(String outputDir) throws IOException { * Args for stats generator. */ private static class MainArgs { - @Parameter( - names = { - "-o", "--output-dir" - }, - description = "The output dir to dump connector docs", - required = true - ) + names = {"-o", "--output-dir"}, + description = "The output dir to dump connector docs", + required = true) String outputDir = null; - @Parameter( - names = { - "-h", "--help" - }, - description = "Show this help message") + @Parameter(names = {"-h", "--help"}, description = "Show this help message") boolean help = false; - } public static void main(String[] args) throws Exception { @@ -169,7 +152,7 @@ public static void main(String[] args) throws Exception { } ConnectorDocGenerator docGen = new ConnectorDocGenerator(); - docGen.generatorConnectorYamls(mainArgs.outputDir); + docGen.generatorConnectorYamlFiles(mainArgs.outputDir); } } diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java index a8c7358bf9415..9f42dbda7be1b 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java @@ -242,6 +242,11 @@ public class ElasticSearchConfig implements Serializable { ) private String primaryFields = ""; + @FieldDoc( + required = false, + defaultValue = "", + help = "The SSL config for elastic search." + ) private ElasticSearchSslConfig ssl = new ElasticSearchSslConfig(); @FieldDoc( From 93fb4f8eb6bd12872f05d7c5ec6c00b719d53009 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 17 Apr 2023 11:46:34 +0800 Subject: [PATCH 113/280] [improve][build] Upgrade snakeyaml version to 2.0 (#20085) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 32 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++++++------- pom.xml | 6 ++-- pulsar-broker-auth-oidc/pom.xml | 12 +++++++ pulsar-functions/runtime/pom.xml | 14 ++++++++ .../KubernetesSecretsTokenAuthProvider.java | 10 +++--- .../runtime/kubernetes/KubernetesRuntime.java | 11 +++---- .../kubernetes/KubernetesRuntimeFactory.java | 2 +- ...ubernetesSecretsTokenAuthProviderTest.java | 2 +- .../KubernetesRuntimeFactoryTest.java | 6 ++-- pulsar-functions/secrets/pom.xml | 14 ++++++++ pulsar-io/kafka-connect-adaptor/pom.xml | 7 ++++ pulsar-sql/presto-distribution/LICENSE | 30 ++++++++--------- src/owasp-dependency-check-suppressions.xml | 8 ----- 15 files changed, 109 insertions(+), 71 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index b22abf286f693..04b25cced42ac 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -52,7 +52,7 @@ 4.2.3 31.0.1-jre 1.10.12 - 1.32 + 2.0 3.12.4 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c321e54de2e01..50ff83a0ab11f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -246,21 +246,21 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.13.4.jar - - com.fasterxml.jackson.core-jackson-core-2.13.4.jar - - com.fasterxml.jackson.core-jackson-databind-2.13.4.2.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.4.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.4.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.4.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.4.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.4.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.13.4.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.13.4.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.13.4.jar + - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar + - com.fasterxml.jackson.core-jackson-core-2.14.2.jar + - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar - * Bitbucket -- org.bitbucket.b_c-jose4j-0.7.6.jar + * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.3.jar * Gson - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar @@ -402,7 +402,7 @@ The Apache Software License, Version 2.0 - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar - * SnakeYaml -- org.yaml-snakeyaml-1.32.jar + * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar @@ -453,9 +453,9 @@ The Apache Software License, Version 2.0 * Apache Yetus - org.apache.yetus-audience-annotations-0.12.0.jar * Kubernetes Client - - io.kubernetes-client-java-12.0.1.jar - - io.kubernetes-client-java-api-12.0.1.jar - - io.kubernetes-client-java-proto-12.0.1.jar + - io.kubernetes-client-java-18.0.0.jar + - io.kubernetes-client-java-api-18.0.0.jar + - io.kubernetes-client-java-proto-18.0.0.jar * Dropwizard - io.dropwizard.metrics-metrics-core-4.1.12.1.jar - io.dropwizard.metrics-metrics-graphite-4.1.12.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 0ac08caa1c01d..b4355ca8a38fa 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -311,17 +311,17 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * JCommander -- jcommander-1.82.jar * Jackson - - jackson-annotations-2.13.4.jar - - jackson-core-2.13.4.jar - - jackson-databind-2.13.4.2.jar - - jackson-dataformat-yaml-2.13.4.jar - - jackson-jaxrs-base-2.13.4.jar - - jackson-jaxrs-json-provider-2.13.4.jar - - jackson-module-jaxb-annotations-2.13.4.jar - - jackson-module-jsonSchema-2.13.4.jar - - jackson-datatype-jdk8-2.13.4.jar - - jackson-datatype-jsr310-2.13.4.jar - - jackson-module-parameter-names-2.13.4.jar + - jackson-annotations-2.14.2.jar + - jackson-core-2.14.2.jar + - jackson-databind-2.14.2.jar + - jackson-dataformat-yaml-2.14.2.jar + - jackson-jaxrs-base-2.14.2.jar + - jackson-jaxrs-json-provider-2.14.2.jar + - jackson-module-jaxb-annotations-2.14.2.jar + - jackson-module-jsonSchema-2.14.2.jar + - jackson-datatype-jdk8-2.14.2.jar + - jackson-datatype-jsr310-2.14.2.jar + - jackson-module-parameter-names-2.14.2.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson - gson-2.8.9.jar @@ -407,7 +407,7 @@ The Apache Software License, Version 2.0 - websocket-api-9.4.48.v20220622.jar - websocket-client-9.4.48.v20220622.jar - websocket-common-9.4.48.v20220622.jar - * SnakeYaml -- snakeyaml-1.32.jar + * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar * Javassist -- javassist-3.25.0-GA.jar * Apache Avro diff --git a/pom.xml b/pom.xml index 094a35b5ed13a..e18222604e621 100644 --- a/pom.xml +++ b/pom.xml @@ -148,7 +148,7 @@ flexible messaging model and an intuitive client API. 1.69 1.0.6 1.0.2.3 - 2.13.4.20221013 + 2.14.2 0.10.2 1.6.2 8.37 @@ -224,7 +224,7 @@ flexible messaging model and an intuitive client API. 2.3.3 2.0.2 5.12.1 - 12.0.1 + 18.0.0 4.9.3 2.8.0 @@ -236,7 +236,7 @@ flexible messaging model and an intuitive client API. 4.5.13 4.4.15 0.5.11 - 1.32 + 2.0 1.10.12 5.3.3 3.4.3 diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index 9ad0363775a13..7bf3226da761b 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -83,6 +83,18 @@ io.prometheus simpleclient_httpserver + + bcpkix-jdk18on + org.bouncycastle + + + bcutil-jdk18on + org.bouncycastle + + + bcprov-jdk18on + org.bouncycastle + diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 5558ced4c0749..0bc86b83c1668 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -64,6 +64,20 @@ io.kubernetes client-java ${kubernetesclient.version} + + + bcpkix-jdk18on + org.bouncycastle + + + bcutil-jdk18on + org.bouncycastle + + + bcprov-jdk18on + org.bouncycastle + + ${project.groupId} diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java index 1053e6e170ef1..e0543f28ac160 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java @@ -205,8 +205,7 @@ public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional { try { - coreClient.readNamespacedSecret(secretName, kubeNamespace, - null, null, null); + coreClient.readNamespacedSecret(secretName, kubeNamespace, null); } catch (ApiException e) { // statefulset is gone @@ -305,12 +304,13 @@ private void upsertSecret(String token, Function.FunctionDetails funcDetails, St .data(buildSecretMap(token)); try { - coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null); + coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null, null); } catch (ApiException e) { if (e.getCode() == HTTP_CONFLICT) { try { coreClient - .replaceNamespacedSecret(secretName, kubeNamespace, v1Secret, null, null, null); + .replaceNamespacedSecret(secretName, kubeNamespace, v1Secret, + null, null, null, null); return Actions.ActionResult.builder().success(true).build(); } catch (ApiException e1) { @@ -366,7 +366,7 @@ private String createSecret(String token, Function.FunctionDetails funcDetails) .metadata(new V1ObjectMeta().name(getSecretName(id))) .data(buildSecretMap(token)); try { - coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null); + coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null, null); } catch (ApiException e) { // already exists if (e.getCode() == HTTP_CONFLICT) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index e206862b68a67..0a5e1268c89a5 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -472,7 +472,7 @@ private void submitService() throws Exception { .supplier(() -> { final V1Service response; try { - response = coreClient.createNamespacedService(jobNamespace, service, null, null, null); + response = coreClient.createNamespacedService(jobNamespace, service, null, null, null, null); } catch (ApiException e) { // already exists if (e.getCode() == HTTP_CONFLICT) { @@ -561,7 +561,8 @@ private void submitStatefulSet() throws Exception { .supplier(() -> { final V1StatefulSet response; try { - response = appsClient.createNamespacedStatefulSet(jobNamespace, statefulSet, null, null, null); + response = appsClient.createNamespacedStatefulSet(jobNamespace, statefulSet, + null, null, null, null); } catch (ApiException e) { // already exists if (e.getCode() == HTTP_CONFLICT) { @@ -657,8 +658,7 @@ public void deleteStatefulSet() throws InterruptedException { .supplier(() -> { V1StatefulSet response; try { - response = appsClient.readNamespacedStatefulSet(statefulSetName, jobNamespace, - null, null, null); + response = appsClient.readNamespacedStatefulSet(statefulSetName, jobNamespace, null); } catch (ApiException e) { // statefulset is gone if (e.getCode() == HTTP_NOT_FOUND) { @@ -805,8 +805,7 @@ public void deleteService() throws InterruptedException { .supplier(() -> { V1Service response; try { - response = coreClient.readNamespacedService(serviceName, jobNamespace, - null, null, null); + response = coreClient.readNamespacedService(serviceName, jobNamespace, null); } catch (ApiException e) { // service is gone diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 895304138a530..3e1d40e80dc6e 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -405,7 +405,7 @@ static void fetchConfigMap(CoreV1Api coreClient, String changeConfigMap, KubernetesRuntimeFactory kubernetesRuntimeFactory) { try { V1ConfigMap v1ConfigMap = - coreClient.readNamespacedConfigMap(changeConfigMap, changeConfigMapNamespace, null, true, false); + coreClient.readNamespacedConfigMap(changeConfigMap, changeConfigMapNamespace, null); Map data = v1ConfigMap.getData(); if (data != null) { overRideKubernetesConfig(data, kubernetesRuntimeFactory); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java index 081e693b6a321..cf294afcf9b78 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java @@ -103,7 +103,7 @@ public void testConfigureAuthDataStatefulSetNoCa() { @Test public void testCacheAuthData() throws ApiException { CoreV1Api coreV1Api = mock(CoreV1Api.class); - doReturn(new V1Secret()).when(coreV1Api).createNamespacedSecret(anyString(), any(), anyString(), anyString(), anyString()); + doReturn(new V1Secret()).when(coreV1Api).createNamespacedSecret(anyString(), any(), anyString(), anyString(), anyString(), anyString()); KubernetesSecretsTokenAuthProvider kubernetesSecretsTokenAuthProvider = new KubernetesSecretsTokenAuthProvider(); kubernetesSecretsTokenAuthProvider.initialize(coreV1Api, null, (fd) -> "default"); Function.FunctionDetails funcDetails = Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-ns").setName("test-func").build(); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java index a5fc8f231a6b5..48497bf218d40 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java @@ -468,9 +468,9 @@ public void testDynamicConfigMapLoading() throws Exception { KubernetesRuntimeFactory kubernetesRuntimeFactory = getKuberentesRuntimeFactory(); CoreV1Api coreV1Api = Mockito.mock(CoreV1Api.class); V1ConfigMap v1ConfigMap = new V1ConfigMap(); - Mockito.doReturn(v1ConfigMap).when(coreV1Api).readNamespacedConfigMap(any(), any(), any(), any(), any()); + Mockito.doReturn(v1ConfigMap).when(coreV1Api).readNamespacedConfigMap(any(), any(), any()); KubernetesRuntimeFactory.fetchConfigMap(coreV1Api, changeConfigMap, changeConfigNamespace, kubernetesRuntimeFactory); - Mockito.verify(coreV1Api, Mockito.times(1)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null), eq(true), eq(false)); + Mockito.verify(coreV1Api, Mockito.times(1)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null)); KubernetesRuntimeFactory expected = getKuberentesRuntimeFactory(); assertEquals(kubernetesRuntimeFactory, expected); @@ -479,7 +479,7 @@ public void testDynamicConfigMapLoading() throws Exception { configs.put("imagePullPolicy", "test_imagePullPolicy2"); v1ConfigMap.setData(configs); KubernetesRuntimeFactory.fetchConfigMap(coreV1Api, changeConfigMap, changeConfigNamespace, kubernetesRuntimeFactory); - Mockito.verify(coreV1Api, Mockito.times(2)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null), eq(true), eq(false)); + Mockito.verify(coreV1Api, Mockito.times(2)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null)); assertEquals(kubernetesRuntimeFactory.getPulsarDockerImageName(), "test_dockerImage2"); assertEquals(kubernetesRuntimeFactory.getImagePullPolicy(), "test_imagePullPolicy2"); diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 08d4e9bf63ab8..f1fc43fcdcc5a 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -35,6 +35,20 @@ io.kubernetes client-java ${kubernetesclient.version} + + + bcpkix-jdk18on + org.bouncycastle + + + bcutil-jdk18on + org.bouncycastle + + + bcprov-jdk18on + org.bouncycastle + + diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 12ebda087eb3d..8651b378ed118 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -166,6 +166,13 @@ test + + org.bouncycastle + bc-fips + ${bouncycastle.bc-fips.version} + test + + com.typesafe.netty netty-reactive-streams diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 214a928a74941..d281ba1f3643e 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.13.4.jar - - jackson-core-2.13.4.jar - - jackson-databind-2.13.4.2.jar - - jackson-dataformat-smile-2.13.4.jar - - jackson-datatype-guava-2.13.4.jar - - jackson-datatype-jdk8-2.13.4.jar - - jackson-datatype-joda-2.13.4.jar - - jackson-datatype-jsr310-2.13.4.jar - - jackson-dataformat-yaml-2.13.4.jar - - jackson-jaxrs-base-2.13.4.jar - - jackson-jaxrs-json-provider-2.13.4.jar - - jackson-module-jaxb-annotations-2.13.4.jar - - jackson-module-jsonSchema-2.13.4.jar + - jackson-annotations-2.14.2.jar + - jackson-core-2.14.2.jar + - jackson-databind-2.14.2.jar + - jackson-dataformat-smile-2.14.2.jar + - jackson-datatype-guava-2.14.2.jar + - jackson-datatype-jdk8-2.14.2.jar + - jackson-datatype-joda-2.14.2.jar + - jackson-datatype-jsr310-2.14.2.jar + - jackson-dataformat-yaml-2.14.2.jar + - jackson-jaxrs-base-2.14.2.jar + - jackson-jaxrs-json-provider-2.14.2.jar + - jackson-module-jaxb-annotations-2.14.2.jar + - jackson-module-jsonSchema-2.14.2.jar * Guava - guava-31.0.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -401,7 +401,7 @@ The Apache Software License, Version 2.0 * RocksDB JNI - rocksdbjni-7.9.2.jar * SnakeYAML - - snakeyaml-1.32.jar + - snakeyaml-2.0.jar * Bean Validation API - validation-api-2.0.1.Final.jar * Objectsize @@ -456,7 +456,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.8.4.jar * Jackson - - jackson-module-parameter-names-2.13.4.jar + - jackson-module-parameter-names-2.14.2.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 84ed2a3332cd4..d599f2de3e85a 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -37,14 +37,6 @@ .* - - - e80612549feb5c9191c498de628c1aa80693cf0b - CVE-2022-1471 - - Date: Mon, 17 Apr 2023 19:32:13 +0800 Subject: [PATCH 114/280] [improve] [broker] Close temporary open ledger in BookkeeperBucketSnapshotStorage (#20111) --- .../BookkeeperBucketSnapshotStorage.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index e7d4f9301dd36..9c30ccf1c0b7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -66,22 +66,41 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0). - thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture snapshotFuture = + getLedgerEntry(ledgerHandle, 0, 0) + .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); + + snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return snapshotFuture; + }); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, - lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries)); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture> parseFuture = + getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) + .thenApply(this::parseSnapshotSegmentEntries); + + parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return parseFuture; + }); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenApply(LedgerHandle::getLength); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture lengthFuture = + CompletableFuture.completedFuture(ledgerHandle.getLength()); + + lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return lengthFuture; + }); } @Override From 092819b5c4a68dbc39d9a650d5370af9455e805d Mon Sep 17 00:00:00 2001 From: xiaolong ran Date: Tue, 18 Apr 2023 09:45:19 +0800 Subject: [PATCH 115/280] [improve] [broker] Fix broker restart logic (#20113) --- .../java/org/apache/pulsar/broker/PulsarService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index aad7d32f732c5..5fc9920d0f215 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -467,6 +467,12 @@ public CompletableFuture closeAsync() { protocolHandlers = null; } + // cancel loadShedding task and shutdown the loadManager executor before shutting down the broker + if (this.loadSheddingTask != null) { + this.loadSheddingTask.cancel(); + } + executorServicesShutdown.shutdown(loadManagerExecutor); + List> asyncCloseFutures = new ArrayList<>(); if (this.brokerService != null) { CompletableFuture brokerCloseFuture = this.brokerService.closeAsync(); @@ -501,12 +507,6 @@ public CompletableFuture closeAsync() { this.leaderElectionService = null; } - // cancel loadShedding task and shutdown the loadManager executor before shutting down the broker - if (this.loadSheddingTask != null) { - this.loadSheddingTask.cancel(); - } - executorServicesShutdown.shutdown(loadManagerExecutor); - if (adminClient != null) { adminClient.close(); adminClient = null; From 6cfa4683a44e7cce39fa6cb70e0fe1fb3d5eae56 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Mon, 17 Apr 2023 22:57:48 -0700 Subject: [PATCH 116/280] [improve][broker] Fix ServiceUnitStateCompactionStrategy to cover fast-forward cursor behavior after compaction (#20110) Master Issue: https://github.com/apache/pulsar/issues/16691 ### Motivation Raising a PR to implement: https://github.com/apache/pulsar/issues/16691 After the compaction, the cursor can fast-forward to the compacted horizon when a large number of messages are compacted before the next read. Hence, ServiceUnitStateCompactionStrategy also needs to cover this case. Currently, the existing and slow(their states are far behind) tableviews with ServiceUnitStateCompactionStrategy could not accept those compacted messages. In the load balance extension context, this means the ownership data could be inconsistent among brokers. ### Modifications This PR - fixes ServiceUnitStateCompactionStrategy to accept the state data if its version is bigger than the current version +1. - (minor fix) does not repeatedly update the replication_clusters in the policies when creating the system namespace. This update redundantly triggers ZK watchers when restarting brokers. - sets closeWithoutWaitingClientDisconnect=true, upon unload(following the same setting as the modular LM's) --- .../pulsar/PulsarClusterMetadataSetup.java | 17 +++- .../channel/ServiceUnitStateChannelImpl.java | 2 +- .../ServiceUnitStateCompactionStrategy.java | 10 ++- .../StrategicTwoPhaseCompactor.java | 2 +- .../ExtensibleLoadManagerImplTest.java | 2 + ...erviceUnitStateCompactionStrategyTest.java | 4 + .../ServiceUnitStateCompactionTest.java | 79 +++++++++++++++++++ .../pulsar/client/impl/TableViewImpl.java | 7 ++ 8 files changed, 114 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index 0badbda1afdfd..9b757c55ccd1d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -385,10 +385,19 @@ static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName nam namespaceResources.createPolicies(namespaceName, policies); } else { log.info("Namespace {} already exists.", namespaceName); - namespaceResources.setPolicies(namespaceName, policies -> { - policies.replication_clusters.add(cluster); - return policies; - }); + var replicaClusterFound = false; + var policiesOptional = namespaceResources.getPolicies(namespaceName); + if (policiesOptional.isPresent() && policiesOptional.get().replication_clusters.contains(cluster)) { + replicaClusterFound = true; + } + if (!replicaClusterFound) { + namespaceResources.setPolicies(namespaceName, policies -> { + policies.replication_clusters.add(cluster); + return policies; + }); + log.info("Updated namespace:{} policies. Added the replication cluster:{}", + namespaceName, cluster); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 68c6440e68e4b..b4c4e7fd5d42a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -819,7 +819,7 @@ private CompletableFuture closeServiceUnit(String serviceUnit) { NamespaceBundle bundle = getNamespaceBundle(serviceUnit); return pulsar.getBrokerService().unloadServiceUnit( bundle, - false, + true, pulsar.getConfig().getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS) .thenApply(numUnloadedTopics -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index ceb3ea3e9cb6c..72b05b5cd62c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -52,9 +52,13 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to return false; } - // Skip the compaction case where from = null and to.versionId > 1 - if (from != null && from.versionId() + 1 != to.versionId()) { - return true; + if (from != null) { + if (from.versionId() == Long.MAX_VALUE && to.versionId() == Long.MIN_VALUE) { // overflow + } else if (from.versionId() >= to.versionId()) { + return true; + } else if (from.versionId() < to.versionId() - 1) { // Compacted + return false; + } // else from.versionId() == to.versionId() - 1 // continue to check further } if (to.force()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index 557d4a6580161..a6b0942742763 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -379,7 +379,7 @@ private CompletableFuture runPhaseTwo( }); }) .thenCompose(v -> { - log.info("Acking ledger id {}", phaseOneResult.firstId); + log.info("Acking ledger id {}", phaseOneResult.lastId); return ((CompactionReaderImpl) reader) .acknowledgeCumulativeAsync( phaseOneResult.lastId, Map.of(COMPACTED_TOPIC_LEDGER_PROPERTY, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 9ab2467d0ef1b..b71eeb4745b87 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -592,6 +592,8 @@ public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Except restartBroker(); pulsar1 = pulsar; setPrimaryLoadManager(); + admin.namespaces().setNamespaceReplicationClusters("public/default", + Sets.newHashSet(this.conf.getClusterName())); var serviceUnitStateChannelPrimaryNew = (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 64964826af652..62de91dab292b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -85,6 +85,10 @@ public void testVersionId(){ new ServiceUnitStateData(Owned, dst, src, 10), new ServiceUnitStateData(Releasing, "broker2", dst, 5))); + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, 10), + new ServiceUnitStateData(Owned, "broker2", dst, 12))); + } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 1a69a86f7c6b2..e4f0750a981c9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -27,6 +27,9 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -49,6 +52,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.bookkeeper.client.BookKeeper; import org.apache.commons.lang.reflect.FieldUtils; @@ -69,6 +73,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.impl.ReaderImpl; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; @@ -628,6 +633,80 @@ public void testSlowTableviewAfterCompaction() throws Exception { } + @Test + public void testSlowReceiveTableviewAfterCompaction() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; + String strategyClassName = "topicCompactionStrategyClassName"; + + pulsarClient.newConsumer(schema) + .topic(topic) + .subscriptionName("sub1") + .readCompacted(true) + .subscribe().close(); + + var tv = pulsar.getClient().newTableViewBuilder(schema) + .topic(topic) + .subscriptionName("slowTV") + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + + // Configure retention to ensue data is retained for reader + admin.namespaces().setRetention("my-property/use/my-ns", + new RetentionPolicies(-1, -1)); + + Producer producer = pulsarClient.newProducer(schema) + .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) + .enableBatching(true) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + StrategicTwoPhaseCompactor compactor + = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + + var reader = ((CompletableFuture>) FieldUtils + .readDeclaredField(tv, "reader", true)).get(); + var consumer = spy(reader.getConsumer()); + FieldUtils.writeDeclaredField(reader, "consumer", consumer, true); + String bundle = "bundle1"; + final AtomicInteger versionId = new AtomicInteger(0); + final AtomicInteger cnt = new AtomicInteger(1); + int msgAddCount = 1000; // has to be big enough to cover compacted cursor fast-forward. + doAnswer(invocationOnMock -> { + if (cnt.decrementAndGet() == 0) { + var msg = consumer.receiveAsync(); + for (int i = 0; i < msgAddCount; i++) { + producer.newMessage().key(bundle).value( + new ServiceUnitStateData(Owned, "broker" + versionId.incrementAndGet(), true, + versionId.get())).send(); + } + compactor.compact(topic, strategy).join(); + return msg; + } + // Call the real method + reset(consumer); + return consumer.receiveAsync(); + }).when(consumer).receiveAsync(); + producer.newMessage().key(bundle).value( + new ServiceUnitStateData(Owned, "broker", true, + versionId.incrementAndGet())).send(); + producer.newMessage().key(bundle).value( + new ServiceUnitStateData(Owned, "broker" + versionId.incrementAndGet(), true, + versionId.get())).send(); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted( + () -> { + var val = tv.get(bundle); + assertNotNull(val); + assertEquals(val.dstBroker(), "broker" + versionId.get()); + } + ); + + producer.close(); + tv.close(); + } + @Test public void testBrokerRestartAfterCompaction() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index ff5b251ad55ff..77aba7e48cbad 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -192,6 +192,13 @@ private void handleMessage(Message msg) { if (compactionStrategy != null) { T prev = data.get(key); update = !compactionStrategy.shouldKeepLeft(prev, cur); + if (!update) { + log.info("Skipped the message from topic {}. key={} value={} prev={}", + conf.getTopicName(), + key, + cur, + prev); + } } if (update) { From 6c89467c74bf7e4bc7de06ed5e287643819118ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Mikl=C3=B3s?= Date: Tue, 18 Apr 2023 11:13:50 +0200 Subject: [PATCH 117/280] [improve][fn] Upgrade Kotlin version to1.8.20 (#20114) --- distribution/server/src/assemble/LICENSE.bin.txt | 8 ++++---- pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 50ff83a0ab11f..a45c5b7f04c14 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -412,10 +412,10 @@ The Apache Software License, Version 2.0 * Okio - com.squareup.okio-okio-2.8.0.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar * Kotlin Standard Lib - - org.jetbrains.kotlin-kotlin-stdlib-1.4.32.jar - - org.jetbrains.kotlin-kotlin-stdlib-common-1.4.32.jar - - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.4.32.jar - - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.4.32.jar + - org.jetbrains.kotlin-kotlin-stdlib-1.8.20.jar + - org.jetbrains.kotlin-kotlin-stdlib-common-1.8.20.jar + - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.8.20.jar + - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar - org.jetbrains-annotations-13.0.jar * gRPC - io.grpc-grpc-all-1.45.1.jar diff --git a/pom.xml b/pom.xml index e18222604e621..0945b00c8d659 100644 --- a/pom.xml +++ b/pom.xml @@ -229,7 +229,7 @@ flexible messaging model and an intuitive client API. 2.8.0 - 1.4.32 + 1.8.20 1.0 9.1.6 5.3.26 From 78cf5a6097fe4b70e51724921ce174bcac919a77 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 19 Apr 2023 01:27:39 +0800 Subject: [PATCH 118/280] [fix][admin] Add javax.xml.bind:jaxb-api to shade (#20106) Signed-off-by: nodece --- pulsar-client-admin-shaded/pom.xml | 5 +++++ pulsar-client-all/pom.xml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index cf669b5869afb..b7c2d996bc5e5 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -131,6 +131,7 @@ com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* javax.ws.rs:* + javax.xml.bind:jaxb-api jakarta.annotation:* org.glassfish.hk2*:* io.grpc:* @@ -230,6 +231,10 @@ javax.annotation org.apache.pulsar.shade.javax.annotation + + javax.xml.bind + org.apache.pulsar.shade.javax.xml.bind + jersey org.apache.pulsar.shade.jersey diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 73621954e1fb8..cf39c5db256ae 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -182,6 +182,7 @@ com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* javax.ws.rs:* + javax.xml.bind:jaxb-api jakarta.annotation:* org.glassfish.hk2*:* io.grpc:* @@ -285,6 +286,10 @@ javax.annotation org.apache.pulsar.shade.javax.annotation + + javax.xml.bind + org.apache.pulsar.shade.javax.xml.bind + jersey org.apache.pulsar.shade.jersey From bdb2a96a41a4ada356888fe01fc1a8e0a345ddd4 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Wed, 19 Apr 2023 03:01:31 +0800 Subject: [PATCH 119/280] [improve] [broker] Upgrade the BookKeeper dependency to 4.16.1 (#20127) --- .../server/src/assemble/LICENSE.bin.txt | 56 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 6 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 30 +++++----- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a45c5b7f04c14..26651246ddf48 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -346,34 +346,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.0.jar - - org.apache.bookkeeper-circe-checksum-4.16.0.jar - - org.apache.bookkeeper-cpu-affinity-4.16.0.jar - - org.apache.bookkeeper-statelib-4.16.0.jar - - org.apache.bookkeeper-stream-storage-api-4.16.0.jar - - org.apache.bookkeeper-stream-storage-common-4.16.0.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.0.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.0.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.0.jar - - org.apache.bookkeeper-stream-storage-server-4.16.0.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.0.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.0.jar - - org.apache.bookkeeper.http-http-server-4.16.0.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.0.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.0.jar - - org.apache.distributedlog-distributedlog-common-4.16.0.jar - - org.apache.distributedlog-distributedlog-core-4.16.0-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.0.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.0.jar - - org.apache.bookkeeper-native-io-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.1.jar + - org.apache.bookkeeper-circe-checksum-4.16.1.jar + - org.apache.bookkeeper-cpu-affinity-4.16.1.jar + - org.apache.bookkeeper-statelib-4.16.1.jar + - org.apache.bookkeeper-stream-storage-api-4.16.1.jar + - org.apache.bookkeeper-stream-storage-common-4.16.1.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.1.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.1.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.1.jar + - org.apache.bookkeeper-stream-storage-server-4.16.1.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.1.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.1.jar + - org.apache.bookkeeper.http-http-server-4.16.1.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.1.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.1.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.1.jar + - org.apache.distributedlog-distributedlog-common-4.16.1.jar + - org.apache.distributedlog-distributedlog-core-4.16.1-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.1.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.1.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.1.jar + - org.apache.bookkeeper-native-io-4.16.1.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index b4355ca8a38fa..711890809f1bf 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -390,9 +390,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.0.jar - - cpu-affinity-4.16.0.jar - - circe-checksum-4.16.0.jar + - bookkeeper-common-allocator-4.16.1.jar + - cpu-affinity-4.16.1.jar + - circe-checksum-4.16.1.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 0945b00c8d659..0b5ab09d4581a 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.16.0 + 4.16.1 3.8.1 1.5.0 1.10.0 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d281ba1f3643e..09d1396b70419 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -426,21 +426,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.16.0.jar - - bookkeeper-common-allocator-4.16.0.jar - - bookkeeper-proto-4.16.0.jar - - bookkeeper-server-4.16.0.jar - - bookkeeper-stats-api-4.16.0.jar - - bookkeeper-tools-framework-4.16.0.jar - - circe-checksum-4.16.0.jar - - codahale-metrics-provider-4.16.0.jar - - cpu-affinity-4.16.0.jar - - http-server-4.16.0.jar - - prometheus-metrics-provider-4.16.0.jar - - codahale-metrics-provider-4.16.0.jar - - bookkeeper-slogger-api-4.16.0.jar - - bookkeeper-slogger-slf4j-4.16.0.jar - - native-io-4.16.0.jar + - bookkeeper-common-4.16.1.jar + - bookkeeper-common-allocator-4.16.1.jar + - bookkeeper-proto-4.16.1.jar + - bookkeeper-server-4.16.1.jar + - bookkeeper-stats-api-4.16.1.jar + - bookkeeper-tools-framework-4.16.1.jar + - circe-checksum-4.16.1.jar + - codahale-metrics-provider-4.16.1.jar + - cpu-affinity-4.16.1.jar + - http-server-4.16.1.jar + - prometheus-metrics-provider-4.16.1.jar + - codahale-metrics-provider-4.16.1.jar + - bookkeeper-slogger-api-4.16.1.jar + - bookkeeper-slogger-slf4j-4.16.1.jar + - native-io-4.16.1.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar From 46a65fdc182a34753ebabfa35f63c0f6c765462f Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Tue, 18 Apr 2023 14:37:41 -0700 Subject: [PATCH 120/280] [fix][io] KCA: handle kafka sources that use commitRecord (#20121) --- .../io/kafka/connect/KafkaConnectSource.java | 43 ++++++- .../connect/ErrRecFileStreamSourceTask.java | 33 +++++ .../connect/KafkaConnectSourceErrRecTest.java | 118 ++++++++++++++++++ 3 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index f5f6efd08bd99..f2ee8a8e6cafe 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -27,6 +27,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.kafka.connect.source.SourceRecord; import org.apache.pulsar.client.api.Schema; @@ -57,6 +59,7 @@ public void open(Map config, SourceContext sourceContext) throws config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, false); } log.info("jsonWithEnvelope: {}", jsonWithEnvelope); + super.open(config, sourceContext); } @@ -69,17 +72,26 @@ public synchronized KafkaSourceRecord processSourceRecord(final SourceRecord src private static final AvroData avroData = new AvroData(1000); - private class KafkaSourceRecord extends AbstractKafkaSourceRecord> + public class KafkaSourceRecord extends AbstractKafkaSourceRecord> implements KVRecord { + final int keySize; + final int valueSize; + + final SourceRecord srcRecord; + KafkaSourceRecord(SourceRecord srcRecord) { super(srcRecord); + this.srcRecord = srcRecord; + byte[] keyBytes = keyConverter.fromConnectData( srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); + keySize = keyBytes != null ? keyBytes.length : 0; this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); byte[] valueBytes = valueConverter.fromConnectData( srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value()); + valueSize = valueBytes != null ? valueBytes.length : 0; this.value = new KeyValue<>(keyBytes, valueBytes); @@ -145,6 +157,35 @@ public KeyValueEncodingType getKeyValueEncodingType() { } } + @Override + public void ack() { + // first try to commitRecord() for the current record in the batch + // then call super.ack() which calls commit() after complete batch of records is processed + try { + if (log.isDebugEnabled()) { + log.debug("commitRecord() for record: {}", srcRecord); + } + getSourceTask().commitRecord(srcRecord, + new RecordMetadata( + new TopicPartition(srcRecord.topic() == null + ? topicName.orElse("UNDEFINED") + : srcRecord.topic(), + srcRecord.kafkaPartition() == null ? 0 : srcRecord.kafkaPartition()), + -1L, // baseOffset == -1L means no offset + 0, // batchIndex, doesn't matter if baseOffset == -1L + null == srcRecord.timestamp() ? -1L : srcRecord.timestamp(), + keySize, // serializedKeySize + valueSize // serializedValueSize + )); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("Source task failed to commit record, " + + "source task should resend data, will get duplicate", e); + return; + } + super.ack(); + } + } } diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java new file mode 100644 index 0000000000000..cbdd4c41bf692 --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.io.kafka.connect; + +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.connect.file.FileStreamSourceTask; +import org.apache.kafka.connect.source.SourceRecord; + +public class ErrRecFileStreamSourceTask extends FileStreamSourceTask { + + @Override + public void commitRecord(SourceRecord record, RecordMetadata metadata) throws InterruptedException { + throw new org.apache.kafka.connect.errors.ConnectException("blah"); + } + +} diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java new file mode 100644 index 0000000000000..9872e1fbc7e2f --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.kafka.connect; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.connect.file.FileStreamSourceConnector; +import org.apache.kafka.connect.runtime.TaskConfig; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.SourceContext; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import java.io.File; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * Test the implementation of {@link KafkaConnectSource}. + */ +@Slf4j +public class KafkaConnectSourceErrRecTest extends ProducerConsumerBase { + + private Map config = new HashMap<>(); + private String offsetTopicName; + // The topic to publish data to, for kafkaSource + private String topicName; + private KafkaConnectSource kafkaConnectSource; + private File tempFile; + private SourceContext context; + private PulsarClient client; + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + + config.put(TaskConfig.TASK_CLASS_CONFIG, "org.apache.pulsar.io.kafka.connect.ErrRecFileStreamSourceTask"); + config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); + config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); + + this.offsetTopicName = "persistent://my-property/my-ns/kafka-connect-source-offset"; + config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName); + + this.topicName = "persistent://my-property/my-ns/kafka-connect-source"; + config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName); + tempFile = File.createTempFile("some-file-name", null); + config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString()); + config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE)); + + this.context = mock(SourceContext.class); + this.client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .build(); + when(context.getPulsarClient()).thenReturn(this.client); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + if (this.client != null) { + this.client.close(); + } + tempFile.delete(); + super.internalCleanup(); + } + + @Test + public void testCommitRecordCalled() throws Exception { + kafkaConnectSource = new KafkaConnectSource(); + kafkaConnectSource.open(config, context); + + // use FileStreamSourceConnector, each line is a record, need "\n" and end of each record. + OutputStream os = Files.newOutputStream(tempFile.toPath()); + + String line1 = "This is the first line\n"; + os.write(line1.getBytes()); + os.flush(); + os.close(); + + Record> record = kafkaConnectSource.read(); + + assertTrue(record instanceof KafkaConnectSource.KafkaSourceRecord); + + try { + record.ack(); + fail("expected exception"); + } catch (Exception e) { + log.info("got exception", e); + assertTrue(e instanceof org.apache.kafka.connect.errors.ConnectException); + } + } +} From 58ccf020fc0fa5f7d9b70032ae60c8bf9a3f234f Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 18 Apr 2023 20:24:58 -0500 Subject: [PATCH 121/280] [fix][broker] Implement authenticateAsync for AuthenticationProviderList (#20132) PIP: #12105 and #19771 ### Motivation With the implementation of asynchronous authentication in PIP 97, I missed a case in the `AuthenticationProviderList` where we need to implement the `authenticateAsync` methods. This PR is necessary for making the `AuthenticationProviderToken` and the `AuthenticationProviderOpenID` work together, which is necessary for anyone transitioning to `AuthenticationProviderOpenID`. ### Modifications * Implement `AuthenticationListState#authenticateAsync` using a recursive algorithm that first attempts to authenticate the client using the current `authState` and then tries the remaining options. * Implement `AuthenticationProviderList#authenticateAsync` using a recursive algorithm that attempts each provider sequentially. * Add test to `AuthenticationProviderListTest` that exercises this method. It didn't technically fail previously, but it's worth adding. * Add test to `AuthenticationProviderOpenIDIntegrationTest` to cover the exact failures that were causing problems. --- ...ticationProviderOpenIDIntegrationTest.java | 57 +++++++++++ .../AuthenticationProviderList.java | 95 ++++++++++++++++++- .../AuthenticationProviderListTest.java | 24 +++++ 3 files changed, 175 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java index 298492652c0a8..0075d70f599d3 100644 --- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java @@ -31,6 +31,7 @@ import com.github.tomakehurst.wiremock.WireMockServer; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.impl.DefaultJwtBuilder; +import io.jsonwebtoken.io.Decoders; import io.jsonwebtoken.security.Keys; import java.io.IOException; import java.nio.file.Files; @@ -41,13 +42,19 @@ import java.util.Base64; import java.util.Date; import java.util.HashMap; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; import javax.naming.AuthenticationException; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.common.api.AuthData; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -438,6 +445,56 @@ public void testAuthenticationStateOpenIDForTokenExpiration() throws Exception { assertTrue(state.isExpired()); } + /** + * This test covers the migration scenario where you have both the Token and OpenID providers. It ensures + * both kinds of authentication work. + * @throws Exception + */ + @Test + public void testAuthenticationProviderListStateSuccess() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName(), + AuthenticationProviderToken.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); + + // Set up static token + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + // Use public key for validation + String publicKeyStr = AuthTokenUtils.encodeKeyBase64(keyPair.getPublic()); + props.setProperty("tokenPublicKey", publicKeyStr); + // Use private key to generate token + String privateKeyStr = AuthTokenUtils.encodeKeyBase64(keyPair.getPrivate()); + PrivateKey privateKey = AuthTokenUtils.decodePrivateKey(Decoders.BASE64.decode(privateKeyStr), + SignatureAlgorithm.RS256); + String staticToken = AuthTokenUtils.createToken(privateKey, "superuser", Optional.empty()); + + @Cleanup + AuthenticationService service = new AuthenticationService(conf); + AuthenticationProvider provider = service.getAuthenticationProvider("token"); + + // First, authenticate using OIDC + String role = "superuser"; + String oidcToken = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(oidcToken)).get()); + + // Authenticate using the static token + assertEquals("superuser", provider.authenticateAsync(new AuthenticationDataCommand(staticToken)).get()); + + // Use authenticationState to authentication using OIDC + AuthenticationState state1 = service.getAuthenticationProvider("token").newAuthState(null, null, null); + assertNull(state1.authenticateAsync(AuthData.of(oidcToken.getBytes())).get()); + assertEquals(state1.getAuthRole(), role); + + // Use authenticationState to authentication using static token + AuthenticationState state2 = service.getAuthenticationProvider("token").newAuthState(null, null, null); + assertNull(state2.authenticateAsync(AuthData.of(staticToken.getBytes())).get()); + assertEquals(state1.getAuthRole(), role); + } + @Test void ensureRoleClaimForNonSubClaimReturnsRole() throws Exception { AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java index f8a96aa624e12..16d6f9859a0b4 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java @@ -22,6 +22,7 @@ import java.net.SocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; import javax.servlet.http.HttpServletRequest; @@ -76,9 +77,12 @@ static T applyAuthProcessor(List processors, AuthProcessor authF private static class AuthenticationListState implements AuthenticationState { private final List states; - private AuthenticationState authState; + private volatile AuthenticationState authState; AuthenticationListState(List states) { + if (states == null || states.isEmpty()) { + throw new IllegalArgumentException("Authentication state requires at least one state"); + } this.states = states; this.authState = states.get(0); } @@ -96,6 +100,61 @@ public String getAuthRole() throws AuthenticationException { return getAuthState().getAuthRole(); } + @Override + public CompletableFuture authenticateAsync(AuthData authData) { + // First, attempt to authenticate with the current auth state + CompletableFuture authChallengeFuture = new CompletableFuture<>(); + authState + .authenticateAsync(authData) + .whenComplete((authChallenge, ex) -> { + if (ex == null) { + // Current authState is still correct. Just need to return the authChallenge. + authChallengeFuture.complete(authChallenge); + } else { + if (log.isDebugEnabled()) { + log.debug("Authentication failed for auth provider " + authState.getClass() + ": ", ex); + } + authenticateRemainingAuthStates(authChallengeFuture, authData, ex, states.size() - 1); + } + }); + return authChallengeFuture; + } + + private void authenticateRemainingAuthStates(CompletableFuture authChallengeFuture, + AuthData clientAuthData, + Throwable previousException, + int index) { + if (index < 0) { + if (previousException == null) { + previousException = new AuthenticationException("Authentication required"); + } + AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), + "authentication-provider-list", "Authentication required"); + authChallengeFuture.completeExceptionally(previousException); + return; + } + AuthenticationState state = states.get(index); + if (state == authState) { + // Skip the current auth state + authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, null, index - 1); + } else { + state.authenticateAsync(clientAuthData) + .whenComplete((authChallenge, ex) -> { + if (ex == null) { + // Found the correct auth state + authState = state; + authChallengeFuture.complete(authChallenge); + } else { + if (log.isDebugEnabled()) { + log.debug("Authentication failed for auth provider " + + authState.getClass() + ": ", ex); + } + authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, ex, index - 1); + } + }); + } + } + @Override public AuthData authenticate(AuthData authData) throws AuthenticationException { return applyAuthProcessor( @@ -160,6 +219,40 @@ public String getAuthMethodName() { return providers.get(0).getAuthMethodName(); } + @Override + public CompletableFuture authenticateAsync(AuthenticationDataSource authData) { + CompletableFuture roleFuture = new CompletableFuture<>(); + authenticateRemainingAuthProviders(roleFuture, authData, null, providers.size() - 1); + return roleFuture; + } + + private void authenticateRemainingAuthProviders(CompletableFuture roleFuture, + AuthenticationDataSource authData, + Throwable previousException, + int index) { + if (index < 0) { + if (previousException == null) { + previousException = new AuthenticationException("Authentication required"); + } + AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), + "authentication-provider-list", "Authentication required"); + roleFuture.completeExceptionally(previousException); + return; + } + AuthenticationProvider provider = providers.get(index); + provider.authenticateAsync(authData) + .whenComplete((role, ex) -> { + if (ex == null) { + roleFuture.complete(role); + } else { + if (log.isDebugEnabled()) { + log.debug("Authentication failed for auth provider " + provider.getClass() + ": ", ex); + } + authenticateRemainingAuthProviders(roleFuture, authData, ex, index - 1); + } + }); + } + @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { return applyAuthProcessor( diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java index df011412fee85..7793a5c029f2a 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java @@ -161,6 +161,30 @@ public void testAuthenticate() throws Exception { testAuthenticate(tokenBB, SUBJECT_B); } + private void testAuthenticateAsync(String token, String expectedSubject) throws Exception { + String actualSubject = authProvider.authenticateAsync(new AuthenticationDataSource() { + @Override + public boolean hasDataFromCommand() { + return true; + } + + @Override + public String getCommandData() { + return token; + } + }).get(); + assertEquals(actualSubject, expectedSubject); + } + + @Test + public void testAuthenticateAsync() throws Exception { + testAuthenticateAsync(tokenAA, SUBJECT_A); + testAuthenticateAsync(tokenAB, SUBJECT_B); + testAuthenticateAsync(tokenBA, SUBJECT_A); + testAuthenticateAsync(tokenBB, SUBJECT_B); + } + + private AuthenticationState newAuthState(String token, String expectedSubject) throws Exception { // Must pass the token to the newAuthState for legacy reasons. AuthenticationState authState = authProvider.newAuthState( From 14f4b093cf40213b5a8f79bca7a5d6936bc981e0 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Wed, 19 Apr 2023 12:31:25 +0800 Subject: [PATCH 122/280] [fix][broker] Revert : Forbid uploading BYTES schema with admin API (#18995) (#20123) --- .../admin/impl/SchemasResourceBase.java | 4 ---- .../broker/admin/AdminApiSchemaTest.java | 11 ---------- .../org/apache/pulsar/schema/SchemaTest.java | 21 +++++++------------ 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index 91a8140fde12b..0bab772044a6d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -114,10 +114,6 @@ public CompletableFuture deleteSchemaAsync(boolean authoritative, } public CompletableFuture postSchemaAsync(PostSchemaPayload payload, boolean authoritative) { - if (SchemaType.BYTES.name().equals(payload.getType())) { - return CompletableFuture.failedFuture(new RestException(Response.Status.NOT_ACCEPTABLE, - "Do not upload a BYTES schema, because it's the default schema type")); - } return validateOwnershipAndOperationAsync(authoritative, TopicOperation.PRODUCE) .thenCompose(__ -> getSchemaCompatibilityStrategyAsyncWithoutAuth()) .thenCompose(schemaCompatibilityStrategy -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java index e8e582f80b0dc..b37114f180216 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java @@ -185,17 +185,6 @@ private void testSchemaInfoApi(Schema schema, } - @Test - public void testCreateBytesSchema() { - // forbid admin api creating BYTES schema to be consistent with client side - try { - testSchemaInfoApi(Schema.BYTES, "schematest/test/test-BYTES"); - fail("should fail"); - } catch (Exception e) { - assertTrue(e.getMessage().contains("Do not upload a BYTES schema")); - } - } - @Test(dataProvider = "version") public void testPostSchemaCompatibilityStrategy(ApiVersion version) throws PulsarAdminException { String namespace = format("%s%s%s", "schematest", (ApiVersion.V1.equals(version) ? "/" + cluster + "/" : "/"), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 0ec72b2ef470a..7ba4529cdbdf4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -513,17 +513,13 @@ private void testUseAutoConsumeWithSchemalessTopic(SchemaType schema) throws Exc admin.topics().createPartitionedTopic(topic, 2); // set schema - if (!schema.equals(SchemaType.BYTES)) { - // don't upload bytes schema with admin API - // because null schema means the BYTES schema - SchemaInfo schemaInfo = SchemaInfoImpl - .builder() - .schema(new byte[0]) - .name("dummySchema") - .type(schema) - .build(); - admin.schemas().createSchema(topic, schemaInfo); - } + SchemaInfo schemaInfo = SchemaInfoImpl + .builder() + .schema(new byte[0]) + .name("dummySchema") + .type(schema) + .build(); + admin.schemas().createSchema(topic, schemaInfo); Producer producer = pulsarClient .newProducer() @@ -548,8 +544,7 @@ private void testUseAutoConsumeWithSchemalessTopic(SchemaType schema) throws Exc Message message2 = consumer2.receive(); if (schema == SchemaType.BYTES) { assertEquals(schema, message.getReaderSchema().get().getSchemaInfo().getType()); - // default schema of AUTO_CONSUME is BYTES - assertTrue(message2.getReaderSchema().get().toString().contains("BYTES")); + assertEquals(schema, message2.getReaderSchema().get().getSchemaInfo().getType()); } else if (schema == SchemaType.NONE) { // schema NONE is always reported as BYTES assertEquals(SchemaType.BYTES, message.getReaderSchema().get().getSchemaInfo().getType()); From 866d405726bb6527db954201f2260620181747dc Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 19 Apr 2023 14:45:29 +0800 Subject: [PATCH 123/280] [fix][sec] spring.version=5.3.27 to fix CVE-2023-20863 (#20124) Signed-off-by: tison --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b5ab09d4581a..22589a1719812 100644 --- a/pom.xml +++ b/pom.xml @@ -232,7 +232,7 @@ flexible messaging model and an intuitive client API. 1.8.20 1.0 9.1.6 - 5.3.26 + 5.3.27 4.5.13 4.4.15 0.5.11 From 9b723022436cc1a150af765103e8d343679f92ce Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 19 Apr 2023 16:44:12 +0800 Subject: [PATCH 124/280] [improve][broker] Make timer execute immediately after load index (#20126) --- .../delayed/bucket/BucketDelayedDeliveryTracker.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 6678c6df254b0..ad2fc6fae4c87 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -541,7 +541,7 @@ public long getBufferMemoryUsage() { @Override public synchronized NavigableSet getScheduledMessages(int maxMessages) { - if (!checkPendingOpDone()) { + if (!checkPendingLoadDone()) { if (log.isDebugEnabled()) { log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", dispatcher.getName()); @@ -628,11 +628,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa if (timeout != null) { timeout.cancel(); } - timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS); + timeout = timer.newTimeout(this, 0, TimeUnit.MILLISECONDS); } }); - if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) { + if (!checkPendingLoadDone() || loadFuture.isCompletedExceptionally()) { break; } } @@ -651,7 +651,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa return positions; } - private synchronized boolean checkPendingOpDone() { + private synchronized boolean checkPendingLoadDone() { if (pendingLoad == null || pendingLoad.isDone()) { pendingLoad = null; return true; From fc17c1d98a3c1edd975c131d174a9ef69887d9cd Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Wed, 19 Apr 2023 23:40:28 +0800 Subject: [PATCH 125/280] [fix][broker] Fix tenant admin authorization bug. (#20068) Co-authored-by: druidliu --- .../authorization/AuthorizationService.java | 27 +++++-------------- .../pulsar/broker/auth/AuthorizationTest.java | 22 ++++++++------- .../AuthorizationProducerConsumerTest.java | 16 +++++++++++ .../proxy/ProxyAuthorizationTest.java | 14 +++++++--- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 0c61219b57a50..a9225f5e48fa9 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -28,6 +28,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; @@ -182,13 +183,7 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { - if (isSuperUser) { - return CompletableFuture.completedFuture(true); - } else { - return provider.canProduceAsync(topicName, role, authenticationData); - } - }); + return provider.allowTopicOperationAsync(topicName, role, TopicOperation.PRODUCE, authenticationData); } /** @@ -207,13 +202,9 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { - if (isSuperUser) { - return CompletableFuture.completedFuture(true); - } else { - return provider.canConsumeAsync(topicName, role, authenticationData, subscription); - } - }); + + return provider.allowTopicOperationAsync(topicName, role, TopicOperation.CONSUME, + new AuthenticationDataSubscription(authenticationData, subscription)); } public boolean canProduce(TopicName topicName, String role, AuthenticationDataSource authenticationData) @@ -289,13 +280,7 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { - if (isSuperUser) { - return CompletableFuture.completedFuture(true); - } else { - return provider.canLookupAsync(topicName, role, authenticationData); - } - }); + return provider.allowTopicOperationAsync(topicName, role, TopicOperation.LOOKUP, authenticationData); } public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 58cf4ee418ea4..4fce7c50e1c44 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -79,10 +79,13 @@ public void cleanup() throws Exception { public void simple() throws Exception { AuthorizationService auth = pulsar.getBrokerService().getAuthorizationService(); - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - + try { + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + fail("Should throw exception when tenant not exist"); + } catch (Exception ignored) {} admin.clusters().createCluster("c1", ClusterData.builder().build()); - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); + String tenantAdmin = "role1"; + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -215,21 +218,22 @@ public void simple() throws Exception { SubscriptionAuthMode.Prefix); waitForChange(); - assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null)); + assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null)); - try { - assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "sub1")); - fail(); - } catch (Exception ignored) {} + // tenant admin can consume all topics, even if SubscriptionAuthMode.Prefix mode + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); try { assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "sub2")); fail(); } catch (Exception ignored) {} - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "role1-sub1")); + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "role1-sub1")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "role2-sub2")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "pulsar.super_user", null, "role3-sub1")); + // tenant admin can produce all topics + assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); + admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0ce3b7df07d1f..dd351286d2e5e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -1035,6 +1035,22 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol return CompletableFuture.completedFuture(grantRoles.contains(role)); } + @Override + public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, + TopicOperation operation, + AuthenticationDataSource authData) { + switch (operation) { + + case PRODUCE: + return canProduceAsync(topic, role, authData); + case CONSUME: + return canConsumeAsync(topic, role, authData, authData.getSubscription()); + case LOOKUP: + return canLookupAsync(topic, role, authData); + } + return super.allowTopicOperationAsync(topic, role, operation, authData); + } + @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authData) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java index a3b26a4a9d122..29327d3d4ebe1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.util.EnumSet; import java.util.Optional; @@ -83,10 +84,13 @@ protected void cleanup() throws Exception { public void test() throws Exception { AuthorizationService auth = service.getAuthorizationService(); - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - + try { + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + fail("Should throw exception when tenant not exist"); + } catch (Exception ignored) {} admin.clusters().createCluster(configClusterName, ClusterData.builder().build()); - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); + String tenantAdmin = "role1"; + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -117,6 +121,10 @@ public void test() throws Exception { assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null)); + // tenant admin can produce/consume all topics, even if SubscriptionAuthMode.Prefix mode + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); + assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); + admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); From c9c99aae0d0ca5f8ac20a326d3d76bbf8602c79c Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 19 Apr 2023 10:44:49 -0500 Subject: [PATCH 126/280] [improve] AuthenticationProviderOpenID k8s error logs (#20135) ### Motivation The `AuthenticationProviderOpenID` error logs from the Kubernetes client are not very helpful in certain cases because we only get the error's message and not the error's response body. See https://github.com/kubernetes-client/java/issues/2066 for details on the solution. Here is an example of a problematic error: ``` org.apache.pulsar.broker.authentication.AuthenticationProviderList - Authentication failed for auth provider class org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID: javax.naming.AuthenticationException: Error retrieving OpenID Provider Metadata from Kubernetes API server: at org.apache.pulsar.broker.authentication.oidc.OpenIDProviderMetadataCache$1.onFailure(OpenIDProviderMetadataCache.java:174) ~[org.apache.pulsar-pulsar-broker-auth-oidc-3.0.0.jar:3.0.0] at io.kubernetes.client.openapi.ApiClient$1.onResponse(ApiClient.java:927) ~[io.kubernetes-client-java-api-17.0.2.jar:?] at okhttp3.internal.connection.RealCall$AsyncCall.run(RealCall.kt:519) ~[com.squareup.okhttp3-okhttp-4.9.3.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:833) ~[?:?] ``` When I enable debug logging out of the API Client, I can see: ``` INFO: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"forbidden: User \"system:serviceaccount:michael-test:superuser\" cannot get path \"/.well-known/openid-configuration/\"","reason":"Forbidden","details":{},"code":403} Apr 19, 2023 2:50:25 AM okhttp3.internal.platform.Platform log INFO: <-- END HTTP (246-byte body) 2023-04-19T02:50:25,832+0000 [pulsar-web-40-1] DEBUG ``` (Note: the solution to this problem is to update the `system:service-account-issuer-discovery` `ClusterRole` to include endpoints with trailing slashes. I created https://github.com/kubernetes/kubernetes/issues/117455 to help solve the permission problem in kubernetes.) ### Modifications * Use both the message and the response body when converting a Kubernetes client error into a Pulsar Authentication error. ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: no need for a forked PR --- .../apache/pulsar/broker/authentication/oidc/JwksCache.java | 5 +++-- .../authentication/oidc/OpenIDProviderMetadataCache.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java index 12ea7ec6b906d..b5e038342c2e9 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java @@ -130,9 +130,10 @@ private CompletableFuture> getJwksFromKubernetesApiServer() { @Override public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066. future.completeExceptionally( - new AuthenticationException("Failed to retrieve public key from Kubernetes API server: " - + e.getMessage())); + new AuthenticationException("Failed to retrieve public key from Kubernetes API server. " + + "Message: " + e.getMessage() + " Response body: " + e.getResponseBody())); } @Override diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java index 33d11f35a349a..111399adbd72b 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java @@ -165,9 +165,10 @@ private CompletableFuture loadOpenIDProviderMetadataForK @Override public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066. future.completeExceptionally(new AuthenticationException( - "Error retrieving OpenID Provider Metadata from Kubernetes API server: " - + e.getMessage())); + "Error retrieving OpenID Provider Metadata from Kubernetes API server. Message: " + + e.getMessage() + " Response body: " + e.getResponseBody())); } @Override From 4f2b8e21ff5175f09e9e20ca3ad341db59822c86 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 19 Apr 2023 17:42:23 -0500 Subject: [PATCH 127/280] [fix][fn] Supply download auth params when provided for k8s runtime (#20144) PIP: #19849 ### Motivation The new `KubernetesServiceAccountTokenAuthProvider` introduced by #19888 does not configure the authentication for download because there is an unnecessary check that the `getFunctionAuthenticationSpec` is not `null`. Given that we do not use this spec in creating the auth, it is unnecessary to use here. Further, when we construct the function's authentication, we do not have this null check. See: https://github.com/apache/pulsar/blob/ec102fb024a6ea2b195826778300f20e330dff06/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java#L411-L417 ### Modifications * Update the `KubernetesRuntime` to add authentication params to the download command when provided. ### Verifying this change A new test is added. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping for this trivial PR --- .../runtime/kubernetes/KubernetesRuntime.java | 3 +-- .../kubernetes/KubernetesRuntimeTest.java | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 0a5e1268c89a5..aa474aad801dc 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -878,8 +878,7 @@ private List getDownloadCommand(String tenant, String namespace, String // add auth plugin and parameters if necessary if (authenticationEnabled && authConfig != null) { if (isNotBlank(authConfig.getClientAuthenticationPlugin()) - && isNotBlank(authConfig.getClientAuthenticationParameters()) - && instanceConfig.getFunctionAuthenticationSpec() != null) { + && isNotBlank(authConfig.getClientAuthenticationParameters())) { cmd.addAll(Arrays.asList( "--auth-plugin", authConfig.getClientAuthenticationPlugin(), diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 1e8194eed443a..3facd37fc9244 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -861,6 +861,32 @@ public void testCustomKubernetesDownloadCommandsWithAuth() throws Exception { assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand); } + @Test + public void testCustomKubernetesDownloadCommandsWithAuthWithoutAuthSpec() throws Exception { + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false); + config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false)); + + factory = createKubernetesRuntimeFactory(null, + 10, 1.0, 1.0, Optional.empty(), null, wconfig -> { + wconfig.setAuthenticationEnabled(true); + }, AuthenticationConfig.builder() + .clientAuthenticationPlugin("com.MyAuth") + .clientAuthenticationParameters("{\"authParam1\": \"authParamValue1\"}") + .build()); + + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); + V1StatefulSet spec = container.createStatefulSet(); + String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" + + " functions download " + + "--tenant " + TEST_TENANT + + " --namespace " + TEST_NAMESPACE + + " --name " + TEST_NAME + + " --destination-file " + pulsarRootDir + "/" + userJarFile; + String containerCommand = spec.getSpec().getTemplate().getSpec().getContainers().get(0).getCommand().get(2); + assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand); + } + InstanceConfig createGolangInstanceConfig() { InstanceConfig config = new InstanceConfig(); From 6dc0b0ea38ab9dc410a24b19fd7567b9e013837d Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 20 Apr 2023 08:21:08 +0800 Subject: [PATCH 128/280] [improve][broker] Optimize delayed metadata index bitmap (#20136) --- .../bucket/BucketDelayedDeliveryTracker.java | 3 ++ .../delayed/bucket/ImmutableBucket.java | 53 +++++++++++-------- .../broker/delayed/bucket/MutableBucket.java | 11 ++-- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index ad2fc6fae4c87..5d1dc2e27033a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -485,6 +485,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List> asyncLoadNextBucketSnapshotEntry(b this.setLastSegmentEntryId(metadataList.size()); this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); List firstScheduleTimestamps = metadataList.stream().map( - SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); + SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); this.setFirstScheduleTimestamps(firstScheduleTimestamps); return nextSnapshotEntryIndex + 1; @@ -139,25 +138,37 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b }); } + /** + * Recover delayed index bit map and message numbers. + * @throws InvalidRoaringFormat invalid bitmap serialization format + */ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, - List segmentMetadata) { - this.delayedIndexBitMap.clear(); - MutableLong numberMessages = new MutableLong(0); - for (int i = startSnapshotIndex; i < segmentMetadata.size(); i++) { - Map bitByteStringMap = segmentMetadata.get(i).getDelayedIndexBitMapMap(); - bitByteStringMap.forEach((leaderId, bitSetString) -> { - boolean exist = this.delayedIndexBitMap.containsKey(leaderId); - RoaringBitmap bitSet = - new ImmutableRoaringBitmap(bitSetString.asReadOnlyByteBuffer()).toRoaringBitmap(); - numberMessages.add(bitSet.getCardinality()); - if (!exist) { - this.delayedIndexBitMap.put(leaderId, bitSet); - } else { - this.delayedIndexBitMap.get(leaderId).or(bitSet); + List segmentMetaList) { + delayedIndexBitMap.clear(); // cleanup dirty bm + final var numberMessages = new MutableLong(0); + for (int i = startSnapshotIndex; i < segmentMetaList.size(); i++) { + for (final var entry : segmentMetaList.get(i).getDelayedIndexBitMapMap().entrySet()) { + final var ledgerId = entry.getKey(); + final var bs = entry.getValue(); + final var sbm = new RoaringBitmap(); + try { + sbm.deserialize(bs.asReadOnlyByteBuffer()); + } catch (IOException e) { + throw new InvalidRoaringFormat(e.getMessage()); } - }); + numberMessages.add(sbm.getCardinality()); + delayedIndexBitMap.compute(ledgerId, (lId, bm) -> { + if (bm == null) { + return sbm; + } + bm.or(sbm); + return bm; + }); + } } - this.setNumberBucketDelayedMessages(numberMessages.getValue()); + // optimize bm + delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); + setNumberBucketDelayedMessages(numberMessages.getValue()); } CompletableFuture> getRemainSnapshotSegment() { @@ -193,7 +204,7 @@ CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete); } else { log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey); + dispatcherName, bucketId, bucketKey); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete, System.currentTimeMillis() - deleteStartTime); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index e49ebe9606e01..f404d5d02c15a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -116,10 +116,13 @@ Pair createImmutableBucketAndAsyncPersistent( Iterator> iterator = bitMap.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - byte[] array = new byte[entry.getValue().serializedSizeInBytes()]; - entry.getValue().serialize(ByteBuffer.wrap(array)); - segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array)); + final var entry = iterator.next(); + final var lId = entry.getKey(); + final var bm = entry.getValue(); + bm.runOptimize(); + final var array = new byte[bm.serializedSizeInBytes()]; + bm.serialize(ByteBuffer.wrap(array)); + segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array)); iterator.remove(); } From d3fa998aa7c0a7a9452079ef2ff05bccf6b273cf Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 20 Apr 2023 09:26:30 +0800 Subject: [PATCH 129/280] [improve][broker] Cache LedgerHandle in BookkeeperBucketSnapshotStorage (#20117) --- .../BookkeeperBucketSnapshotStorage.java | 52 +++++++++---------- .../BookkeeperBucketSnapshotStorageTest.java | 43 +++++++++++++++ 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 9c30ccf1c0b7e..18a4c322f7b27 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import javax.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; @@ -48,6 +49,8 @@ public class BookkeeperBucketSnapshotStorage implements BucketSnapshotStorage { private final ServiceConfiguration config; private BookKeeper bookKeeper; + private final Map> ledgerHandleFutureCache = new ConcurrentHashMap<>(); + public BookkeeperBucketSnapshotStorage(PulsarService pulsar) { this.pulsar = pulsar; this.config = pulsar.getConfig(); @@ -66,45 +69,30 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture snapshotFuture = - getLedgerEntry(ledgerHandle, 0, 0) - .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); - - snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return snapshotFuture; - }); + return getLedgerHandle(bucketId).thenCompose(ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0) + .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture> parseFuture = - getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) - .thenApply(this::parseSnapshotSegmentEntries); - - parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return parseFuture; - }); + return getLedgerHandle(bucketId).thenCompose( + ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) + .thenApply(this::parseSnapshotSegmentEntries)); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture lengthFuture = - CompletableFuture.completedFuture(ledgerHandle.getLength()); - - lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return lengthFuture; - }); + return getLedgerHandle(bucketId).thenCompose( + ledgerHandle -> CompletableFuture.completedFuture(ledgerHandle.getLength())); } @Override public CompletableFuture deleteBucketSnapshot(long bucketId) { + CompletableFuture ledgerHandleFuture = ledgerHandleFutureCache.remove(bucketId); + if (ledgerHandleFuture != null) { + ledgerHandleFuture.whenComplete((lh, ex) -> closeLedger(lh)); + } return deleteLedger(bucketId); } @@ -178,6 +166,18 @@ private CompletableFuture createLedger(String bucketKey, String to return future; } + private CompletableFuture getLedgerHandle(Long ledgerId) { + CompletableFuture ledgerHandleCompletableFuture = + ledgerHandleFutureCache.computeIfAbsent(ledgerId, k -> openLedger(ledgerId)); + // remove future of completed exceptionally + ledgerHandleCompletableFuture.whenComplete((__, ex) -> { + if (ex != null) { + ledgerHandleFutureCache.remove(ledgerId, ledgerHandleCompletableFuture); + } + }); + return ledgerHandleCompletableFuture; + } + private CompletableFuture openLedger(Long ledgerId) { final CompletableFuture future = new CompletableFuture<>(); bookKeeper.asyncOpenLedger( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index a628b58e10d32..7cb6b8d5865bb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -204,4 +205,46 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted Assert.assertTrue(bucketSnapshotLength > 0L); } + @Test + public void testConcurrencyGet() throws ExecutionException, InterruptedException { + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + .setMinScheduleTimestamp(System.currentTimeMillis()) + .setMaxScheduleTimestamp(System.currentTimeMillis()) + .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); + + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + .addMetadataList(segmentMetadata) + .build(); + List bucketSnapshotSegments = new ArrayList<>(); + + long timeMillis = System.currentTimeMillis(); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis).build(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + bucketSnapshotSegments.add(snapshotSegment); + bucketSnapshotSegments.add(snapshotSegment); + + CompletableFuture future = + bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, + bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); + Long bucketId = future.get(); + Assert.assertNotNull(bucketId); + + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + CompletableFuture future0 = CompletableFuture.runAsync(() -> { + List list = + bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); + Assert.assertTrue(list.size() > 0); + }); + futures.add(future0); + } + + FutureUtil.waitForAll(futures).join(); + } + } From dc5e497b2a42910146690c6b4e922e0a80bf1587 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 20 Apr 2023 00:17:02 -0500 Subject: [PATCH 130/280] [fix][broker] Producer/Consumer should call allowTopicOperationAsync (#20142) Fixes: https://github.com/apache/pulsar/issues/20066 ### Motivation In https://github.com/apache/pulsar/pull/20068 we changed the way that the `AuthorizationService` is implemented. I think we should change the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method. Given that our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, this change helps us move in the right direction. ### Modifications * Update `Producer` and `Consumer` in broker to call the `AuthorizationService#allowTopicOperationAsync` method. ### Verifying this change This change is trivial. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping PR as I ran tests locally. --- .../java/org/apache/pulsar/broker/service/Consumer.java | 8 ++++++-- .../java/org/apache/pulsar/broker/service/Producer.java | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 1ee3f513ef288..a3f9da41e6b35 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -43,6 +43,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageId; @@ -56,6 +57,7 @@ import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.SchemaType; @@ -901,8 +903,10 @@ public String toString() { public CompletableFuture checkPermissionsAsync() { TopicName topicName = TopicName.get(subscription.getTopicName()); if (cnx.getBrokerService().getAuthorizationService() != null) { - return cnx.getBrokerService().getAuthorizationService().canConsumeAsync(topicName, appId, - cnx.getAuthenticationData(), subscription.getName()) + AuthenticationDataSubscription authData = + new AuthenticationDataSubscription(cnx.getAuthenticationData(), subscription.getName()); + return cnx.getBrokerService().getAuthorizationService() + .allowTopicOperationAsync(topicName, TopicOperation.CONSUME, appId, authData) .handle((ok, e) -> { if (e != null) { log.warn("[{}] Get unexpected error while authorizing [{}] {}", appId, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index 5b62e3261e64f..53b79f06e8e24 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -51,6 +51,7 @@ import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.stats.NonPersistentPublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.protocol.Commands; @@ -781,7 +782,7 @@ public CompletableFuture checkPermissionsAsync() { TopicName topicName = TopicName.get(topic.getName()); if (cnx.getBrokerService().getAuthorizationService() != null) { return cnx.getBrokerService().getAuthorizationService() - .canProduceAsync(topicName, appId, cnx.getAuthenticationData()) + .allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, appId, cnx.getAuthenticationData()) .handle((ok, ex) -> { if (ex != null) { log.warn("[{}] Get unexpected error while autorizing [{}] {}", appId, topic.getName(), From 00dc7a0691b496065dba6af0c6de64af4973886e Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 20 Apr 2023 00:17:22 -0500 Subject: [PATCH 131/280] [revert] "[fix][broker] Fix tenant admin authorization bug. (#20068)" (#20143) This reverts commit fc17c1d98a3c1edd975c131d174a9ef69887d9cd. ### Motivation In https://github.com/apache/pulsar/pull/20068 we changed the way that the `AuthorizationService` is implemented. I think this approach could have unintended consequences. Instead, I think we should change the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method. I propose an update to the `Consumer` and `Producer` logic here #20142. Given that our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, I think we should not change their implementations. ### Modifications * Revert https://github.com/apache/pulsar/pull/20068 ### Verifying this change This change is trivial. It removes certain test changes that were only made to make the previous PR work. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping PR as I ran tests locally. --- .../authorization/AuthorizationService.java | 27 ++++++++++++++----- .../pulsar/broker/auth/AuthorizationTest.java | 22 +++++++-------- .../AuthorizationProducerConsumerTest.java | 16 ----------- .../proxy/ProxyAuthorizationTest.java | 14 +++------- 4 files changed, 33 insertions(+), 46 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index a9225f5e48fa9..0c61219b57a50 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -28,7 +28,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; -import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; @@ -183,7 +182,13 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.allowTopicOperationAsync(topicName, role, TopicOperation.PRODUCE, authenticationData); + return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { + if (isSuperUser) { + return CompletableFuture.completedFuture(true); + } else { + return provider.canProduceAsync(topicName, role, authenticationData); + } + }); } /** @@ -202,9 +207,13 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - - return provider.allowTopicOperationAsync(topicName, role, TopicOperation.CONSUME, - new AuthenticationDataSubscription(authenticationData, subscription)); + return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { + if (isSuperUser) { + return CompletableFuture.completedFuture(true); + } else { + return provider.canConsumeAsync(topicName, role, authenticationData, subscription); + } + }); } public boolean canProduce(TopicName topicName, String role, AuthenticationDataSource authenticationData) @@ -280,7 +289,13 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.allowTopicOperationAsync(topicName, role, TopicOperation.LOOKUP, authenticationData); + return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { + if (isSuperUser) { + return CompletableFuture.completedFuture(true); + } else { + return provider.canLookupAsync(topicName, role, authenticationData); + } + }); } public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 4fce7c50e1c44..58cf4ee418ea4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -79,13 +79,10 @@ public void cleanup() throws Exception { public void simple() throws Exception { AuthorizationService auth = pulsar.getBrokerService().getAuthorizationService(); - try { - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - fail("Should throw exception when tenant not exist"); - } catch (Exception ignored) {} + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + admin.clusters().createCluster("c1", ClusterData.builder().build()); - String tenantAdmin = "role1"; - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -218,22 +215,21 @@ public void simple() throws Exception { SubscriptionAuthMode.Prefix); waitForChange(); - assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); + assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null)); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null)); - // tenant admin can consume all topics, even if SubscriptionAuthMode.Prefix mode - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); + try { + assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "sub1")); + fail(); + } catch (Exception ignored) {} try { assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "sub2")); fail(); } catch (Exception ignored) {} - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "role1-sub1")); + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "role1-sub1")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "role2-sub2")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "pulsar.super_user", null, "role3-sub1")); - // tenant admin can produce all topics - assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); - admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index dd351286d2e5e..0ce3b7df07d1f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -1035,22 +1035,6 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol return CompletableFuture.completedFuture(grantRoles.contains(role)); } - @Override - public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, - TopicOperation operation, - AuthenticationDataSource authData) { - switch (operation) { - - case PRODUCE: - return canProduceAsync(topic, role, authData); - case CONSUME: - return canConsumeAsync(topic, role, authData, authData.getSubscription()); - case LOOKUP: - return canLookupAsync(topic, role, authData); - } - return super.allowTopicOperationAsync(topic, role, operation, authData); - } - @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authData) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java index 29327d3d4ebe1..a3b26a4a9d122 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java @@ -25,7 +25,6 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.util.EnumSet; import java.util.Optional; @@ -84,13 +83,10 @@ protected void cleanup() throws Exception { public void test() throws Exception { AuthorizationService auth = service.getAuthorizationService(); - try { - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - fail("Should throw exception when tenant not exist"); - } catch (Exception ignored) {} + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + admin.clusters().createCluster(configClusterName, ClusterData.builder().build()); - String tenantAdmin = "role1"; - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -121,10 +117,6 @@ public void test() throws Exception { assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null)); - // tenant admin can produce/consume all topics, even if SubscriptionAuthMode.Prefix mode - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); - assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); - admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); From 575cf2331dd3ac0048923487c6c7904eda4301e6 Mon Sep 17 00:00:00 2001 From: ran Date: Thu, 20 Apr 2023 16:18:06 +0800 Subject: [PATCH 132/280] [fix][broker] Fix entry filter feature for the non-persistent topic (#20141) --- .../nonpersistent/NonPersistentTopic.java | 5 ++- .../service/plugin/FilterEntryTest.java | 15 +++++-- .../broker/stats/SubscriptionStatsTest.java | 42 ++++++++++--------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 317b8df6b9a82..33258b06726b5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -26,7 +26,7 @@ import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -199,7 +199,8 @@ public void publishMessage(ByteBuf data, PublishContext callback) { // entry internally retains data so, duplicateBuffer should be release here duplicateBuffer.release(); if (subscription.getDispatcher() != null) { - subscription.getDispatcher().sendMessages(Collections.singletonList(entry)); + // Dispatcher needs to call the set method to support entry filter feature. + subscription.getDispatcher().sendMessages(Arrays.asList(entry)); } else { // it happens when subscription is created but dispatcher is not created as consumer is not added // yet diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java index 4b9d91fbde219..b868858646c50 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.EntryFilterSupport; +import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -286,10 +287,16 @@ public void testFilter() throws Exception { } + @DataProvider(name = "topicProvider") + public Object[][] topicProvider() { + return new Object[][]{ + {"persistent://prop/ns-abc/topic" + UUID.randomUUID()}, + {"non-persistent://prop/ns-abc/topic" + UUID.randomUUID()}, + }; + } - @Test - public void testFilteredMsgCount() throws Throwable { - String topic = "persistent://prop/ns-abc/topic" + UUID.randomUUID(); + @Test(dataProvider = "topicProvider") + public void testFilteredMsgCount(String topic) throws Throwable { String subName = "sub"; try (Producer producer = pulsarClient.newProducer(Schema.STRING) @@ -298,7 +305,7 @@ public void testFilteredMsgCount() throws Throwable { .subscriptionName(subName).subscribe()) { // mock entry filters - PersistentSubscription subscription = (PersistentSubscription) pulsar.getBrokerService() + Subscription subscription = pulsar.getBrokerService() .getTopicReference(topic).get().getSubscription(subName); Dispatcher dispatcher = subscription.getDispatcher(); Field field = EntryFilterSupport.class.getDeclaredField("entryFilters"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java index bf9c1d540bf87..d5e0066a86f15 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java @@ -211,21 +211,22 @@ public void testSubscriptionStats(final String topic, final String subName, bool hasFilterField.set(dispatcher, true); } - for (int i = 0; i < 100; i++) { - producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send(); - } - for (int i = 0; i < 100; i++) { + int rejectedCount = 100; + int acceptCount = 100; + int scheduleCount = 100; + for (int i = 0; i < rejectedCount; i++) { producer.newMessage().property("REJECT", " ").value(UUID.randomUUID().toString()).send(); } - for (int i = 0; i < 100; i++) { + for (int i = 0; i < acceptCount; i++) { + producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send(); + } + for (int i = 0; i < scheduleCount; i++) { producer.newMessage().property("RESCHEDULE", " ").value(UUID.randomUUID().toString()).send(); } - for (;;) { - Message message = consumer.receive(10, TimeUnit.SECONDS); - if (message == null) { - break; - } + for (int i = 0; i < acceptCount; i++) { + Message message = consumer.receive(1, TimeUnit.SECONDS); + Assert.assertNotNull(message); consumer.acknowledge(message); } @@ -263,12 +264,12 @@ public void testSubscriptionStats(final String topic, final String subName, bool .mapToDouble(m-> m.value).sum(); if (setFilter) { - Assert.assertEquals(filterAccepted, 100); - if (isPersistent) { - Assert.assertEquals(filterRejected, 100); - // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount - Assert.assertEquals(throughFilter, filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); - } + Assert.assertEquals(filterAccepted, acceptCount); + Assert.assertEquals(filterRejected, rejectedCount); + // Only works on the test, if there are some markers, + // the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount + Assert.assertEquals(throughFilter, + filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); } else { Assert.assertEquals(throughFilter, 0D); Assert.assertEquals(filterAccepted, 0D); @@ -282,19 +283,20 @@ public void testSubscriptionStats(final String topic, final String subName, bool Assert.assertEquals(rescheduledMetrics.size(), 0); } - testSubscriptionStatsAdminApi(topic, subName, setFilter); + testSubscriptionStatsAdminApi(topic, subName, setFilter, acceptCount, rejectedCount); } - private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter) throws Exception { + private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter, + int acceptCount, int rejectedCount) throws Exception { boolean persistent = TopicName.get(topic).isPersistent(); TopicStats topicStats = admin.topics().getStats(topic); SubscriptionStats stats = topicStats.getSubscriptions().get(subName); Assert.assertNotNull(stats); if (setFilter) { - Assert.assertEquals(stats.getFilterAcceptedMsgCount(), 100); + Assert.assertEquals(stats.getFilterAcceptedMsgCount(), acceptCount); if (persistent) { - Assert.assertEquals(stats.getFilterRejectedMsgCount(), 100); + Assert.assertEquals(stats.getFilterRejectedMsgCount(), rejectedCount); // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount Assert.assertEquals(stats.getFilterProcessedMsgCount(), stats.getFilterAcceptedMsgCount() + stats.getFilterRejectedMsgCount() From 99a68e40c9dcc0771ddfd73321cc967be6433801 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 20 Apr 2023 17:49:27 +0800 Subject: [PATCH 133/280] [fix][client] Move MessageIdAdv to the pulsar-common module (#20139) --- .../pulsar/client/api/TopicMessageId.java | 82 +------------------ .../PulsarClientImplementationBinding.java | 3 + .../pulsar/client/impl/ConsumerImpl.java | 2 +- .../pulsar/client/impl/MessageIdImpl.java | 4 +- ...PulsarClientImplementationBindingImpl.java | 18 +++- .../client/impl/TopicMessageIdImpl.java | 77 +++++++++++++++-- .../pulsar/client/impl/TopicMessageImpl.java | 3 +- .../client/impl/MessageIdCompareToTest.java | 5 -- .../client/impl/TopicMessageIdImplTest.java | 12 +-- .../pulsar/client/api/MessageIdAdv.java | 0 .../pulsar/client/api/package-info.java | 22 +++++ .../kafka/connect/KafkaConnectSinkTest.java | 4 +- .../pulsar/websocket/ConsumerHandler.java | 6 +- 13 files changed, 133 insertions(+), 105 deletions(-) rename {pulsar-client-api => pulsar-common}/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java (100%) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java index b70267bb0fb8b..4d02a7f4096d6 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.client.api; -import java.util.BitSet; +import org.apache.pulsar.client.internal.DefaultImplementation; /** * The MessageId used for a consumer that subscribes multiple topics or partitioned topics. @@ -45,84 +45,6 @@ static TopicMessageId create(String topic, MessageId messageId) { if (messageId instanceof TopicMessageId) { return (TopicMessageId) messageId; } - return new Impl(topic, messageId); - } - - /** - * The simplest implementation of a TopicMessageId interface. - */ - class Impl implements MessageIdAdv, TopicMessageId { - private final String topic; - private final MessageIdAdv messageId; - - public Impl(String topic, MessageId messageId) { - this.topic = topic; - this.messageId = (MessageIdAdv) messageId; - } - - @Override - public byte[] toByteArray() { - return messageId.toByteArray(); - } - - @Override - public String getOwnerTopic() { - return topic; - } - - @Override - public long getLedgerId() { - return messageId.getLedgerId(); - } - - @Override - public long getEntryId() { - return messageId.getEntryId(); - } - - @Override - public int getPartitionIndex() { - return messageId.getPartitionIndex(); - } - - @Override - public int getBatchIndex() { - return messageId.getBatchIndex(); - } - - @Override - public int getBatchSize() { - return messageId.getBatchSize(); - } - - @Override - public BitSet getAckSet() { - return messageId.getAckSet(); - } - - @Override - public MessageIdAdv getFirstChunkMessageId() { - return messageId.getFirstChunkMessageId(); - } - - @Override - public int compareTo(MessageId o) { - return messageId.compareTo(o); - } - - @Override - public boolean equals(Object obj) { - return messageId.equals(obj); - } - - @Override - public int hashCode() { - return messageId.hashCode(); - } - - @Override - public String toString() { - return messageId.toString(); - } + return DefaultImplementation.getDefaultImplementation().newTopicMessageId(topic, messageId); } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java index 875a793023523..8fd05bff265f1 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java @@ -37,6 +37,7 @@ import org.apache.pulsar.client.api.MessagePayloadFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; @@ -252,4 +253,6 @@ static byte[] getBytes(ByteBuffer byteBuffer) { SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, long timestamp, Map propertiesValue); + + TopicMessageId newTopicMessageId(String topic, MessageId messageId); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index cc01609319698..199e8a9ae71b4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2345,7 +2345,7 @@ public CompletableFuture getLastMessageIdAsync() { @Override public CompletableFuture> getLastMessageIdsAsync() { return getLastMessageIdAsync() - .thenApply(msgId -> Collections.singletonList(TopicMessageId.create(topic, msgId))); + .thenApply(msgId -> Collections.singletonList(new TopicMessageIdImpl(topic, (MessageIdAdv) msgId))); } public CompletableFuture internalGetLastMessageIdAsync() { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java index 83ee762578390..8cffba44dc5ca 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java @@ -128,7 +128,7 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName) throw new IOException(e); } - MessageId messageId; + MessageIdAdv messageId; if (idData.hasBatchIndex()) { if (idData.hasBatchSize()) { messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(), @@ -143,7 +143,7 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName) } if (idData.getPartition() > -1 && topicName != null) { messageId = new TopicMessageIdImpl( - topicName.getPartition(idData.getPartition()).toString(), topicName.toString(), messageId); + topicName.getPartition(idData.getPartition()).toString(), messageId); } return messageId; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java index 1b069c5172dd7..346eb20ef4cc5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.client.impl; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -35,9 +34,11 @@ import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessagePayloadFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; @@ -387,4 +388,19 @@ public SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, Map propertiesValue) { return new SchemaInfoImpl(name, schema, type, timestamp, propertiesValue); } + + @Override + public TopicMessageId newTopicMessageId(String topic, MessageId messageId) { + final MessageIdAdv messageIdAdv; + if (messageId instanceof MessageIdAdv) { + messageIdAdv = (MessageIdAdv) messageId; + } else { + try { + messageIdAdv = (MessageIdAdv) MessageId.fromByteArray(messageId.toByteArray()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return new TopicMessageIdImpl(topic, messageIdAdv); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java index 189dc1c608379..00fe12b62b194 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java @@ -18,15 +18,27 @@ */ package org.apache.pulsar.client.impl; +import java.util.BitSet; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.TopicMessageId; -public class TopicMessageIdImpl extends TopicMessageId.Impl { +public class TopicMessageIdImpl implements MessageIdAdv, TopicMessageId { - private final String topicName; + private final String ownerTopic; + private final MessageIdAdv msgId; + private final String topicName; // it's never used + public TopicMessageIdImpl(String topic, MessageIdAdv msgId) { + this.ownerTopic = topic; + this.msgId = msgId; + this.topicName = ""; + } + + @Deprecated public TopicMessageIdImpl(String topicPartitionName, String topicName, MessageId messageId) { - super(topicPartitionName, messageId); + this.msgId = (MessageIdAdv) messageId; + this.ownerTopic = topicPartitionName; this.topicName = topicName; } @@ -55,11 +67,66 @@ public MessageId getInnerMessageId() { @Override public boolean equals(Object obj) { - return super.equals(obj); + return msgId.equals(obj); } @Override public int hashCode() { - return super.hashCode(); + return msgId.hashCode(); + } + + @Override + public int compareTo(MessageId o) { + return msgId.compareTo(o); + } + + @Override + public byte[] toByteArray() { + return msgId.toByteArray(); + } + + @Override + public String getOwnerTopic() { + return ownerTopic; + } + + @Override + public long getLedgerId() { + return msgId.getLedgerId(); + } + + @Override + public long getEntryId() { + return msgId.getEntryId(); + } + + @Override + public int getPartitionIndex() { + return msgId.getPartitionIndex(); + } + + @Override + public int getBatchIndex() { + return msgId.getBatchIndex(); + } + + @Override + public int getBatchSize() { + return msgId.getBatchSize(); + } + + @Override + public BitSet getAckSet() { + return msgId.getAckSet(); + } + + @Override + public MessageIdAdv getFirstChunkMessageId() { + return msgId.getFirstChunkMessageId(); + } + + @Override + public String toString() { + return msgId.toString(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java index d24ecbd6aa917..1b6cba2f7234d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java @@ -22,6 +22,7 @@ import java.util.Optional; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.EncryptionContext; @@ -42,7 +43,7 @@ public class TopicMessageImpl implements Message { this.receivedByconsumer = receivedByConsumer; this.msg = msg; - this.messageId = new TopicMessageIdImpl(topicPartitionName, topicPartitionName, msg.getMessageId()); + this.messageId = new TopicMessageIdImpl(topicPartitionName, (MessageIdAdv) msg.getMessageId()); } /** diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java index 4f0eca6ea4af8..fd81e9d5790ad 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java @@ -148,15 +148,12 @@ public void testMessageIdImplCompareToTopicMessageId() { MessageIdImpl messageIdImpl = new MessageIdImpl(123L, 345L, 567); TopicMessageIdImpl topicMessageId1 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new BatchMessageIdImpl(123L, 345L, 566, 789)); TopicMessageIdImpl topicMessageId2 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new BatchMessageIdImpl(123L, 345L, 567, 789)); TopicMessageIdImpl topicMessageId3 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new BatchMessageIdImpl(messageIdImpl)); assertTrue(messageIdImpl.compareTo(topicMessageId1) > 0, "Expected to be greater than"); assertTrue(messageIdImpl.compareTo(topicMessageId2) < 0, "Expected to be less than"); @@ -173,11 +170,9 @@ public void testBatchMessageIdImplCompareToTopicMessageId() { BatchMessageIdImpl messageIdImpl3 = new BatchMessageIdImpl(123L, 345L, 567, -1); TopicMessageIdImpl topicMessageId1 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new MessageIdImpl(123L, 345L, 566)); TopicMessageIdImpl topicMessageId2 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new MessageIdImpl(123L, 345L, 567)); assertTrue(messageIdImpl1.compareTo(topicMessageId1) > 0, "Expected to be greater than"); assertTrue(messageIdImpl1.compareTo(topicMessageId2) > 0, "Expected to be greater than"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java index d2e2ce9c15c54..daf49f0e7752e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java @@ -28,9 +28,9 @@ public class TopicMessageIdImplTest { public void hashCodeTest() { MessageIdImpl msgId1 = new MessageIdImpl(0, 0, 0); MessageIdImpl msgId2 = new BatchMessageIdImpl(1, 1, 1, 1); - TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", "topic", msgId1); - TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", "topic2", msgId1); - TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", "topic", msgId2); + TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", msgId1); + TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", msgId1); + TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", msgId2); assertEquals(topicMsgId1.hashCode(), topicMsgId1.hashCode()); assertEquals(topic2MsgId1.hashCode(), topic2MsgId1.hashCode()); @@ -43,9 +43,9 @@ public void hashCodeTest() { public void equalsTest() { MessageIdImpl msgId1 = new MessageIdImpl(0, 0, 0); MessageIdImpl msgId2 = new BatchMessageIdImpl(1, 1, 1, 1); - TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", "topic", msgId1); - TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", "topic2", msgId1); - TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", "topic", msgId2); + TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", msgId1); + TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", msgId1); + TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", msgId2); assertEquals(topicMsgId1, topicMsgId1); assertEquals(topicMsgId1, topic2MsgId1); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java similarity index 100% rename from pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java rename to pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java new file mode 100644 index 0000000000000..3f6d1d56e1032 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Additional helper classes to the pulsar-client-api module. + */ +package org.apache.pulsar.client.api; diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index 5410e0bb8d664..1100b13b425b4 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -1572,7 +1572,7 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception { assertNull(ref); ref = KafkaConnectSink.getMessageSequenceRefForBatchMessage( - new TopicMessageIdImpl("topic-0", "topic", new MessageIdImpl(ledgerId, entryId, 0)) + new TopicMessageIdImpl("topic-0", new MessageIdImpl(ledgerId, entryId, 0)) ); assertNull(ref); @@ -1584,7 +1584,7 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception { assertEquals(ref.getBatchIdx(), batchIdx); ref = KafkaConnectSink.getMessageSequenceRefForBatchMessage( - new TopicMessageIdImpl("topic-0", "topic", new BatchMessageIdImpl(ledgerId, entryId, 0, batchIdx)) + new TopicMessageIdImpl("topic-0", new BatchMessageIdImpl(ledgerId, entryId, 0, batchIdx)) ); assertEquals(ref.getLedgerId(), ledgerId); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index 579b423339911..c988fd1e70ce3 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.DeadLetterPolicy; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException; @@ -45,6 +46,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.impl.ConsumerBuilderImpl; +import org.apache.pulsar.client.impl.TopicMessageIdImpl; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.websocket.data.ConsumerCommand; @@ -293,8 +295,8 @@ private void checkResumeReceive() { private void handleAck(ConsumerCommand command) throws IOException { // We should have received an ack - TopicMessageId msgId = TopicMessageId.create(topic.toString(), - MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId))); + TopicMessageId msgId = new TopicMessageIdImpl(topic.toString(), + (MessageIdAdv) MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId))); if (log.isDebugEnabled()) { log.debug("[{}/{}] Received ack request of message {} from {} ", consumer.getTopic(), subscription, msgId, getRemote().getInetSocketAddress().toString()); From 00d09cbbd2b3063fecbdc3988b5eef7824f40ce0 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 20 Apr 2023 18:11:52 +0800 Subject: [PATCH 134/280] [fix][txn] Fix transaction is not aborted when send or ACK failed (#20055) --- .../transaction/TransactionProduceTest.java | 35 ++++++++-- .../impl/transaction/TransactionImpl.java | 70 +++++++++---------- 2 files changed, 62 insertions(+), 43 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index cdbb1563280b4..ddd8cf0790321 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import lombok.Cleanup; @@ -43,6 +45,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; @@ -51,10 +54,11 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.awaitility.Awaitility; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; /** @@ -70,7 +74,7 @@ public class TransactionProduceTest extends TransactionTestBase { private static final String ACK_COMMIT_TOPIC = NAMESPACE1 + "/ack-commit"; private static final String ACK_ABORT_TOPIC = NAMESPACE1 + "/ack-abort"; private static final int NUM_PARTITIONS = 16; - @BeforeMethod + @BeforeClass protected void setup() throws Exception { setUpBase(1, NUM_PARTITIONS, PRODUCE_COMMIT_TOPIC, TOPIC_PARTITION); admin.topics().createPartitionedTopic(PRODUCE_ABORT_TOPIC, TOPIC_PARTITION); @@ -78,7 +82,7 @@ protected void setup() throws Exception { admin.topics().createPartitionedTopic(ACK_ABORT_TOPIC, TOPIC_PARTITION); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { super.internalCleanup(); } @@ -369,5 +373,26 @@ private int getPendingAckCount(String topic, String subscriptionName) throws Exc return pendingAckCount; } - + @Test + public void testCommitFailure() throws Exception { + Transaction txn = pulsarClient.newTransaction().build().get(); + final String topic = NAMESPACE1 + "/test-commit-failure"; + @Cleanup + final Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.newMessage(txn).value(new byte[1024 * 1024 * 10]).sendAsync(); + try { + txn.commit().get(); + Assert.fail(); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof PulsarClientException.TransactionHasOperationFailedException); + Assert.assertEquals(txn.getState(), Transaction.State.ABORTED); + } + try { + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getTxnMeta(txn.getTxnID()) + .getNow(null); + Assert.fail(); + } catch (CompletionException e) { + Assert.assertTrue(e.getCause() instanceof CoordinatorException.TransactionNotFoundException); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index b7e085ed82a85..d1260ba045e6d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import io.netty.util.Timeout; import io.netty.util.TimerTask; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -186,13 +187,14 @@ public void registerAckOp(CompletableFuture newAckFuture) { @Override public CompletableFuture commit() { timeout.cancel(); - return checkIfOpenOrCommitting().thenCompose((value) -> { + return checkState(State.OPEN, State.COMMITTING).thenCompose((value) -> { CompletableFuture commitFuture = new CompletableFuture<>(); this.state = State.COMMITTING; opFuture.whenComplete((v, e) -> { if (hasOpsFailed) { - abort().whenComplete((vx, ex) -> commitFuture.completeExceptionally(new PulsarClientException - .TransactionHasOperationFailedException())); + checkState(State.COMMITTING).thenCompose(__ -> internalAbort()).whenComplete((vx, ex) -> + commitFuture.completeExceptionally( + new PulsarClientException.TransactionHasOperationFailedException())); } else { tcClient.commitAsync(txnId) .whenComplete((vx, ex) -> { @@ -216,28 +218,30 @@ public CompletableFuture commit() { @Override public CompletableFuture abort() { timeout.cancel(); - return checkIfOpenOrAborting().thenCompose(value -> { - CompletableFuture abortFuture = new CompletableFuture<>(); - this.state = State.ABORTING; - opFuture.whenComplete((v, e) -> { - tcClient.abortAsync(txnId).whenComplete((vx, ex) -> { + return checkState(State.OPEN, State.ABORTING).thenCompose(__ -> internalAbort()); + } - if (ex != null) { - if (ex instanceof TransactionNotFoundException - || ex instanceof InvalidTxnStatusException) { - this.state = State.ERROR; - } - abortFuture.completeExceptionally(ex); - } else { - this.state = State.ABORTED; - abortFuture.complete(null); + private CompletableFuture internalAbort() { + CompletableFuture abortFuture = new CompletableFuture<>(); + this.state = State.ABORTING; + opFuture.whenComplete((v, e) -> { + tcClient.abortAsync(txnId).whenComplete((vx, ex) -> { + + if (ex != null) { + if (ex instanceof TransactionNotFoundException + || ex instanceof InvalidTxnStatusException) { + this.state = State.ERROR; } + abortFuture.completeExceptionally(ex); + } else { + this.state = State.ABORTED; + abortFuture.complete(null); + } - }); }); - - return abortFuture; }); + + return abortFuture; } @Override @@ -261,25 +265,15 @@ public boolean checkIfOpen(CompletableFuture completableFuture) { } } - private CompletableFuture checkIfOpenOrCommitting() { - if (state == State.OPEN || state == State.COMMITTING) { - return CompletableFuture.completedFuture(null); - } else { - return invalidTxnStatusFuture(); - } - } - - private CompletableFuture checkIfOpenOrAborting() { - if (state == State.OPEN || state == State.ABORTING) { - return CompletableFuture.completedFuture(null); - } else { - return invalidTxnStatusFuture(); + private CompletableFuture checkState(State... expectedStates) { + final State actualState = STATE_UPDATE.get(this); + for (State expectedState : expectedStates) { + if (actualState == expectedState) { + return CompletableFuture.completedFuture(null); + } } - } - - private CompletableFuture invalidTxnStatusFuture() { return FutureUtil.failedFuture(new InvalidTxnStatusException("[" + txnIdMostBits + ":" - + txnIdLeastBits + "] with unexpected state : " - + state.name() + ", expect " + State.OPEN + " state!")); + + txnIdLeastBits + "] with unexpected state: " + actualState.name() + ", expect: " + + Arrays.toString(expectedStates))); } } From 2b41e4eafb1cba0e548dd90df60e8cdbb24cd490 Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Thu, 20 Apr 2023 20:13:50 +0800 Subject: [PATCH 135/280] [improve] [broker] Pin AppendIndexMetadataInterceptor to field in `ManagedLedgerInterceptorImpl` (#20112) --- .../ManagedLedgerInterceptorImpl.java | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java index 30713c91907ef..02c6c575fd919 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java @@ -40,12 +40,27 @@ public class ManagedLedgerInterceptorImpl implements ManagedLedgerInterceptor { private static final Logger log = LoggerFactory.getLogger(ManagedLedgerInterceptorImpl.class); private static final String INDEX = "index"; private final Set brokerEntryMetadataInterceptors; + + private final AppendIndexMetadataInterceptor appendIndexMetadataInterceptor; private final Set inputProcessors; private final Set outputProcessors; public ManagedLedgerInterceptorImpl(Set brokerEntryMetadataInterceptors, Set brokerEntryPayloadProcessors) { this.brokerEntryMetadataInterceptors = brokerEntryMetadataInterceptors; + + // save appendIndexMetadataInterceptor to field + AppendIndexMetadataInterceptor appendIndexMetadataInterceptor = null; + + for (BrokerEntryMetadataInterceptor interceptor : this.brokerEntryMetadataInterceptors) { + if (interceptor instanceof AppendIndexMetadataInterceptor) { + appendIndexMetadataInterceptor = (AppendIndexMetadataInterceptor) interceptor; + break; + } + } + + this.appendIndexMetadataInterceptor = appendIndexMetadataInterceptor; + if (brokerEntryPayloadProcessors != null) { this.inputProcessors = new LinkedHashSet<>(); this.outputProcessors = new LinkedHashSet<>(); @@ -61,12 +76,11 @@ public ManagedLedgerInterceptorImpl(Set brokerEn public long getIndex() { long index = -1; - for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) { - if (interceptor instanceof AppendIndexMetadataInterceptor) { - index = ((AppendIndexMetadataInterceptor) interceptor).getIndex(); - break; - } + + if (appendIndexMetadataInterceptor != null) { + return appendIndexMetadataInterceptor.getIndex(); } + return index; } @@ -81,10 +95,8 @@ public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { @Override public void afterFailedAddEntry(int numberOfMessages) { - for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) { - if (interceptor instanceof AppendIndexMetadataInterceptor) { - ((AppendIndexMetadataInterceptor) interceptor).decreaseWithNumberOfMessages(numberOfMessages); - } + if (appendIndexMetadataInterceptor != null) { + appendIndexMetadataInterceptor.decreaseWithNumberOfMessages(numberOfMessages); } } @@ -95,12 +107,9 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa } if (propertiesMap.containsKey(INDEX)) { - for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) { - if (interceptor instanceof AppendIndexMetadataInterceptor) { - ((AppendIndexMetadataInterceptor) interceptor) - .recoveryIndexGenerator(Long.parseLong(propertiesMap.get(INDEX))); - break; - } + if (appendIndexMetadataInterceptor != null) { + appendIndexMetadataInterceptor.recoveryIndexGenerator( + Long.parseLong(propertiesMap.get(INDEX))); } } } @@ -108,8 +117,7 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa @Override public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle lh) { CompletableFuture promise = new CompletableFuture<>(); - boolean hasAppendIndexMetadataInterceptor = brokerEntryMetadataInterceptors.stream() - .anyMatch(interceptor -> interceptor instanceof AppendIndexMetadataInterceptor); + boolean hasAppendIndexMetadataInterceptor = appendIndexMetadataInterceptor != null; if (hasAppendIndexMetadataInterceptor && lh.getLastAddConfirmed() >= 0) { lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()).whenComplete((entries, ex) -> { if (ex != null) { @@ -122,14 +130,9 @@ public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, if (ledgerEntry != null) { BrokerEntryMetadata brokerEntryMetadata = Commands.parseBrokerEntryMetadataIfExist(ledgerEntry.getEntryBuffer()); - for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) { - if (interceptor instanceof AppendIndexMetadataInterceptor) { - if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) { - ((AppendIndexMetadataInterceptor) interceptor) - .recoveryIndexGenerator(brokerEntryMetadata.getIndex()); - } - break; - } + if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) { + appendIndexMetadataInterceptor.recoveryIndexGenerator( + brokerEntryMetadata.getIndex()); } } entries.close(); @@ -153,11 +156,8 @@ public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, @Override public void onUpdateManagedLedgerInfo(Map propertiesMap) { - for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) { - if (interceptor instanceof AppendIndexMetadataInterceptor) { - propertiesMap.put(INDEX, String.valueOf(((AppendIndexMetadataInterceptor) interceptor).getIndex())); - break; - } + if (appendIndexMetadataInterceptor != null) { + propertiesMap.put(INDEX, String.valueOf(appendIndexMetadataInterceptor.getIndex())); } } From e1a3fa8f6d9e3cb4e4357ee40263f55e1c1491b8 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Fri, 21 Apr 2023 09:39:57 +0800 Subject: [PATCH 136/280] [improve][broker] Support disabling delayed bucket merging. (#20155) --- conf/broker.conf | 3 ++- conf/standalone.conf | 3 ++- .../java/org/apache/pulsar/broker/ServiceConfiguration.java | 5 +++-- .../broker/delayed/bucket/BucketDelayedDeliveryTracker.java | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 292b1ef1683b7..89d2d85200448 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -577,7 +577,8 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000 # The max number of delayed message index bucket, # after reaching the max buckets limitation, the adjacent buckets will be merged. -delayedDeliveryMaxNumBuckets=50 +# (disable with value -1) +delayedDeliveryMaxNumBuckets=-1 # Size of the lookahead window to use when detecting if all the messages in the topic # have a fixed delay. diff --git a/conf/standalone.conf b/conf/standalone.conf index f141946c29f4e..63bc7a29ae6da 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -1264,4 +1264,5 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000 # The max number of delayed message index bucket, # after reaching the max buckets limitation, the adjacent buckets will be merged. -delayedDeliveryMaxNumBuckets=50 +# (disable with value -1) +delayedDeliveryMaxNumBuckets=-1 diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index b31c86560f848..2c48310f96482 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -372,8 +372,9 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext(category = CATEGORY_SERVER, doc = """ The max number of delayed message index bucket, \ - after reaching the max buckets limitation, the adjacent buckets will be merged.""") - private int delayedDeliveryMaxNumBuckets = 50; + after reaching the max buckets limitation, the adjacent buckets will be merged.\ + (disable with value -1)""") + private int delayedDeliveryMaxNumBuckets = -1; @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use " + "when detecting if all the messages in the topic have a fixed delay. " diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 5d1dc2e27033a..b17387e276e2b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -335,7 +335,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime); lastMutableBucket.resetLastMutableBucketRange(); - if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { + if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { asyncMergeBucketSnapshot(); } } From a20d5e96acc91f8099845e8924e84cb0b5632f3f Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 21 Apr 2023 09:51:55 +0800 Subject: [PATCH 137/280] [improve][broker] Optimization protobuf code in the bucket delayed tracker (#20158) --- pulsar-broker/pom.xml | 2 + .../BookkeeperBucketSnapshotStorage.java | 24 ++--- .../pulsar/broker/delayed/bucket/Bucket.java | 7 +- .../bucket/BucketDelayedDeliveryTracker.java | 13 ++- .../delayed/bucket/BucketSnapshotStorage.java | 4 +- .../CombinedSegmentDelayedIndexQueue.java | 22 ++++- .../delayed/bucket/DelayedIndexQueue.java | 12 ++- .../delayed/bucket/ImmutableBucket.java | 19 ++-- .../broker/delayed/bucket/MutableBucket.java | 29 +++--- .../TripleLongPriorityDelayedIndexQueue.java | 25 +++-- ...> DelayedMessageIndexBucketMetadata.proto} | 11 +-- .../DelayedMessageIndexBucketSegment.proto | 35 +++++++ .../BookkeeperBucketSnapshotStorageTest.java | 95 +++++++++---------- .../delayed/MockBucketSnapshotStorage.java | 14 +-- .../delayed/bucket/DelayedIndexQueueTest.java | 70 ++++++-------- 15 files changed, 210 insertions(+), 172 deletions(-) rename pulsar-broker/src/main/proto/{DelayedMessageIndexBucketSnapshotFormat.proto => DelayedMessageIndexBucketMetadata.proto} (85%) create mode 100644 pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 7442d95e467e6..6e98386e9e931 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -566,6 +566,7 @@ **/ResourceUsage.proto **/TransactionPendingAck.proto + **/DelayedMessageIndexBucketSegment.proto @@ -610,6 +611,7 @@ ${project.basedir}/src/main/proto/TransactionPendingAck.proto ${project.basedir}/src/main/proto/ResourceUsage.proto + ${project.basedir}/src/main/proto/DelayedMessageIndexBucketSegment.proto generated-sources/lightproto/java generated-sources/lightproto/java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 18a4c322f7b27..040bbbc586f49 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import com.google.protobuf.InvalidProtocolBufferException; -import java.io.IOException; +import io.netty.buffer.ByteBuf; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -36,8 +36,8 @@ import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -126,7 +126,8 @@ private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle, private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { try { - return SnapshotMetadata.parseFrom(ledgerEntry.getEntry()); + ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer()); } catch (InvalidProtocolBufferException e) { throw new BucketSnapshotSerializationException(e); } @@ -134,15 +135,14 @@ private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { private List parseSnapshotSegmentEntries(Enumeration entryEnumeration) { List snapshotMetadataList = new ArrayList<>(); - try { - while (entryEnumeration.hasMoreElements()) { - LedgerEntry ledgerEntry = entryEnumeration.nextElement(); - snapshotMetadataList.add(SnapshotSegment.parseFrom(ledgerEntry.getEntry())); - } - return snapshotMetadataList; - } catch (IOException e) { - throw new BucketSnapshotSerializationException(e); + while (entryEnumeration.hasMoreElements()) { + LedgerEntry ledgerEntry = entryEnumeration.nextElement(); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + snapshotSegment.parseFrom(entryBuffer, entryBuffer.readableBytes()); + snapshotMetadataList.add(snapshotSegment); } + return snapshotMetadataList; } @NotNull diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 4d7d3aa512be6..a1693b1553d97 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -31,7 +31,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.RoaringBitmap; @@ -132,8 +133,8 @@ long getAndUpdateBucketId() { } CompletableFuture asyncSaveBucketSnapshot( - ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata, - List bucketSnapshotSegments) { + ImmutableBucket bucket, SnapshotMetadata snapshotMetadata, + List bucketSnapshotSegments) { final String bucketKey = bucket.bucketKey(); final String cursorName = Codec.decode(cursor.getName()); final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index b17387e276e2b..c90064c9137bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -53,8 +53,8 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.util.FutureUtil; @@ -286,8 +286,7 @@ private void afterCreateImmutableBucket(Pair immu // Put indexes back into the shared queue and downgrade to memory mode synchronized (BucketDelayedDeliveryTracker.this) { immutableBucket.getSnapshotSegments().ifPresent(snapshotSegments -> { - for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment : - snapshotSegments) { + for (SnapshotSegment snapshotSegment : snapshotSegments) { for (DelayedIndex delayedIndex : snapshotSegment.getIndexesList()) { sharedBucketPriorityQueue.add(delayedIndex.getTimestamp(), delayedIndex.getLedgerId(), delayedIndex.getEntryId()); @@ -450,7 +449,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List>> getRemainFutures = + List>> getRemainFutures = buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList(); return FutureUtil.waitForAll(getRemainFutures) @@ -601,11 +600,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa bucket.asyncDeleteBucketSnapshot(stats); return; } - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex + DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId(), bucket); - for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : indexList) { + for (DelayedIndex index : indexList) { sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java index 51c89bed47af2..7464ef9cd3f63 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java @@ -20,8 +20,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; public interface BucketSnapshotStorage { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java index 5655a26878296..006938e9ed271 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java @@ -24,8 +24,8 @@ import java.util.PriorityQueue; import javax.annotation.concurrent.NotThreadSafe; import lombok.AllArgsConstructor; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; @NotThreadSafe class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue { @@ -40,8 +40,8 @@ static class Node { } private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare( - node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor), - node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor)); + node1.segmentList.get(node1.segmentListCursor).getIndexeAt(node1.segmentCursor), + node2.segmentList.get(node2.segmentListCursor).getIndexeAt(node2.segmentCursor)); private final PriorityQueue kpq; @@ -77,7 +77,7 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { Objects.requireNonNull(node); SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor); - DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor); + DelayedIndex delayedIndex = snapshotSegment.getIndexeAt(node.segmentCursor); if (!needAdvanceCursor) { return delayedIndex; } @@ -104,4 +104,16 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { return delayedIndex; } + + @Override + public void popToObject(DelayedIndex delayedIndex) { + DelayedIndex value = getValue(true); + delayedIndex.copyFrom(value); + } + + @Override + public long peekTimestamp() { + DelayedIndex value = getValue(false); + return value.getTimestamp(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java index dee476c376ec9..f1209a3137a45 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java @@ -20,10 +20,10 @@ import java.util.Comparator; import java.util.Objects; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; interface DelayedIndexQueue { - Comparator COMPARATOR = (o1, o2) -> { + Comparator COMPARATOR = (o1, o2) -> { if (!Objects.equals(o1.getTimestamp(), o2.getTimestamp())) { return Long.compare(o1.getTimestamp(), o2.getTimestamp()); } else if (!Objects.equals(o1.getLedgerId(), o2.getLedgerId())) { @@ -35,7 +35,11 @@ interface DelayedIndexQueue { boolean isEmpty(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek(); + DelayedIndex peek(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop(); + DelayedIndex pop(); + + void popToObject(DelayedIndex delayedIndex); + + long peekTimestamp(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 57de5c84fcd82..0932f51f350ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -32,9 +32,9 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.mutable.MutableLong; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.InvalidRoaringFormat; import org.roaringbitmap.RoaringBitmap; @@ -43,7 +43,7 @@ class ImmutableBucket extends Bucket { @Setter - private List snapshotSegments; + private List snapshotSegments; boolean merging = false; @@ -55,7 +55,7 @@ class ImmutableBucket extends Bucket { super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId); } - public Optional> getSnapshotSegments() { + public Optional> getSnapshotSegments() { return Optional.ofNullable(snapshotSegments); } @@ -84,7 +84,7 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) .thenApply(snapshotMetadata -> { - List metadataList = + List metadataList = snapshotMetadata.getMetadataListList(); // Skip all already reach schedule time snapshot segments @@ -125,10 +125,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b return Collections.emptyList(); } - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - List indexList = - snapshotSegment.getIndexesList(); + List indexList = snapshotSegment.getIndexesList(); this.setCurrentSegmentEntryId(nextSegmentEntryId); if (isRecover) { this.asyncUpdateSnapshotLength(); @@ -171,7 +170,7 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, setNumberBucketDelayedMessages(numberMessages.getValue()); } - CompletableFuture> getRemainSnapshotSegment() { + CompletableFuture> getRemainSnapshotSegment() { int nextSegmentEntryId = currentSegmentEntryId + 1; if (nextSegmentEntryId > lastSegmentEntryId) { return CompletableFuture.completedFuture(Collections.emptyList()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index f404d5d02c15a..b7e9e68f1bdc7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -30,10 +30,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.roaringbitmap.RoaringBitmap; @@ -75,14 +75,16 @@ Pair createImmutableBucketAndAsyncPersistent( List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); Map bitMap = new HashMap<>(); - SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); + SnapshotSegment snapshotSegment = new SnapshotSegment(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); List firstScheduleTimestamps = new ArrayList<>(); long currentTimestampUpperLimit = 0; long currentFirstTimestamp = 0L; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex delayedIndex = delayedIndexQueue.peek(); + DelayedIndex delayedIndex = snapshotSegment.addIndexe(); + delayedIndexQueue.popToObject(delayedIndex); + long timestamp = delayedIndex.getTimestamp(); if (currentTimestampUpperLimit == 0) { currentFirstTimestamp = timestamp; @@ -100,16 +102,13 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } - delayedIndexQueue.pop(); numMessages++; bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); - snapshotSegmentBuilder.addIndexes(delayedIndex); - - if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit + if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit || (maxIndexesPerBucketSnapshotSegment != -1 - && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { + && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp); currentTimestampUpperLimit = 0; @@ -129,8 +128,8 @@ Pair createImmutableBucketAndAsyncPersistent( segmentMetadataList.add(segmentMetadataBuilder.build()); segmentMetadataBuilder.clear(); - bucketSnapshotSegments.add(snapshotSegmentBuilder.build()); - snapshotSegmentBuilder.clear(); + bucketSnapshotSegments.add(snapshotSegment); + snapshotSegment = new SnapshotSegment(); } } @@ -153,8 +152,8 @@ Pair createImmutableBucketAndAsyncPersistent( // Add the first snapshot segment last message to snapshotSegmentLastMessageTable checkArgument(!bucketSnapshotSegments.isEmpty()); - SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - DelayedIndex lastDelayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); + SnapshotSegment firstSnapshotSegment = bucketSnapshotSegments.get(0); + DelayedIndex lastDelayedIndex = firstSnapshotSegment.getIndexeAt(firstSnapshotSegment.getIndexesCount() - 1); Pair result = Pair.of(bucket, lastDelayedIndex); CompletableFuture future = asyncSaveBucketSnapshot(bucket, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java index b8d54bd78b428..4faee3b17f17f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import javax.annotation.concurrent.NotThreadSafe; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @NotThreadSafe @@ -41,17 +41,28 @@ public boolean isEmpty() { } @Override - public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek() { - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setTimestamp(queue.peekN1()) - .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()).build(); + public DelayedIndex peek() { + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(queue.peekN1()) + .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); return delayedIndex; } @Override - public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop() { - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek = peek(); + public DelayedIndex pop() { + DelayedIndex peek = peek(); queue.pop(); return peek; } + + @Override + public void popToObject(DelayedIndex delayedIndex) { + delayedIndex.setTimestamp(queue.peekN1()) + .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); + queue.pop(); + } + + @Override + public long peekTimestamp() { + return queue.peekN1(); + } } diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto similarity index 85% rename from pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto rename to pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto index 6996b860c5249..01b770c567d09 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto @@ -21,12 +21,7 @@ syntax = "proto2"; package pulsar.delay; option java_package = "org.apache.pulsar.broker.delayed.proto"; option optimize_for = SPEED; - -message DelayedIndex { - required uint64 timestamp = 1; - required uint64 ledger_id = 2; - required uint64 entry_id = 3; -} +option java_multiple_files = true; message SnapshotSegmentMetadata { map delayed_index_bit_map = 1; @@ -34,10 +29,6 @@ message SnapshotSegmentMetadata { required uint64 min_schedule_timestamp = 3; } -message SnapshotSegment { - repeated DelayedIndex indexes = 1; -} - message SnapshotMetadata { repeated SnapshotSegmentMetadata metadata_list = 1; } diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto new file mode 100644 index 0000000000000..633d6a8f1615c --- /dev/null +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +syntax = "proto2"; + +package pulsar.delay; +option java_package = "org.apache.pulsar.broker.delayed.proto"; +option optimize_for = SPEED; +option java_multiple_files = true; + +message DelayedIndex { + required uint64 timestamp = 1; + required uint64 ledger_id = 2; + required uint64 entry_id = 3; +} + +message SnapshotSegment { + repeated DelayedIndex indexes = 1; + map delayed_index_bit_map = 2; +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index 7cb6b8d5865bb..d26f38fa2bc2b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -29,7 +29,10 @@ import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -60,9 +63,8 @@ protected void cleanup() throws Exception { @Test public void testCreateSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + SnapshotMetadata snapshotMetadata = SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -72,24 +74,23 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException @Test public void testGetSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -99,13 +100,13 @@ public void testGetSnapshot() throws ExecutionException, InterruptedException { Long bucketId = future.get(); Assert.assertNotNull(bucketId); - CompletableFuture> bucketSnapshotSegment = + CompletableFuture> bucketSnapshotSegment = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3); - List snapshotSegments = bucketSnapshotSegment.get(); + List snapshotSegments = bucketSnapshotSegment.get(); Assert.assertEquals(2, snapshotSegments.size()); - for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment segment : snapshotSegments) { - for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : segment.getIndexesList()) { + for (SnapshotSegment segment : snapshotSegments) { + for (DelayedIndex index : segment.getIndexesList()) { Assert.assertEquals(100L, index.getLedgerId()); Assert.assertEquals(10L, index.getEntryId()); Assert.assertEquals(timeMillis, index.getTimestamp()); @@ -121,17 +122,17 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce map.put(100L, ByteString.copyFrom("test1", StandardCharsets.UTF_8)); map.put(200L, ByteString.copyFrom("test2", StandardCharsets.UTF_8)); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMaxScheduleTimestamp(timeMillis) .setMinScheduleTimestamp(timeMillis) .putAllDelayedIndexBitMap(map).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, @@ -139,10 +140,10 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce Long bucketId = future.get(); Assert.assertNotNull(bucketId); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata bucketSnapshotMetadata = + SnapshotMetadata bucketSnapshotMetadata = bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId).get(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata metadata = + SnapshotSegmentMetadata metadata = bucketSnapshotMetadata.getMetadataList(0); Assert.assertEquals(timeMillis, metadata.getMaxScheduleTimestamp()); @@ -152,9 +153,9 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce @Test public void testDeleteSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -173,24 +174,22 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException @Test public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -207,24 +206,22 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted @Test public void testConcurrencyGet() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -237,7 +234,7 @@ public void testConcurrencyGet() throws ExecutionException, InterruptedException List> futures = new ArrayList<>(); for (int i = 0; i < 100; i++) { CompletableFuture future0 = CompletableFuture.runAsync(() -> { - List list = + List list = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); Assert.assertTrue(list.size() > 0); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index 9e924bdeda341..dc1c0e09ca276 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -36,8 +36,8 @@ import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -139,13 +139,9 @@ public CompletableFuture> getBucketSnapshotSegment(long bu long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size()); for (int i = (int) firstSegmentEntryId; i <= lastEntryId ; i++) { ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(i); - SnapshotSegment snapshotSegment; - try { - snapshotSegment = SnapshotSegment.parseFrom(byteBuf.nioBuffer()); - snapshotSegments.add(snapshotSegment); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.parseFrom(byteBuf, byteBuf.readableBytes()); + snapshotSegments.add(snapshotSegment); } return snapshotSegments; }, executorService); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java index 8f87f0d49a20c..5dc3dcc7cb9a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java @@ -23,8 +23,8 @@ import java.util.List; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.testng.Assert; import org.testng.annotations.Test; @@ -35,27 +35,19 @@ public class DelayedIndexQueueTest { @Test public void testCompare() { DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(2).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(2).setLedgerId(2L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(2L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); - delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); - delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(2L) - .build(); + delayedIndex = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); + delayedIndex2 = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); } @@ -63,21 +55,19 @@ public void testCompare() { public void testCombinedSegmentDelayedIndexQueue() { List listA = new ArrayList<>(); for (int i = 0; i < 10; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); listA.add(delayedIndex); } - SnapshotSegment snapshotSegmentA1 = SnapshotSegment.newBuilder().addAllIndexes(listA).build(); + SnapshotSegment snapshotSegmentA1 = new SnapshotSegment(); + snapshotSegmentA1.addAllIndexes(listA); List listA2 = new ArrayList<>(); for (int i = 10; i < 20; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); listA2.add(delayedIndex); } - SnapshotSegment snapshotSegmentA2 = SnapshotSegment.newBuilder().addAllIndexes(listA2).build(); + SnapshotSegment snapshotSegmentA2 = new SnapshotSegment(); + snapshotSegmentA2.addAllIndexes(listA2); List segmentListA = new ArrayList<>(); segmentListA.add(snapshotSegmentA1); @@ -85,36 +75,32 @@ public void testCombinedSegmentDelayedIndexQueue() { List listB = new ArrayList<>(); for (int i = 0; i < 9; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); - DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) - .build(); + DelayedIndex delayedIndex2 = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); listB.add(delayedIndex); listB.add(delayedIndex2); } - SnapshotSegment snapshotSegmentB = SnapshotSegment.newBuilder().addAllIndexes(listB).build(); + SnapshotSegment snapshotSegmentB = new SnapshotSegment(); + snapshotSegmentB.addAllIndexes(listB); List segmentListB = new ArrayList<>(); segmentListB.add(snapshotSegmentB); - segmentListB.add(SnapshotSegment.newBuilder().build()); + segmentListB.add(new SnapshotSegment()); List listC = new ArrayList<>(); for (int i = 10; i < 30; i+=2) { DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); listC.add(delayedIndex); listC.add(delayedIndex2); } - SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build(); + SnapshotSegment snapshotSegmentC = new SnapshotSegment(); + snapshotSegmentC.addAllIndexes(listC); List segmentListC = new ArrayList<>(); segmentListC.add(snapshotSegmentC); @@ -123,11 +109,14 @@ public void testCombinedSegmentDelayedIndexQueue() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = delayedIndexQueue.pop(); + DelayedIndex pop = new DelayedIndex(); + delayedIndexQueue.popToObject(pop); log.info("{} , {}, {}", pop.getTimestamp(), pop.getLedgerId(), pop.getEntryId()); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); + long timestamp = delayedIndexQueue.peekTimestamp(); + Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } @@ -147,10 +136,13 @@ public void TripleLongPriorityDelayedIndexQueueTest() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = delayedIndexQueue.pop(); + DelayedIndex pop = new DelayedIndex(); + delayedIndexQueue.popToObject(pop); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); + long timestamp = delayedIndexQueue.peekTimestamp(); + Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } From fce6e737a7ff785859117b216cbb12f999ddeb94 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 21 Apr 2023 09:57:06 +0800 Subject: [PATCH 138/280] [improve][schema] Add admin cli for testCompatibility (#19974) ### Motivation 1. add admin cli `testCompatibility` 2. add `admin.schemas().testCompatibility()` test ### Modifications 1. add admin cli `testCompatibility` 2. add `admin.schemas().testCompatibility()` test ### Verifying this change add test for it - [ ] Make sure that the change passes the CI checks. *(Please pick either of the following options)* This change is a trivial rework / code cleanup without any test coverage. *(or)* This change is already covered by existing tests, such as *(please describe tests)*. *(or)* This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Does this pull request potentially affect one of the following parts: *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [ ] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [x] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [x] `doc-not-needed` - [ ] `doc-complete` ### Matching PR in forked repository PR in forked repository: https://github.com/congbobo184/pulsar/pull/14 --- .../broker/admin/AdminApiSchemaTest.java | 29 +++++++++++++++++++ .../schema/IsCompatibilityResponse.java | 1 - .../pulsar/admin/cli/PulsarAdminToolTest.java | 5 ++++ .../apache/pulsar/admin/cli/CmdSchemas.java | 17 +++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java index b37114f180216..f67bd6fcfce5b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java @@ -49,6 +49,8 @@ import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse; +import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaInfoWithVersion; import org.apache.pulsar.common.schema.SchemaType; @@ -438,4 +440,31 @@ public void testGetSchemaCompatibilityStrategyWhenSetBrokerLevelAndSchemaAutoUpd admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace), SchemaCompatibilityStrategy.UNDEFINED)); } + + @Test + public void testCompatibility() throws Exception { + String topicName = schemaCompatibilityNamespace + "/testCompatibility"; + try { + admin.schemas().getSchemaInfo(topicName); + fail(); + } catch (PulsarAdminException.NotFoundException e) { + assertEquals(e.getMessage(), "Schema not found"); + } + Map properties = new HashMap<>(); + PostSchemaPayload postSchemaPayload = new PostSchemaPayload("STRING", "", properties); + admin.schemas().createSchema(topicName, postSchemaPayload); + IsCompatibilityResponse isCompatibilityResponse = + admin.schemas().testCompatibility(topicName, postSchemaPayload); + + assertTrue(isCompatibilityResponse.isCompatibility()); + assertEquals(isCompatibilityResponse.getSchemaCompatibilityStrategy(), SchemaCompatibilityStrategy.FULL.name()); + postSchemaPayload = new PostSchemaPayload("INT8", "", properties); + try { + admin.schemas().testCompatibility(topicName, postSchemaPayload); + fail(); + } catch (Exception e) { + assertTrue(e instanceof PulsarAdminException.ServerSideErrorException); + assertTrue(e.getMessage().contains("Incompatible schema: exists schema type STRING, new schema type INT8")); + } + } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java index d5edd96fab563..aa8b54d1a77be 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java @@ -33,5 +33,4 @@ public class IsCompatibilityResponse { boolean isCompatibility; String schemaCompatibilityStrategy; - } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 2fe7b0236f0e8..a1d5d695c6398 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -2360,6 +2360,11 @@ void schemas() throws Exception { PostSchemaPayload input = new ObjectMapper().readValue(new File(schemaFile), PostSchemaPayload.class); verify(schemas).createSchema("persistent://tn1/ns1/tp1", input); + cmdSchemas = new CmdSchemas(() -> admin); + cmdSchemas.run(split("compatibility -f " + schemaFile + " persistent://tn1/ns1/tp1")); + input = new ObjectMapper().readValue(new File(schemaFile), PostSchemaPayload.class); + verify(schemas).testCompatibility("persistent://tn1/ns1/tp1", input); + cmdSchemas = new CmdSchemas(() -> admin); String jarFile = PulsarAdminToolTest.class.getClassLoader() .getResource("dummyexamples.jar").getFile(); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java index 5383e39807b05..44ac143e3507e 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java @@ -42,6 +42,7 @@ public CmdSchemas(Supplier admin) { jcommander.addCommand("delete", new DeleteSchema()); jcommander.addCommand("upload", new UploadSchema()); jcommander.addCommand("extract", new ExtractSchema()); + jcommander.addCommand("compatibility", new TestCompatibility()); } @Parameters(commandDescription = "Get the schema for a topic") @@ -164,4 +165,20 @@ void run() throws Exception { } } + @Parameters(commandDescription = "Test schema compatibility") + private class TestCompatibility extends CliCommand { + @Parameter(description = "persistent://tenant/namespace/topic", required = true) + private java.util.List params; + + @Parameter(names = { "-f", "--filename" }, description = "filename", required = true) + private String schemaFileName; + + @Override + void run() throws Exception { + String topic = validateTopicName(params); + PostSchemaPayload input = MAPPER.readValue(new File(schemaFileName), PostSchemaPayload.class); + getAdmin().schemas().testCompatibility(topic, input); + } + } + } From e5a833a2dcb7ce13ada4ca94714cc045a02de276 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 21 Apr 2023 16:06:22 +0800 Subject: [PATCH 139/280] [improve][broker] Move bitmap from lastMutableBucket to ImmutableBucket (#20156) --- .../bucket/BucketDelayedDeliveryTracker.java | 3 +- .../broker/delayed/bucket/MutableBucket.java | 41 +++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index c90064c9137bb..67a7de1f01339 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -168,7 +168,7 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException { } try { - FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 2, TimeUnit.SECONDS); + FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 5, TimeUnit.SECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("[{}] Failed to recover delayed message index bucket snapshot.", dispatcher.getName(), e); if (e instanceof InterruptedException) { @@ -343,6 +343,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver // If (ledgerId < startLedgerId || existBucket) means that message index belong to previous bucket range, // enter sharedBucketPriorityQueue directly sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); + lastMutableBucket.putIndexBit(ledgerId, entryId); } else { checkArgument(ledgerId >= lastMutableBucket.endLedgerId); lastMutableBucket.addMessage(ledgerId, entryId, deliverAt); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index b7e9e68f1bdc7..1173a401a8903 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import static com.google.common.base.Preconditions.checkArgument; -import com.google.protobuf.ByteString; +import com.google.protobuf.UnsafeByteOperations; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -74,6 +74,8 @@ Pair createImmutableBucketAndAsyncPersistent( List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); + Map immutableBucketBitMap = new HashMap<>(); + Map bitMap = new HashMap<>(); SnapshotSegment snapshotSegment = new SnapshotSegment(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); @@ -82,18 +84,20 @@ Pair createImmutableBucketAndAsyncPersistent( long currentTimestampUpperLimit = 0; long currentFirstTimestamp = 0L; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex delayedIndex = snapshotSegment.addIndexe(); - delayedIndexQueue.popToObject(delayedIndex); - - long timestamp = delayedIndex.getTimestamp(); + final long timestamp = delayedIndexQueue.peekTimestamp(); if (currentTimestampUpperLimit == 0) { currentFirstTimestamp = timestamp; firstScheduleTimestamps.add(currentFirstTimestamp); currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1; } - long ledgerId = delayedIndex.getLedgerId(); - long entryId = delayedIndex.getEntryId(); + DelayedIndex delayedIndex = snapshotSegment.addIndexe(); + delayedIndexQueue.popToObject(delayedIndex); + + final long ledgerId = delayedIndex.getLedgerId(); + final long entryId = delayedIndex.getEntryId(); + + removeIndexBit(ledgerId, entryId); checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); @@ -102,10 +106,10 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } - numMessages++; - bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); + numMessages++; + if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit || (maxIndexesPerBucketSnapshotSegment != -1 && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { @@ -119,9 +123,17 @@ Pair createImmutableBucketAndAsyncPersistent( final var lId = entry.getKey(); final var bm = entry.getValue(); bm.runOptimize(); - final var array = new byte[bm.serializedSizeInBytes()]; - bm.serialize(ByteBuffer.wrap(array)); - segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array)); + ByteBuffer byteBuffer = ByteBuffer.allocate(bm.serializedSizeInBytes()); + bm.serialize(byteBuffer); + byteBuffer.flip(); + segmentMetadataBuilder.putDelayedIndexBitMap(lId, UnsafeByteOperations.unsafeWrap(byteBuffer)); + immutableBucketBitMap.compute(lId, (__, bm0) -> { + if (bm0 == null) { + return bm; + } + bm0.or(bm); + return bm0; + }); iterator.remove(); } @@ -133,6 +145,10 @@ Pair createImmutableBucketAndAsyncPersistent( } } + // optimize bm + immutableBucketBitMap.values().forEach(RoaringBitmap::runOptimize); + this.delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); + SnapshotMetadata bucketSnapshotMetadata = SnapshotMetadata.newBuilder() .addAllMetadataList(segmentMetadataList) .build(); @@ -145,6 +161,7 @@ Pair createImmutableBucketAndAsyncPersistent( bucket.setNumberBucketDelayedMessages(numMessages); bucket.setLastSegmentEntryId(lastSegmentEntryId); bucket.setFirstScheduleTimestamps(firstScheduleTimestamps); + bucket.setDelayedIndexBitMap(immutableBucketBitMap); // Skip first segment, because it has already been loaded List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size()); From 6036dcce8d48f8e1ae311153eeb625f51c5aa827 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Sat, 22 Apr 2023 10:58:05 +0800 Subject: [PATCH 140/280] [fix][client] Fix breaking changes for the deprecated methods of TopicMessageIdImpl (#20163) --- .../apache/pulsar/client/impl/TopicMessageIdImpl.java | 2 +- .../pulsar/client/impl/TopicMessageIdImplTest.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java index 00fe12b62b194..3dc9b23e93e86 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java @@ -62,7 +62,7 @@ public String getTopicPartitionName() { @Deprecated public MessageId getInnerMessageId() { - return new MessageIdImpl(getLedgerId(), getEntryId(), getPartitionIndex()); + return msgId; } @Override diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java index daf49f0e7752e..1ddd47af91dff 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertSame; import org.testng.annotations.Test; @@ -54,4 +55,12 @@ public void equalsTest() { assertNotEquals(topicMsgId1, topicMsgId2); } + @Test + public void testDeprecatedMethods() { + BatchMessageIdImpl msgId = new BatchMessageIdImpl(1, 2, 3, 4); + TopicMessageIdImpl topicMsgId = new TopicMessageIdImpl("topic-partition-0", "topic", msgId); + assertSame(topicMsgId.getInnerMessageId(), msgId); + assertEquals(topicMsgId.getTopicPartitionName(), topicMsgId.getOwnerTopic()); + assertEquals(topicMsgId.getTopicName(), "topic"); + } } From c58da14968f27cf2a4813db01f387b61a5a03636 Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Sat, 22 Apr 2023 21:22:26 +0800 Subject: [PATCH 141/280] [fix][test] fix bug for testcase(PersistentTopicsTest#testExamineMessage) (#20154) Co-authored-by: lushiji --- .../org/apache/pulsar/broker/admin/PersistentTopicsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 6949fe931ca5a..f80d9863a26a6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -1144,6 +1144,7 @@ public void testExamineMessage() throws Exception { // Check examine message not allowed on partitioned topic. try { admin.topics().examineMessage(topicName, "earliest", 1); + Assert.fail("fail to check examine message not allowed on partitioned topic"); } catch (PulsarAdminException e) { Assert.assertEquals(e.getMessage(), "Examine messages on a partitioned topic is not allowed, please try examine message on specific " From 4190e40ab4ac448a2f94edd1c621dc1c67d9ee4b Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 22 Apr 2023 13:28:53 -0700 Subject: [PATCH 142/280] [improve] Allow to build and push multi-arch Docker images (#19432) Co-authored-by: Lari Hotari Co-authored-by: Yong Zhang Co-authored-by: Zixuan Liu Co-authored-by: tison --- .github/workflows/pulsar-ci.yaml | 1 + build/build_java_test_image.sh | 1 + build/pulsar_ci_tool.sh | 3 +- docker/pulsar-all/pom.xml | 42 ++++++++++++--- docker/pulsar/pom.xml | 51 +++++++++++++++---- pom.xml | 8 ++- .../latest-version-image/pom.xml | 6 +++ 7 files changed, 94 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f7dbc755264d6..4d4f2902d8f27 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -798,6 +798,7 @@ jobs: # build docker image # include building of Pulsar SQL, Connectors, Offloaders and server distros mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ + -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh index 0747e6dacb82a..459bf26f98eff 100755 --- a/build/build_java_test_image.sh +++ b/build/build_java_test_image.sh @@ -27,5 +27,6 @@ if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]] SQUASH_PARAM="-Ddocker.squash=true" fi mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \ + -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \ "$@" install \ No newline at end of file diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh index 61199eda2c5d8..d946edd395789 100755 --- a/build/pulsar_ci_tool.sh +++ b/build/pulsar_ci_tool.sh @@ -46,7 +46,8 @@ function ci_print_thread_dumps() { # runs maven function _ci_mvn() { - mvn -B -ntp "$@" + mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ + "$@" } # runs OWASP Dependency Check for all projects diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index c63ff6d656957..2edd6c776dcf5 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -68,10 +68,6 @@ docker - - target/apache-pulsar-io-connectors-${project.version}-bin - target/pulsar-offloader-distribution-${project.version}-bin.tar.gz - @@ -143,17 +139,25 @@ - pulsar-all + ${docker.organization}/pulsar-all ${project.basedir} latest + ${project.version} + + target/apache-pulsar-io-connectors-${project.version}-bin + target/pulsar-offloader-distribution-${project.version}-bin.tar.gz + + + + ${docker.platforms} + + - latest - ${docker.organization} @@ -161,5 +165,29 @@ + + + docker-push + + + + io.fabric8 + docker-maven-plugin + + + default + package + + build + tag + push + + + + + + + + diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 647f68bf1672c..8c3b868315535 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -47,15 +47,14 @@ + + mirror://mirrors.ubuntu.com/mirrors.txt + http://security.ubuntu.com/ubuntu/ + + docker - - target/pulsar-server-distribution-${project.version}-bin.tar.gz - ${pulsar.client.python.version} - ${env.UBUNTU_MIRROR} - ${env.UBUNTU_SECURITY_MIRROR} - @@ -72,17 +71,27 @@ - pulsar + ${docker.organization}/pulsar + + target/pulsar-server-distribution-${project.version}-bin.tar.gz + ${pulsar.client.python.version} + ${UBUNTU_MIRROR} + ${UBUNTU_SECURITY_MIRROR} + ${project.basedir} latest + ${project.version} + + + ${docker.platforms} + + - latest - ${docker.organization} @@ -108,5 +117,29 @@ + + + docker-push + + + + io.fabric8 + docker-maven-plugin + + + default + package + + build + tag + push + + + + + + + + diff --git a/pom.xml b/pom.xml index 22589a1719812..aef380c5cd09c 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,12 @@ flexible messaging model and an intuitive client API. UTF-8 ${maven.build.timestamp} true + + + + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED @@ -152,7 +158,7 @@ flexible messaging model and an intuitive client API. 0.10.2 1.6.2 8.37 - 0.40.2 + 0.42.1 true 0.5.0 3.19.6 diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 8474f820c9566..c73a6f8a83970 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -147,6 +147,7 @@ package build + tag @@ -159,6 +160,11 @@ ${project.version} true + + + ${docker.platforms} + + From 0c50866fbc18525f82a04c3a918628b8b50a4de8 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 23 Apr 2023 09:41:28 +0800 Subject: [PATCH 143/280] [fix] [broker] Fast fix infinite HTTP call getSubscriptions caused by wrong topicName (#20131) --- .../admin/impl/PersistentTopicsBase.java | 32 +++- ...rInfiniteHttpCallGetSubscriptionsTest.java | 143 ++++++++++++++++++ 2 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 7347d6dbf20a2..6b163ae04469b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; +import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectReader; import com.github.zafarkhaja.semver.Version; @@ -1171,6 +1172,21 @@ protected CompletableFuture internalDeleteTopicAsync(boolean authoritative .thenCompose(__ -> pulsar().getBrokerService().deleteTopic(topicName.toString(), force)); } + /** + * There has a known bug will make Pulsar misidentifies "tp-partition-0-DLQ-partition-0" as "tp-partition-0-DLQ". + * You can see the details from PR https://github.com/apache/pulsar/pull/19841. + * This method is a quick fix and will be removed in master branch after #19841 and PIP 263 are done. + */ + private boolean isUnexpectedTopicName(PartitionedTopicMetadata topicMetadata) { + if (!topicName.toString().contains(PARTITIONED_TOPIC_SUFFIX)){ + return false; + } + if (topicMetadata.partitions <= 0){ + return false; + } + return topicName.getPartition(0).toString().equals(topicName.toString()); + } + protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean authoritative) { CompletableFuture future; if (topicName.isGlobal()) { @@ -1188,7 +1204,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut } else { getPartitionedTopicMetadataAsync(topicName, authoritative, false) .thenAccept(partitionMetadata -> { - if (partitionMetadata.partitions > 0) { + if (partitionMetadata.partitions > 0 && !isUnexpectedTopicName(partitionMetadata)) { try { final Set subscriptions = Collections.newSetFromMap( @@ -3716,7 +3732,7 @@ protected CompletableFuture preValidation(boolean authoritative) { .thenCompose(metadata -> { if (metadata.partitions > 0) { return validateTopicOwnershipAsync(TopicName.get(topicName.toString() - + TopicName.PARTITIONED_TOPIC_SUFFIX + 0), authoritative); + + PARTITIONED_TOPIC_SUFFIX + 0), authoritative); } else { return validateTopicOwnershipAsync(topicName, authoritative); } @@ -4543,7 +4559,7 @@ protected CompletableFuture internalValidateClientVersionAsync() { private CompletableFuture validatePartitionTopicUpdateAsync(String topicName, int numberOfPartition) { return internalGetListAsync().thenCompose(existingTopicList -> { TopicName partitionTopicName = TopicName.get(domain(), namespaceName, topicName); - String prefix = partitionTopicName.getPartitionedTopicName() + TopicName.PARTITIONED_TOPIC_SUFFIX; + String prefix = partitionTopicName.getPartitionedTopicName() + PARTITIONED_TOPIC_SUFFIX; return getPartitionedTopicMetadataAsync(partitionTopicName, false, false) .thenAccept(metadata -> { int oldPartition = metadata.partitions; @@ -4551,8 +4567,8 @@ private CompletableFuture validatePartitionTopicUpdateAsync(String topicNa if (existingTopicName.startsWith(prefix)) { try { long suffix = Long.parseLong(existingTopicName.substring( - existingTopicName.indexOf(TopicName.PARTITIONED_TOPIC_SUFFIX) - + TopicName.PARTITIONED_TOPIC_SUFFIX.length())); + existingTopicName.indexOf(PARTITIONED_TOPIC_SUFFIX) + + PARTITIONED_TOPIC_SUFFIX.length())); // Skip partition of partitioned topic by making sure // the numeric suffix greater than old partition number. if (suffix >= oldPartition && suffix <= (long) numberOfPartition) { @@ -4593,12 +4609,12 @@ private CompletableFuture validatePartitionTopicUpdateAsync(String topicNa */ private CompletableFuture validateNonPartitionTopicNameAsync(String topicName) { CompletableFuture ret = CompletableFuture.completedFuture(null); - if (topicName.contains(TopicName.PARTITIONED_TOPIC_SUFFIX)) { + if (topicName.contains(PARTITIONED_TOPIC_SUFFIX)) { try { // First check if what's after suffix "-partition-" is number or not, if not number then can create. - int partitionIndex = topicName.indexOf(TopicName.PARTITIONED_TOPIC_SUFFIX); + int partitionIndex = topicName.indexOf(PARTITIONED_TOPIC_SUFFIX); long suffix = Long.parseLong(topicName.substring(partitionIndex - + TopicName.PARTITIONED_TOPIC_SUFFIX.length())); + + PARTITIONED_TOPIC_SUFFIX.length())); TopicName partitionTopicName = TopicName.get(domain(), namespaceName, topicName.substring(0, partitionIndex)); ret = getPartitionedTopicMetadataAsync(partitionTopicName, false, false) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java new file mode 100644 index 0000000000000..2efc4f4e7803f --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicType; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class TopicNameForInfiniteHttpCallGetSubscriptionsTest extends ProducerConsumerBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED); + conf.setDefaultNumPartitions(1); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testInfiniteHttpCallGetSubscriptions() throws Exception { + final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); + final String partitionedTopicName = "persistent://my-property/my-ns/tp1_" + randomStr; + final String topic_p0 = partitionedTopicName + TopicName.PARTITIONED_TOPIC_SUFFIX + "0"; + final String subscriptionName = "sub1"; + final String topicDLQ = topic_p0 + "-" + subscriptionName + "-DLQ"; + + admin.topics().createPartitionedTopic(partitionedTopicName, 2); + + // Do test. + ProducerAndConsumerEntry pcEntry = triggerDLQCreated(topic_p0, topicDLQ, subscriptionName); + admin.topics().getSubscriptions(topicDLQ); + + // cleanup. + pcEntry.consumer.close(); + pcEntry.producer.close(); + admin.topics().deletePartitionedTopic(partitionedTopicName); + } + + @Test + public void testInfiniteHttpCallGetSubscriptions2() throws Exception { + final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); + final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0-abc"; + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + + // Do test. + admin.topics().getSubscriptions(topicName); + + // cleanup. + producer.close(); + } + + @Test + public void testInfiniteHttpCallGetSubscriptions3() throws Exception { + final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); + final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0"; + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + + // Do test. + admin.topics().getSubscriptions(topicName); + + // cleanup. + producer.close(); + } + + @AllArgsConstructor + private static class ProducerAndConsumerEntry { + private Producer producer; + private Consumer consumer; + } + + private ProducerAndConsumerEntry triggerDLQCreated(String topicName, String DLQName, String subscriptionName) throws Exception { + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .enableRetry(true) + .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic(DLQName).maxRedeliverCount(2).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + // send messages. + for (int i = 0; i < 5; i++) { + producer.newMessage() + .value("value-" + i) + .sendAsync(); + } + producer.flush(); + // trigger the DLQ created. + for (int i = 0; i < 20; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + if (msg != null) { + consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS); + } else { + break; + } + } + + return new ProducerAndConsumerEntry(producer, consumer); + } +} \ No newline at end of file From d3158bfebf9565ea9422a9116749628648e9f90d Mon Sep 17 00:00:00 2001 From: WangJialing <65590138+wangjialing218@users.noreply.github.com> Date: Sun, 23 Apr 2023 10:07:02 +0800 Subject: [PATCH 144/280] [fix][broker] Fix getPartitionedStats miss subscription's messageAckRate (#19870) --- .../common/policies/data/stats/SubscriptionStatsImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index 25fa666523f3c..7ae439829b319 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -160,6 +160,8 @@ public void reset() { bytesOutCounter = 0; msgOutCounter = 0; msgRateRedeliver = 0; + messageAckRate = 0; + chunkedMessageRate = 0; msgBacklog = 0; backlogSize = 0; msgBacklogNoDelayed = 0; @@ -190,6 +192,8 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { this.bytesOutCounter += stats.bytesOutCounter; this.msgOutCounter += stats.msgOutCounter; this.msgRateRedeliver += stats.msgRateRedeliver; + this.messageAckRate += stats.messageAckRate; + this.chunkedMessageRate += stats.chunkedMessageRate; this.msgBacklog += stats.msgBacklog; this.backlogSize += stats.backlogSize; this.msgBacklogNoDelayed += stats.msgBacklogNoDelayed; From 963260abfa142b8cf9ffe372d85d470a78c17235 Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Sun, 23 Apr 2023 11:27:48 +0800 Subject: [PATCH 145/280] [fix] [broker] Make `LeastResourceUsageWithWeight` thread safe (#20159) Fix #20160 ### Motivation LeastResourceUsageWithWeight is an stateful object and current code will be accessed by multithread. thread 1: is execute https://github.com/apache/pulsar/blob/2b41e4eafb1cba0e548dd90df60e8cdbb24cd490/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java#L89-L91 and thread 2: is execute https://github.com/apache/pulsar/blob/2b41e4eafb1cba0e548dd90df60e8cdbb24cd490/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java#L147-L150 so an IndexOutOfBound occurs. ### Modifications change the state to `ThreadLocal` --- .../strategy/LeastResourceUsageWithWeight.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java index 902cfdaf73f5f..98986d84b9858 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java @@ -23,6 +23,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; +import javax.annotation.concurrent.ThreadSafe; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; @@ -35,14 +36,15 @@ * cause cluster fluctuations due to short-term load jitter. */ @Slf4j +@ThreadSafe public class LeastResourceUsageWithWeight implements BrokerSelectionStrategy { // Maintain this list to reduce object creation. - private final ArrayList bestBrokers; - private final Set noLoadDataBrokers; + private final ThreadLocal> bestBrokers; + private final ThreadLocal> noLoadDataBrokers; public LeastResourceUsageWithWeight() { - this.bestBrokers = new ArrayList<>(); - this.noLoadDataBrokers = new HashSet<>(); + this.bestBrokers = ThreadLocal.withInitial(ArrayList::new); + this.noLoadDataBrokers = ThreadLocal.withInitial(HashSet::new); } // A broker's max resource usage with weight using its historical load and short-term load data with weight. @@ -70,7 +72,6 @@ private double getMaxResourceUsageWithWeight(final String broker, final BrokerLo /** * Find a suitable broker to assign the given bundle to. - * This method is not thread safety. * * @param candidates The candidates for which the bundle may be assigned. * @param bundleToAssign The data for the bundle to assign. @@ -86,6 +87,9 @@ public Optional select( return Optional.empty(); } + ArrayList bestBrokers = this.bestBrokers.get(); + HashSet noLoadDataBrokers = this.noLoadDataBrokers.get(); + bestBrokers.clear(); noLoadDataBrokers.clear(); // Maintain of list of all the best scoring brokers and then randomly @@ -135,9 +139,7 @@ public Optional select( log.info("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, " + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size()); } - for (String broker : candidates) { - bestBrokers.add(broker); - } + bestBrokers.addAll(candidates); } if (debugMode) { From 6e585acd7d788c0979c113a92e341364c987cbe4 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 23 Apr 2023 13:42:10 +0800 Subject: [PATCH 146/280] [fix] [broker] Fix wrong logic of method TopicName.getPartition(int index) (#19841) --- .../PartitionKeywordCompatibilityTest.java | 3 +- ...rInfiniteHttpCallGetSubscriptionsTest.java | 143 ------------------ .../pulsar/common/naming/TopicName.java | 2 +- .../pulsar/common/naming/TopicNameTest.java | 8 + 4 files changed, 11 insertions(+), 145 deletions(-) delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java index fdf2eb29c5e9e..86a5fcdc05aec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java @@ -26,6 +26,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -66,7 +67,7 @@ public void testAutoCreatePartitionTopicWithKeywordAndDeleteIt() .subscribe(); List topics = admin.topics().getList("public/default"); List partitionedTopicList = admin.topics().getPartitionedTopicList("public/default"); - Assert.assertTrue(topics.contains(topicName)); + Assert.assertTrue(topics.contains(TopicName.get(topicName).getPartition(0).toString())); Assert.assertTrue(partitionedTopicList.contains(topicName)); consumer.close(); admin.topics().deletePartitionedTopic(topicName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java deleted file mode 100644 index 2efc4f4e7803f..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.api; - -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.TopicType; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -@Slf4j -@Test(groups = "broker") -public class TopicNameForInfiniteHttpCallGetSubscriptionsTest extends ProducerConsumerBase { - - @BeforeMethod(alwaysRun = true) - @Override - protected void setup() throws Exception { - super.internalSetup(); - super.producerBaseSetup(); - } - - @Override - protected void doInitConf() throws Exception { - super.doInitConf(); - conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED); - conf.setDefaultNumPartitions(1); - } - - @AfterMethod(alwaysRun = true) - @Override - protected void cleanup() throws Exception { - super.internalCleanup(); - } - - @Test - public void testInfiniteHttpCallGetSubscriptions() throws Exception { - final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); - final String partitionedTopicName = "persistent://my-property/my-ns/tp1_" + randomStr; - final String topic_p0 = partitionedTopicName + TopicName.PARTITIONED_TOPIC_SUFFIX + "0"; - final String subscriptionName = "sub1"; - final String topicDLQ = topic_p0 + "-" + subscriptionName + "-DLQ"; - - admin.topics().createPartitionedTopic(partitionedTopicName, 2); - - // Do test. - ProducerAndConsumerEntry pcEntry = triggerDLQCreated(topic_p0, topicDLQ, subscriptionName); - admin.topics().getSubscriptions(topicDLQ); - - // cleanup. - pcEntry.consumer.close(); - pcEntry.producer.close(); - admin.topics().deletePartitionedTopic(partitionedTopicName); - } - - @Test - public void testInfiniteHttpCallGetSubscriptions2() throws Exception { - final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); - final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0-abc"; - Producer producer = pulsarClient.newProducer(Schema.STRING) - .topic(topicName) - .create(); - - // Do test. - admin.topics().getSubscriptions(topicName); - - // cleanup. - producer.close(); - } - - @Test - public void testInfiniteHttpCallGetSubscriptions3() throws Exception { - final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); - final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0"; - Producer producer = pulsarClient.newProducer(Schema.STRING) - .topic(topicName) - .create(); - - // Do test. - admin.topics().getSubscriptions(topicName); - - // cleanup. - producer.close(); - } - - @AllArgsConstructor - private static class ProducerAndConsumerEntry { - private Producer producer; - private Consumer consumer; - } - - private ProducerAndConsumerEntry triggerDLQCreated(String topicName, String DLQName, String subscriptionName) throws Exception { - Consumer consumer = pulsarClient.newConsumer(Schema.STRING) - .topic(topicName) - .subscriptionName(subscriptionName) - .subscriptionType(SubscriptionType.Shared) - .enableRetry(true) - .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic(DLQName).maxRedeliverCount(2).build()) - .receiverQueueSize(100) - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscribe(); - Producer producer = pulsarClient.newProducer(Schema.STRING) - .topic(topicName) - .create(); - // send messages. - for (int i = 0; i < 5; i++) { - producer.newMessage() - .value("value-" + i) - .sendAsync(); - } - producer.flush(); - // trigger the DLQ created. - for (int i = 0; i < 20; i++) { - Message msg = consumer.receive(5, TimeUnit.SECONDS); - if (msg != null) { - consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS); - } else { - break; - } - } - - return new ProducerAndConsumerEntry(producer, consumer); - } -} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index bab9c443ceaeb..79ef64c1ae459 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -231,7 +231,7 @@ public String getEncodedLocalName() { } public TopicName getPartition(int index) { - if (index == -1 || this.toString().contains(PARTITIONED_TOPIC_SUFFIX)) { + if (index == -1 || this.toString().endsWith(PARTITIONED_TOPIC_SUFFIX + index)) { return this; } String partitionName = this.toString() + PARTITIONED_TOPIC_SUFFIX + index; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java index 0e4533e7a08d0..8e32fbe3d33c0 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java @@ -289,4 +289,12 @@ public void testShortTopicName() throws Exception { // Ok } } + + @Test + public void testTwoKeyWordPartition(){ + TopicName tp1 = TopicName.get("tenant1/namespace1/tp1-partition-0-DLQ"); + TopicName tp2 = tp1.getPartition(0); + assertNotEquals(tp2.toString(), tp1.toString()); + assertEquals(tp2.toString(), "persistent://tenant1/namespace1/tp1-partition-0-DLQ-partition-0"); + } } From d3d36bdbd22a1b1fa6cee41a91acd9ec8a4c9341 Mon Sep 17 00:00:00 2001 From: tiny-rain Date: Sun, 23 Apr 2023 13:45:03 +0800 Subject: [PATCH 147/280] [fix][meta] deadlock of zkSessionWatcher when zkConnection loss (#20122) --- .../org/apache/pulsar/metadata/impl/ZKSessionWatcher.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java index 1ce01f57d4fbe..a840721023080 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java @@ -81,7 +81,11 @@ public void close() throws Exception { } // task that runs every TICK_TIME to check zk connection - private synchronized void checkConnectionStatus() { + // NOT ThreadSafe: + // If zk client can't ensure the order, it may lead to problems. + // Currently,we only use it in single thread, it will be fine. but we shouldn't leave any potential problems + // in the future. + private void checkConnectionStatus() { try { CompletableFuture future = new CompletableFuture<>(); zk.exists("/", false, (StatCallback) (rc, path, ctx, stat) -> { @@ -126,7 +130,7 @@ synchronized void setSessionInvalid() { currentStatus = SessionEvent.SessionLost; } - private void checkState(Watcher.Event.KeeperState zkClientState) { + private synchronized void checkState(Watcher.Event.KeeperState zkClientState) { switch (zkClientState) { case Expired: if (currentStatus != SessionEvent.SessionLost) { From 795d60acc7a27a2d6bddfc60ffd7c9cf52a91cb4 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Sun, 23 Apr 2023 16:15:35 +0800 Subject: [PATCH 148/280] [improve][cli] Add `--cleanupSubscription` to pulsar-admin (#20028) Co-authored-by: tison --- .../java/org/apache/pulsar/admin/cli/CmdFunctions.java | 7 +++++++ .../java/org/apache/pulsar/admin/cli/CmdSinks.java | 8 ++++++++ .../apache/pulsar/functions/utils/SinkConfigUtils.java | 10 ---------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 05bab9c6f198b..91ec7183ff13c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -201,6 +201,9 @@ abstract class FunctionDetailsCommand extends BaseCommand { protected String className; @Parameter(names = { "-t", "--function-type" }, description = "The built-in Pulsar Function type") protected String functionType; + @Parameter(names = "--cleanup-subscription", description = "Whether delete the subscription " + + "when function is deleted") + protected Boolean cleanupSubscription; @Parameter(names = "--jar", description = "Path to the JAR file for the function " + "(if the function is written in Java). It also supports URL path [http/https/file " + "(file protocol assumes that file already exists on worker host)/function " @@ -471,6 +474,10 @@ void processArguments() throws Exception { } } + if (null != cleanupSubscription) { + functionConfig.setCleanupSubscription(cleanupSubscription); + } + if (null != inputs) { List inputTopics = Arrays.asList(inputs.split(",")); functionConfig.setInputs(inputTopics); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index 4af9221dd2eab..0b27dd8d0a737 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -291,6 +291,10 @@ abstract class SinkDetailsCommand extends BaseCommand { @Parameter(names = { "-t", "--sink-type" }, description = "The sinks's connector provider") protected String sinkType; + @Parameter(names = "--cleanup-subscription", description = "Whether delete the subscription " + + "when sink is deleted") + protected Boolean cleanupSubscription; + @Parameter(names = { "-i", "--inputs" }, description = "The sink's input topic or topics " + "(multiple topics can be specified as a comma-separated list)") @@ -469,6 +473,10 @@ void processArguments() throws Exception { sinkConfig.setProcessingGuarantees(processingGuarantees); } + if (null != cleanupSubscription) { + sinkConfig.setCleanupSubscription(cleanupSubscription); + } + if (retainOrdering != null) { sinkConfig.setRetainOrdering(retainOrdering); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index d79f787588c95..1a4009f5295e2 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -202,12 +202,6 @@ public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetail sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(sinkConfig.getNegativeAckRedeliveryDelayMs()); } - if (sinkConfig.getCleanupSubscription() != null) { - sourceSpecBuilder.setCleanupSubscription(sinkConfig.getCleanupSubscription()); - } else { - sourceSpecBuilder.setCleanupSubscription(true); - } - if (sinkConfig.getSourceSubscriptionPosition() == SubscriptionInitialPosition.Earliest) { sourceSpecBuilder.setSubscriptionPosition(Function.SubscriptionPosition.EARLIEST); } else { @@ -329,7 +323,6 @@ public static SinkConfig convertFromDetails(FunctionDetails functionDetails) { // Set subscription position sinkConfig.setSourceSubscriptionPosition( convertFromFunctionDetailsSubscriptionPosition(functionDetails.getSource().getSubscriptionPosition())); - sinkConfig.setCleanupSubscription(functionDetails.getSource().getCleanupSubscription()); if (functionDetails.getSource().getTimeoutMs() != 0) { sinkConfig.setTimeoutMs(functionDetails.getSource().getTimeoutMs()); @@ -671,9 +664,6 @@ public static SinkConfig validateUpdate(SinkConfig existingConfig, SinkConfig ne if (!StringUtils.isEmpty(newConfig.getCustomRuntimeOptions())) { mergedConfig.setCustomRuntimeOptions(newConfig.getCustomRuntimeOptions()); } - if (newConfig.getCleanupSubscription() != null) { - mergedConfig.setCleanupSubscription(newConfig.getCleanupSubscription()); - } if (newConfig.getTransformFunction() != null) { mergedConfig.setTransformFunction(newConfig.getTransformFunction()); } From ab8a8c9ef58c0d3b7c0838df81f9762637c24f90 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 24 Apr 2023 00:26:23 +0800 Subject: [PATCH 149/280] [fix][broker] Remove useless field in the DelayedMessageIndexBucketSegment.proto (#20166) --- .../src/main/proto/DelayedMessageIndexBucketSegment.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto index 633d6a8f1615c..a6ed30cfe8cd4 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto @@ -31,5 +31,4 @@ message DelayedIndex { message SnapshotSegment { repeated DelayedIndex indexes = 1; - map delayed_index_bit_map = 2; } From 35e9897742b7db4bd29349940075a819b2ad6999 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 24 Apr 2023 10:34:20 +0800 Subject: [PATCH 150/280] [fix][broker] Release EntryBuffer after parse proto object (#20170) --- .../BookkeeperBucketSnapshotStorage.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 040bbbc586f49..e99f39b382f56 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -20,6 +20,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -38,6 +39,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -60,8 +62,9 @@ public BookkeeperBucketSnapshotStorage(PulsarService pulsar) { public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey, String topicName, String cursorName) { + ByteBuf metadataByteBuf = Unpooled.wrappedBuffer(snapshotMetadata.toByteArray()); return createLedger(bucketKey, topicName, cursorName) - .thenCompose(ledgerHandle -> addEntry(ledgerHandle, snapshotMetadata.toByteArray()) + .thenCompose(ledgerHandle -> addEntry(ledgerHandle, metadataByteBuf) .thenCompose(__ -> addSnapshotSegments(ledgerHandle, bucketSnapshotSegments)) .thenCompose(__ -> closeLedger(ledgerHandle)) .thenApply(__ -> ledgerHandle.getId())); @@ -117,19 +120,32 @@ public void close() throws Exception { private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle, List bucketSnapshotSegments) { List> addFutures = new ArrayList<>(); + ByteBuf byteBuf; for (SnapshotSegment bucketSnapshotSegment : bucketSnapshotSegments) { - addFutures.add(addEntry(ledgerHandle, bucketSnapshotSegment.toByteArray())); + byteBuf = PulsarByteBufAllocator.DEFAULT.directBuffer(bucketSnapshotSegment.getSerializedSize()); + try { + bucketSnapshotSegment.writeTo(byteBuf); + } catch (Exception e){ + byteBuf.release(); + throw e; + } + addFutures.add(addEntry(ledgerHandle, byteBuf)); } return FutureUtil.waitForAll(addFutures); } private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { + ByteBuf entryBuffer = null; try { - ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + entryBuffer = ledgerEntry.getEntryBuffer(); return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer()); } catch (InvalidProtocolBufferException e) { throw new BucketSnapshotSerializationException(e); + } finally { + if (entryBuffer != null) { + entryBuffer.release(); + } } } @@ -139,7 +155,11 @@ private List parseSnapshotSegmentEntries(Enumeration closeLedger(LedgerHandle ledgerHandle) { return future; } - private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data) { + private CompletableFuture addEntry(LedgerHandle ledgerHandle, ByteBuf data) { final CompletableFuture future = new CompletableFuture<>(); ledgerHandle.asyncAddEntry(data, (rc, handle, entryId, ctx) -> { From fd60e9e8380af1c7680999cbb5bff8160ba3571a Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 25 Apr 2023 17:22:41 +0800 Subject: [PATCH 151/280] [improve][build] Upgrade jackson version to 2.15.0 for CVE-2022-1471 (#20177) --- .../server/src/assemble/LICENSE.bin.txt | 22 +++++++-------- .../shell/src/assemble/LICENSE.bin.txt | 22 +++++++-------- pom.xml | 2 +- .../pulsar/common/util/FieldParser.java | 7 ++--- pulsar-sql/presto-distribution/LICENSE | 28 +++++++++---------- 5 files changed, 39 insertions(+), 42 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 26651246ddf48..312c79dee1ab6 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -246,17 +246,17 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar - - com.fasterxml.jackson.core-jackson-core-2.14.2.jar - - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar + - com.fasterxml.jackson.core-jackson-annotations-2.15.0.jar + - com.fasterxml.jackson.core-jackson-core-2.15.0.jar + - com.fasterxml.jackson.core-jackson-databind-2.15.0.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.15.0.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.15.0.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.15.0.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.15.0.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.15.0.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.15.0.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.15.0.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.15.0.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 711890809f1bf..11ef4f3d4e745 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -311,17 +311,17 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * JCommander -- jcommander-1.82.jar * Jackson - - jackson-annotations-2.14.2.jar - - jackson-core-2.14.2.jar - - jackson-databind-2.14.2.jar - - jackson-dataformat-yaml-2.14.2.jar - - jackson-jaxrs-base-2.14.2.jar - - jackson-jaxrs-json-provider-2.14.2.jar - - jackson-module-jaxb-annotations-2.14.2.jar - - jackson-module-jsonSchema-2.14.2.jar - - jackson-datatype-jdk8-2.14.2.jar - - jackson-datatype-jsr310-2.14.2.jar - - jackson-module-parameter-names-2.14.2.jar + - jackson-annotations-2.15.0.jar + - jackson-core-2.15.0.jar + - jackson-databind-2.15.0.jar + - jackson-dataformat-yaml-2.15.0.jar + - jackson-jaxrs-base-2.15.0.jar + - jackson-jaxrs-json-provider-2.15.0.jar + - jackson-module-jaxb-annotations-2.15.0.jar + - jackson-module-jsonSchema-2.15.0.jar + - jackson-datatype-jdk8-2.15.0.jar + - jackson-datatype-jsr310-2.15.0.jar + - jackson-module-parameter-names-2.15.0.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson - gson-2.8.9.jar diff --git a/pom.xml b/pom.xml index aef380c5cd09c..28155b20cb4f5 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ flexible messaging model and an intuitive client API. 1.69 1.0.6 1.0.2.3 - 2.14.2 + 2.15.0 0.10.2 1.6.2 8.37 diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java index 626a14b92eedd..c1c17419abbcb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java @@ -21,8 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.databind.AnnotationIntrospector; -import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; import com.fasterxml.jackson.databind.util.EnumResolver; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -58,8 +56,6 @@ public final class FieldParser { private static final Map CONVERTERS = new HashMap<>(); private static final Map, Class> WRAPPER_TYPES = new HashMap<>(); - private static final AnnotationIntrospector ANNOTATION_INTROSPECTOR = new JacksonAnnotationIntrospector(); - static { // Preload converters and wrapperTypes. initConverters(); @@ -100,7 +96,8 @@ public static T convert(Object from, Class to) { if (to.isEnum()) { // Converting string to enum - EnumResolver r = EnumResolver.constructUsingToString((Class>) to, ANNOTATION_INTROSPECTOR); + EnumResolver r = EnumResolver.constructUsingToString( + ObjectMapperFactory.getMapper().getObjectMapper().getDeserializationConfig(), to); T value = (T) r.findEnum((String) from); if (value == null) { throw new RuntimeException("Invalid value '" + from + "' for enum " + to); diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 09d1396b70419..407cd9a0eb9e4 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.14.2.jar - - jackson-core-2.14.2.jar - - jackson-databind-2.14.2.jar - - jackson-dataformat-smile-2.14.2.jar - - jackson-datatype-guava-2.14.2.jar - - jackson-datatype-jdk8-2.14.2.jar - - jackson-datatype-joda-2.14.2.jar - - jackson-datatype-jsr310-2.14.2.jar - - jackson-dataformat-yaml-2.14.2.jar - - jackson-jaxrs-base-2.14.2.jar - - jackson-jaxrs-json-provider-2.14.2.jar - - jackson-module-jaxb-annotations-2.14.2.jar - - jackson-module-jsonSchema-2.14.2.jar + - jackson-annotations-2.15.0.jar + - jackson-core-2.15.0.jar + - jackson-databind-2.15.0.jar + - jackson-dataformat-smile-2.15.0.jar + - jackson-datatype-guava-2.15.0.jar + - jackson-datatype-jdk8-2.15.0.jar + - jackson-datatype-joda-2.15.0.jar + - jackson-datatype-jsr310-2.15.0.jar + - jackson-dataformat-yaml-2.15.0.jar + - jackson-jaxrs-base-2.15.0.jar + - jackson-jaxrs-json-provider-2.15.0.jar + - jackson-module-jaxb-annotations-2.15.0.jar + - jackson-module-jsonSchema-2.15.0.jar * Guava - guava-31.0.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -456,7 +456,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.8.4.jar * Jackson - - jackson-module-parameter-names-2.14.2.jar + - jackson-module-parameter-names-2.15.0.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access From 7bd9998c9974ff17b1fbd5b3db7cf5bd7d3bc518 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 26 Apr 2023 10:49:30 +0800 Subject: [PATCH 152/280] [improve][build] Revert "Upgrade jackson version to 2.15.0 for CVE-2022-1471 (#20177)" (#20182) --- .../server/src/assemble/LICENSE.bin.txt | 22 +++++++-------- .../shell/src/assemble/LICENSE.bin.txt | 22 +++++++-------- pom.xml | 2 +- .../pulsar/common/util/FieldParser.java | 7 +++-- pulsar-sql/presto-distribution/LICENSE | 28 +++++++++---------- 5 files changed, 42 insertions(+), 39 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 312c79dee1ab6..26651246ddf48 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -246,17 +246,17 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.15.0.jar - - com.fasterxml.jackson.core-jackson-core-2.15.0.jar - - com.fasterxml.jackson.core-jackson-databind-2.15.0.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.15.0.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.15.0.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.15.0.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.15.0.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.15.0.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.15.0.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.15.0.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.15.0.jar + - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar + - com.fasterxml.jackson.core-jackson-core-2.14.2.jar + - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 11ef4f3d4e745..711890809f1bf 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -311,17 +311,17 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * JCommander -- jcommander-1.82.jar * Jackson - - jackson-annotations-2.15.0.jar - - jackson-core-2.15.0.jar - - jackson-databind-2.15.0.jar - - jackson-dataformat-yaml-2.15.0.jar - - jackson-jaxrs-base-2.15.0.jar - - jackson-jaxrs-json-provider-2.15.0.jar - - jackson-module-jaxb-annotations-2.15.0.jar - - jackson-module-jsonSchema-2.15.0.jar - - jackson-datatype-jdk8-2.15.0.jar - - jackson-datatype-jsr310-2.15.0.jar - - jackson-module-parameter-names-2.15.0.jar + - jackson-annotations-2.14.2.jar + - jackson-core-2.14.2.jar + - jackson-databind-2.14.2.jar + - jackson-dataformat-yaml-2.14.2.jar + - jackson-jaxrs-base-2.14.2.jar + - jackson-jaxrs-json-provider-2.14.2.jar + - jackson-module-jaxb-annotations-2.14.2.jar + - jackson-module-jsonSchema-2.14.2.jar + - jackson-datatype-jdk8-2.14.2.jar + - jackson-datatype-jsr310-2.14.2.jar + - jackson-module-parameter-names-2.14.2.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson - gson-2.8.9.jar diff --git a/pom.xml b/pom.xml index 28155b20cb4f5..aef380c5cd09c 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ flexible messaging model and an intuitive client API. 1.69 1.0.6 1.0.2.3 - 2.15.0 + 2.14.2 0.10.2 1.6.2 8.37 diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java index c1c17419abbcb..626a14b92eedd 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java @@ -21,6 +21,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import com.fasterxml.jackson.databind.AnnotationIntrospector; +import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; import com.fasterxml.jackson.databind.util.EnumResolver; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -56,6 +58,8 @@ public final class FieldParser { private static final Map CONVERTERS = new HashMap<>(); private static final Map, Class> WRAPPER_TYPES = new HashMap<>(); + private static final AnnotationIntrospector ANNOTATION_INTROSPECTOR = new JacksonAnnotationIntrospector(); + static { // Preload converters and wrapperTypes. initConverters(); @@ -96,8 +100,7 @@ public static T convert(Object from, Class to) { if (to.isEnum()) { // Converting string to enum - EnumResolver r = EnumResolver.constructUsingToString( - ObjectMapperFactory.getMapper().getObjectMapper().getDeserializationConfig(), to); + EnumResolver r = EnumResolver.constructUsingToString((Class>) to, ANNOTATION_INTROSPECTOR); T value = (T) r.findEnum((String) from); if (value == null) { throw new RuntimeException("Invalid value '" + from + "' for enum " + to); diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 407cd9a0eb9e4..09d1396b70419 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.15.0.jar - - jackson-core-2.15.0.jar - - jackson-databind-2.15.0.jar - - jackson-dataformat-smile-2.15.0.jar - - jackson-datatype-guava-2.15.0.jar - - jackson-datatype-jdk8-2.15.0.jar - - jackson-datatype-joda-2.15.0.jar - - jackson-datatype-jsr310-2.15.0.jar - - jackson-dataformat-yaml-2.15.0.jar - - jackson-jaxrs-base-2.15.0.jar - - jackson-jaxrs-json-provider-2.15.0.jar - - jackson-module-jaxb-annotations-2.15.0.jar - - jackson-module-jsonSchema-2.15.0.jar + - jackson-annotations-2.14.2.jar + - jackson-core-2.14.2.jar + - jackson-databind-2.14.2.jar + - jackson-dataformat-smile-2.14.2.jar + - jackson-datatype-guava-2.14.2.jar + - jackson-datatype-jdk8-2.14.2.jar + - jackson-datatype-joda-2.14.2.jar + - jackson-datatype-jsr310-2.14.2.jar + - jackson-dataformat-yaml-2.14.2.jar + - jackson-jaxrs-base-2.14.2.jar + - jackson-jaxrs-json-provider-2.14.2.jar + - jackson-module-jaxb-annotations-2.14.2.jar + - jackson-module-jsonSchema-2.14.2.jar * Guava - guava-31.0.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -456,7 +456,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.8.4.jar * Jackson - - jackson-module-parameter-names-2.15.0.jar + - jackson-module-parameter-names-2.14.2.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access From 0ec576b25fdd7e4890799184ac03f66deb4735f6 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 26 Apr 2023 22:16:39 +0800 Subject: [PATCH 153/280] [fix][test] Fix flaky test `ConcurrentBitmapSortedLongPairSetTest` (#20165) --- .../utils/ConcurrentBitmapSortedLongPairSetTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java index 47379c3b6f17c..5f8f13288cfe8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java @@ -26,7 +26,6 @@ import org.testng.annotations.Test; import java.util.ArrayList; import java.util.List; -import java.util.Random; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -188,15 +187,12 @@ public void concurrentInsertions() throws Throwable { List> futures = new ArrayList<>(); for (int i = 0; i < nThreads; i++) { final int threadIdx = i; - futures.add(executor.submit(() -> { - Random random = new Random(); + int start = N * (threadIdx + 1); for (int j = 0; j < N; j++) { - int key = random.nextInt(); + int key = start + j; // Ensure keys are unique - key -= key % (threadIdx + 1); - key = Math.abs(key); set.add(key, key); } })); From f7c0b3c49c9ad8c28d0b00aa30d727850eb8bc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 26 Apr 2023 17:36:56 +0200 Subject: [PATCH 154/280] [improve][fn] Allow unknown fields in connectors config (#20116) --- conf/functions_worker.yml | 6 + .../functions/instance/InstanceConfig.java | 1 + .../instance/JavaInstanceRunnable.java | 97 ++++++++++++++-- .../instance/JavaInstanceRunnableTest.java | 104 +++++++++++++++--- .../runtime/JavaInstanceStarter.java | 7 ++ .../functions/runtime/RuntimeUtils.java | 6 +- .../pulsar/functions/worker/WorkerConfig.java | 11 ++ .../functions/worker/FunctionActioner.java | 1 + .../META-INF/services/pulsar-io.yaml | 3 +- .../META-INF/services/pulsar-io.yaml | 1 + .../META-INF/services/pulsar-io.yaml | 1 + .../META-INF/services/pulsar-io.yaml | 1 + .../META-INF/services/pulsar-io.yaml | 1 + .../META-INF/services/pulsar-io.yaml | 1 + 14 files changed, 217 insertions(+), 24 deletions(-) diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index bb15e0ca416da..4c5b6aab1b7f4 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -407,6 +407,12 @@ validateConnectorConfig: false # If it is set to true, you must ensure that it has been initialized by "bin/pulsar initialize-cluster-metadata" command. initializedDlogMetadata: false +# Whether to ignore unknown properties when deserializing the connector configuration. +# After upgrading a connector to a new version with a new configuration, the new configuration may not be compatible with the old connector. +# In case of rollback, it's required to also rollback the connector configuration. +# Ignoring unknown fields makes possible to keep the new configuration and only rollback the connector. +ignoreUnknownConfigFields: false + ########################### # Arbitrary Configuration ########################### diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java index 1a89505d9bb11..fcee6d734d6c9 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java @@ -48,6 +48,7 @@ public class InstanceConfig { private boolean exposePulsarAdminClientEnabled = false; private int metricsPort; private List additionalJavaRuntimeArguments = Collections.emptyList(); + private boolean ignoreUnknownConfigFields; /** * Get the string representation of {@link #getInstanceId()}. diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index e2ad9e4c989d1..c3f36f754daca 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -19,13 +19,20 @@ package org.apache.pulsar.functions.instance; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.BeanDeserializer; import com.google.common.annotations.VisibleForTesting; import com.scurrilous.circe.checksum.Crc32cIntChecksum; import io.netty.buffer.ByteBuf; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; @@ -34,6 +41,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import lombok.Getter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import net.jodah.typetools.TypeResolver; import org.apache.commons.lang3.StringUtils; @@ -59,6 +67,7 @@ import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.ProducerConfig; +import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; @@ -94,6 +103,7 @@ import org.apache.pulsar.functions.source.batch.BatchSourceExecutor; import org.apache.pulsar.functions.utils.CryptoUtils; import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.functions.utils.io.ConnectorUtils; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.Source; import org.slf4j.Logger; @@ -855,10 +865,7 @@ private void setupInput(ContextImpl contextImpl) throws Exception { if (sourceSpec.getConfigs().isEmpty()) { this.source.open(new HashMap<>(), contextImpl); } else { - this.source.open( - ObjectMapperFactory.getMapper().reader().forType(new TypeReference>() { - }).readValue(sourceSpec.getConfigs()) - , contextImpl); + this.source.open(parseComponentConfig(sourceSpec.getConfigs()), contextImpl); } if (this.source instanceof PulsarSource) { contextImpl.setInputConsumers(((PulsarSource) this.source).getInputConsumers()); @@ -870,6 +877,83 @@ private void setupInput(ContextImpl contextImpl) throws Exception { Thread.currentThread().setContextClassLoader(this.instanceClassLoader); } } + private Map parseComponentConfig(String connectorConfigs) throws IOException { + return parseComponentConfig(connectorConfigs, instanceConfig, componentClassLoader, componentType); + } + + static Map parseComponentConfig(String connectorConfigs, + InstanceConfig instanceConfig, + ClassLoader componentClassLoader, + org.apache.pulsar.functions.proto.Function + .FunctionDetails.ComponentType componentType) + throws IOException { + final Map config = ObjectMapperFactory + .getMapper() + .reader() + .forType(new TypeReference>() {}) + .readValue(connectorConfigs); + if (instanceConfig.isIgnoreUnknownConfigFields() && componentClassLoader instanceof NarClassLoader) { + final String configClassName; + if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { + configClassName = ConnectorUtils + .getConnectorDefinition((NarClassLoader) componentClassLoader).getSourceConfigClass(); + } else if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) { + configClassName = ConnectorUtils + .getConnectorDefinition((NarClassLoader) componentClassLoader).getSinkConfigClass(); + } else { + return config; + } + if (configClassName != null) { + + Class configClass; + try { + configClass = Class.forName(configClassName, + true, Thread.currentThread().getContextClassLoader()); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Config class not found: " + configClassName, e); + } + final List allFields = BeanPropertiesReader.getBeanProperties(configClass); + + for (String s : config.keySet()) { + if (!allFields.contains(s)) { + log.error("Field '{}' not defined in the {} configuration {}, the field will be ignored", + s, + componentType, + configClass); + config.remove(s); + } + } + } + } + return config; + } + + static final class BeanPropertiesReader { + + private static final MapperBeanReader reader = new MapperBeanReader(); + + private static final class MapperBeanReader extends ObjectMapper { + @SneakyThrows + List getBeanProperties(Class valueType) { + final JsonParser parser = ObjectMapperFactory + .getMapper() + .getObjectMapper() + .createParser(""); + DeserializationConfig config = getDeserializationConfig(); + DeserializationContext ctxt = createDeserializationContext(parser, config); + BeanDeserializer deser = (BeanDeserializer) + _findRootDeserializer(ctxt, _typeFactory.constructType(valueType)); + List list = new ArrayList<>(); + deser.properties().forEachRemaining(p -> list.add(p.getName())); + return list; + } + } + + static List getBeanProperties(Class valueType) { + return reader.getBeanProperties(valueType); + } + } + private void setupOutput(ContextImpl contextImpl) throws Exception { @@ -940,9 +1024,8 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec, contextImpl.toString()); } - this.sink.open(ObjectMapperFactory.getMapper().reader().forType( - new TypeReference>() { - }).readValue(sinkSpec.getConfigs()), contextImpl); + final Map config = parseComponentConfig(sinkSpec.getConfigs()); + this.sink.open(config, contextImpl); } } catch (Exception e) { log.error("Sink open produced uncaught exception: ", e); diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java index a36a3ca62d178..5fea8bcc9fde9 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java @@ -18,19 +18,27 @@ */ package org.apache.pulsar.functions.instance; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; + import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.TreeSet; + +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; @@ -38,11 +46,10 @@ import org.apache.pulsar.functions.instance.stats.ComponentStatsManager; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.SinkSpec; -import org.apache.pulsar.functions.proto.Function.SinkSpecOrBuilder; -import org.apache.pulsar.functions.proto.Function.SourceSpecOrBuilder; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.jetbrains.annotations.NotNull; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class JavaInstanceRunnableTest { @@ -159,7 +166,7 @@ public void testFunctionResultNull() throws Exception { @NotNull private JavaInstanceRunnable getJavaInstanceRunnable(boolean autoAck, - org.apache.pulsar.functions.proto.Function.ProcessingGuarantees processingGuarantees) throws Exception { + org.apache.pulsar.functions.proto.Function.ProcessingGuarantees processingGuarantees) throws Exception { FunctionDetails functionDetails = FunctionDetails.newBuilder() .setAutoAck(autoAck) .setProcessingGuarantees(processingGuarantees).build(); @@ -184,23 +191,90 @@ public void testStatsManagerNull() throws Exception { @Test public void testSinkConfigParsingPreservesOriginalType() throws Exception { - SinkSpecOrBuilder sinkSpec = mock(SinkSpecOrBuilder.class); - when(sinkSpec.getConfigs()).thenReturn("{\"ttl\": 9223372036854775807}"); - Map parsedConfig = - new ObjectMapper().readValue(sinkSpec.getConfigs(), new TypeReference>() { - }); + final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig( + "{\"ttl\": 9223372036854775807}", + new InstanceConfig(), + null, + FunctionDetails.ComponentType.SINK + ); Assert.assertEquals(parsedConfig.get("ttl").getClass(), Long.class); Assert.assertEquals(parsedConfig.get("ttl"), Long.MAX_VALUE); } @Test public void testSourceConfigParsingPreservesOriginalType() throws Exception { - SourceSpecOrBuilder sourceSpec = mock(SourceSpecOrBuilder.class); - when(sourceSpec.getConfigs()).thenReturn("{\"ttl\": 9223372036854775807}"); - Map parsedConfig = - new ObjectMapper().readValue(sourceSpec.getConfigs(), new TypeReference>() { - }); + final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig( + "{\"ttl\": 9223372036854775807}", + new InstanceConfig(), + null, + FunctionDetails.ComponentType.SOURCE + ); Assert.assertEquals(parsedConfig.get("ttl").getClass(), Long.class); Assert.assertEquals(parsedConfig.get("ttl"), Long.MAX_VALUE); } + + + public static class ConnectorTestConfig1 { + public String field1; + } + + @DataProvider(name = "configIgnoreUnknownFields") + public static Object[][] configIgnoreUnknownFields() { + return new Object[][]{ + {false, FunctionDetails.ComponentType.SINK}, + {true, FunctionDetails.ComponentType.SINK}, + {false, FunctionDetails.ComponentType.SOURCE}, + {true, FunctionDetails.ComponentType.SOURCE} + }; + } + + @Test(dataProvider = "configIgnoreUnknownFields") + public void testSinkConfigIgnoreUnknownFields(boolean ignoreUnknownConfigFields, + FunctionDetails.ComponentType type) throws Exception { + NarClassLoader narClassLoader = mock(NarClassLoader.class); + final ConnectorDefinition connectorDefinition = new ConnectorDefinition(); + if (type == FunctionDetails.ComponentType.SINK) { + connectorDefinition.setSinkConfigClass(ConnectorTestConfig1.class.getName()); + } else { + connectorDefinition.setSourceConfigClass(ConnectorTestConfig1.class.getName()); + } + when(narClassLoader.getServiceDefinition(any())).thenReturn(ObjectMapperFactory + .getMapper().writer().writeValueAsString(connectorDefinition)); + final InstanceConfig instanceConfig = new InstanceConfig(); + instanceConfig.setIgnoreUnknownConfigFields(ignoreUnknownConfigFields); + + final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig( + "{\"field1\": \"value\", \"field2\": \"value2\"}", + instanceConfig, + narClassLoader, + type + ); + if (ignoreUnknownConfigFields) { + Assert.assertEquals(parsedConfig.size(), 1); + Assert.assertEquals(parsedConfig.get("field1"), "value"); + } else { + Assert.assertEquals(parsedConfig.size(), 2); + Assert.assertEquals(parsedConfig.get("field1"), "value"); + Assert.assertEquals(parsedConfig.get("field2"), "value2"); + } + } + + public static class ConnectorTestConfig2 { + public static int constantField = 1; + public String field1; + private long withGetter; + @JsonIgnore + private ConnectorTestConfig1 ignore; + + public long getWithGetter() { + return withGetter; + } + } + + @Test + public void testBeanPropertiesReader() throws Exception { + final List beanProperties = JavaInstanceRunnable.BeanPropertiesReader + .getBeanProperties(ConnectorTestConfig2.class); + Assert.assertEquals(new TreeSet<>(beanProperties), new TreeSet<>(Arrays.asList("field1", "withGetter"))); + } } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index deff690815d9c..c4f44be3df380 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -150,6 +150,12 @@ public class JavaInstanceStarter implements AutoCloseable { + "exposed to function context, default is disabled.", required = false) public Boolean exposePulsarAdminClientEnabled = false; + @Parameter(names = "--ignore_unknown_config_fields", + description = "Whether to ignore unknown properties when deserializing the connector configuration.", + required = false) + public Boolean ignoreUnknownConfigFields = false; + + private Server server; private RuntimeSpawner runtimeSpawner; private ThreadRuntimeFactory containerFactory; @@ -177,6 +183,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL instanceConfig.setClusterName(clusterName); instanceConfig.setMaxPendingAsyncRequests(maxPendingAsyncRequests); instanceConfig.setExposePulsarAdminClientEnabled(exposePulsarAdminClientEnabled); + instanceConfig.setIgnoreUnknownConfigFields(ignoreUnknownConfigFields); Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder(); if (functionDetailsJsonString.charAt(0) == '\'') { functionDetailsJsonString = functionDetailsJsonString.substring(1); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 53ebfcbfaf068..5392697e9283b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -435,10 +435,14 @@ && isNotBlank(authConfig.getClientAuthenticationParameters())) { args.add("--metrics_port"); args.add(String.valueOf(instanceConfig.getMetricsPort())); - // only the Java instance supports --pending_async_requests right now. + // params supported only by the Java instance runtime. if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.JAVA) { args.add("--pending_async_requests"); args.add(String.valueOf(instanceConfig.getMaxPendingAsyncRequests())); + + if (instanceConfig.isIgnoreUnknownConfigFields()) { + args.add("--ignore_unknown_config_fields"); + } } // state storage configs diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index 3b8ddf774d162..0ed73953d7aa7 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -738,6 +738,17 @@ public String getFunctionAuthProviderClassName() { ) private List additionalJavaRuntimeArguments = new ArrayList<>(); + @FieldContext( + category = CATEGORY_CONNECTORS, + doc = "Whether to ignore unknown properties when deserializing the connector configuration. " + + "After upgrading a connector to a new version with a new configuration, " + + "the new configuration may not be compatible with the old connector. " + + "In case of rollback, it's required to also rollback the connector configuration. " + + "Ignoring unknown fields makes possible to keep the new configuration and " + + "only rollback the connector." + ) + private boolean ignoreUnknownConfigFields = false; + public String getFunctionMetadataTopic() { return String.format("persistent://%s/%s", pulsarFunctionsNamespace, functionMetadataTopicName); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index c587a8a734888..03c6eb7921840 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -221,6 +221,7 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu if (workerConfig.getAdditionalJavaRuntimeArguments() != null) { instanceConfig.setAdditionalJavaRuntimeArguments(workerConfig.getAdditionalJavaRuntimeArguments()); } + instanceConfig.setIgnoreUnknownConfigFields(workerConfig.isIgnoreUnknownConfigFields()); return instanceConfig; } diff --git a/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml index 02314241f28e6..2fc04034f655f 100644 --- a/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml +++ b/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml @@ -18,4 +18,5 @@ # name: alluxio description: Writes data into Alluxio -sinkClass: org.apache.pulsar.io.alluxio.sink.AlluxioSink \ No newline at end of file +sinkClass: org.apache.pulsar.io.alluxio.sink.AlluxioSink +sinkConfigClass: org.apache.pulsar.io.alluxio.sink.AlluxioSinkConfig \ No newline at end of file diff --git a/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml index dd6ba976adb10..50cad346b3e2a 100644 --- a/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml +++ b/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml @@ -20,3 +20,4 @@ name: dynamodb description: DynamoDB connectors sourceClass: org.apache.pulsar.io.dynamodb.DynamoDBSource +sourceConfigClass: org.apache.pulsar.io.dynamodb.DynamoDBSourceConfig diff --git a/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml index 1b45638dd4de9..4b80907ee861a 100644 --- a/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml +++ b/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml @@ -20,3 +20,4 @@ name: jdbc-clickhouse description: JDBC sink for ClickHouse sinkClass: org.apache.pulsar.io.jdbc.ClickHouseJdbcAutoSchemaSink +sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig diff --git a/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml index 463c8cba1621f..81dd7277e1f15 100644 --- a/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml +++ b/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml @@ -20,3 +20,4 @@ name: jdbc-mariadb description: JDBC sink for MariaDB sinkClass: org.apache.pulsar.io.jdbc.MariadbJdbcAutoSchemaSink +sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig diff --git a/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml index f6262df6997b4..38c120f089369 100644 --- a/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml +++ b/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml @@ -20,3 +20,4 @@ name: jdbc-openmldb description: JDBC sink for OpenMLDB sinkClass: org.apache.pulsar.io.jdbc.OpenMLDBJdbcAutoSchemaSink +sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig diff --git a/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml index a8d192a45688f..282ac8a023016 100644 --- a/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml +++ b/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml @@ -20,3 +20,4 @@ name: jdbc-postgres description: JDBC sink for PostgreSQL sinkClass: org.apache.pulsar.io.jdbc.PostgresJdbcAutoSchemaSink +sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig \ No newline at end of file From 77f00413fffe2132c7bc9b1af740b57e221df6d3 Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Fri, 28 Apr 2023 04:53:11 +0300 Subject: [PATCH 155/280] [improve][misc] Change PIP issue template (#19832) --- .github/ISSUE_TEMPLATE/pip.md | 167 +++++++++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/pip.yml | 82 ---------------- 2 files changed, 167 insertions(+), 82 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/pip.md delete mode 100644 .github/ISSUE_TEMPLATE/pip.yml diff --git a/.github/ISSUE_TEMPLATE/pip.md b/.github/ISSUE_TEMPLATE/pip.md new file mode 100644 index 0000000000000..520826f8b1ceb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/pip.md @@ -0,0 +1,167 @@ +--- +name: PIP +about: Submit a Pulsar Improvement Proposal (PIP) +title: 'PIP-XYZ: ' +labels: PIP +--- + + + +# Background knowledge + + + +# Motivation + + + +# Goals + +## In Scope + + + +## Out of Scope + + + + +# High Level Design + + + +# Detailed Design + +## Design & Implementation Details + + + +## Public-facing Changes + + + +### Public API + + +### Binary protocol + +### Configuration + +### CLI + +### Metrics + + + + +# Monitoring + + + +# Security Considerations + + +# Backward & Forward Compatability + +## Revert + + + +## Upgrade + + + +# Alternatives + + + +# General Notes + +# Links + + +* Mailing List discussion thread: +* Mailing List voting thread: diff --git a/.github/ISSUE_TEMPLATE/pip.yml b/.github/ISSUE_TEMPLATE/pip.yml deleted file mode 100644 index d494d69947252..0000000000000 --- a/.github/ISSUE_TEMPLATE/pip.yml +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: PIP -title: "PIP-XYZ: " -description: Submit a Pulsar Improvement Proposal (PIP) -labels: [ "PIP" ] -body: - - type: markdown - attributes: - value: | - Thank you very much for submitting a Pulsar Improvement Proposal (PIP)! Here are instructions for creating a PIP using this issue template. - - Please send a note to the dev@pulsar.apache.org mailing list to start the discussion, using subject prefix `[DISCUSS] PIP-XYZ`. To determine the appropriate PIP number XYZ, inspect the [mailing list](https://lists.apache.org/list.html?dev@pulsar.apache.org) for the most recent PIP. Add 1 to that PIP's number to get your PIP's number. - - Based on the discussion and feedback, some changes might be applied by the author(s) to the text of the proposal. - - Once some consensus is reached, there will be a vote to formally approve the proposal. The vote will be held on the dev@pulsar.apache.org mailing list. Everyone is welcome to vote on the proposal, though it will considered to be binding only the vote of PMC members. It will be required to have a lazy majority of at least 3 binding +1s votes. The vote should stay open for at least 48 hours. - - When the vote is closed, if the outcome is positive, the state of the proposal is updated and the Pull Requests associated with this proposal can start to get merged into the master branch. - - type: textarea - attributes: - label: Motivation - description: | - Explain why this change is needed, what benefits it would bring to Apache Pulsar and what problem it's trying to solve. - validations: - required: true - - type: textarea - attributes: - label: Goal - description: | - Define the scope of this proposal. Given the motivation stated above, what are the problems that this proposal is addressing and what other items will be considering out of scope, perhaps to be left to a different PIP. - validations: - required: true - - type: textarea - attributes: - label: API Changes - description: | - Illustrate all the proposed changes to the API or wire protocol, with examples of all the newly added classes/methods, including Javadoc. - - type: textarea - attributes: - label: Implementation - description: | - This should be a detailed description of all the changes that are expected to be made. It should be detailed enough that any developer that is familiar with Pulsar internals would be able to understand all the parts of the code changes for this proposal. - - This should also serve as documentation for any person that is trying to understand or debug the behavior of a certain feature. - validations: - required: true - - type: textarea - attributes: - label: Security Considerations - description: | - A detailed description of the security details that ought to be considered for the PIP. This is most relevant for any new HTTP endpoints, new Pulsar Protocol Commands, and new security features. The goal is to describe details like which role will have permission to perform an action. - - If there is uncertainty for this section, please submit the PIP and request for feedback on the mailing list. - validations: - required: true - - type: textarea - attributes: - label: Alternatives - description: | - If there are alternatives that were already considered by the authors or, after the discussion, by the community, and were rejected, please list them here along with the reason why they were rejected. - - type: textarea - attributes: - label: Anything else? - - type: markdown - attributes: - value: "Thanks for completing our form!" From 8b7aa6312b2af49d0c6edde85c6b0108007a0a96 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Fri, 28 Apr 2023 16:46:51 +0800 Subject: [PATCH 156/280] [fix][admin] Fix examine messages if total message is zero (#20152) --- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 4 ++++ .../apache/pulsar/broker/admin/PersistentTopicsTest.java | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 6b163ae04469b..fcade8270cb12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3061,6 +3061,10 @@ protected CompletableFuture internalExamineMessageAsync(String initial try { PersistentTopic persistentTopic = (PersistentTopic) topic; long totalMessage = persistentTopic.getNumberOfEntries(); + if (totalMessage <= 0) { + throw new RestException(Status.PRECONDITION_FAILED, + "Could not examine messages due to the total message is zero"); + } PositionImpl startPosition = persistentTopic.getFirstPosition(); long messageToSkip = initialPositionLocal.equals("earliest") ? messagePositionLocal : diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index f80d9863a26a6..284e50c830286 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -1151,6 +1151,14 @@ public void testExamineMessage() throws Exception { + "topic partition"); } + try { + admin.topics().examineMessage(topicName + "-partition-0", "earliest", 1); + Assert.fail(); + } catch (PulsarAdminException e) { + Assert.assertEquals(e.getMessage(), + "Could not examine messages due to the total message is zero"); + } + producer.send("message1"); Assert.assertEquals( new String(admin.topics().examineMessage(topicName + "-partition-0", "earliest", 1).getData()), From b8543ad979798d89da9f35a9375de7e16b7e5e25 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 29 Apr 2023 01:52:52 +0800 Subject: [PATCH 157/280] [improve][broker] Support cgroup v2 by using `jdk.internal.platform.Metrics` in Pulsar Loadbalancer (#16832) --- bin/pulsar | 2 + buildtools/pom.xml | 1 + pom.xml | 1 + .../broker/loadbalance/LinuxInfoUtils.java | 66 ++++++++++++++++--- .../impl/LinuxBrokerHostUsageImpl.java | 2 +- .../loadbalance/SimpleBrokerStartTest.java | 26 ++++++++ .../impl/LinuxBrokerHostUsageImplTest.java | 37 ++++++++++- 7 files changed, 122 insertions(+), 13 deletions(-) diff --git a/bin/pulsar b/bin/pulsar index a033de947d4b3..e3b22caced52e 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -307,6 +307,8 @@ if [[ -z "$IS_JAVA_8" ]]; then OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED" # MBeanStatsGenerator OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED" + # LinuxInfoUtils + OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED" fi OPTS="-cp $PULSAR_CLASSPATH $OPTS" diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 04b25cced42ac..a21618004e31a 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -58,6 +58,7 @@ --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED diff --git a/pom.xml b/pom.xml index aef380c5cd09c..c5d7092c7b4ec 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ flexible messaging model and an intuitive client API. --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED true 4 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 42ef264b6db04..1405979ae3b12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.broker.loadbalance; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -45,6 +47,7 @@ public class LinuxInfoUtils { private static final String CGROUPS_CPU_USAGE_PATH = "/sys/fs/cgroup/cpu/cpuacct.usage"; private static final String CGROUPS_CPU_LIMIT_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"; private static final String CGROUPS_CPU_LIMIT_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"; + // proc states private static final String PROC_STAT_PATH = "/proc/stat"; private static final String NIC_PATH = "/sys/class/net/"; @@ -52,6 +55,30 @@ public class LinuxInfoUtils { private static final int ARPHRD_ETHER = 1; private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed"; + private static Object /*jdk.internal.platform.Metrics*/ metrics; + private static Method getMetricsProviderMethod; + private static Method getCpuQuotaMethod; + private static Method getCpuPeriodMethod; + private static Method getCpuUsageMethod; + + static { + try { + metrics = Class.forName("jdk.internal.platform.Container").getMethod("metrics") + .invoke(null); + if (metrics != null) { + getMetricsProviderMethod = metrics.getClass().getMethod("getProvider"); + getMetricsProviderMethod.setAccessible(true); + getCpuQuotaMethod = metrics.getClass().getMethod("getCpuQuota"); + getCpuQuotaMethod.setAccessible(true); + getCpuPeriodMethod = metrics.getClass().getMethod("getCpuPeriod"); + getCpuPeriodMethod.setAccessible(true); + getCpuUsageMethod = metrics.getClass().getMethod("getCpuUsage"); + getCpuUsageMethod.setAccessible(true); + } + } catch (Throwable e) { + log.warn("Failed to get runtime metrics", e); + } + } /** * Determine whether the OS is the linux kernel. @@ -66,9 +93,14 @@ public static boolean isLinux() { */ public static boolean isCGroupEnabled() { try { - return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH)); + if (metrics == null) { + return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH)); + } + String provider = (String) getMetricsProviderMethod.invoke(metrics); + log.info("[LinuxInfo] The system metrics provider is: {}", provider); + return provider.contains("cgroup"); } catch (Exception e) { - log.warn("[LinuxInfo] Failed to check cgroup CPU usage file: {}", e.getMessage()); + log.warn("[LinuxInfo] Failed to check cgroup CPU: {}", e.getMessage()); return false; } } @@ -81,13 +113,21 @@ public static boolean isCGroupEnabled() { public static double getTotalCpuLimit(boolean isCGroupsEnabled) { if (isCGroupsEnabled) { try { - long quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH)); - long period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH)); + long quota; + long period; + if (metrics != null && getCpuQuotaMethod != null && getCpuPeriodMethod != null) { + quota = (long) getCpuQuotaMethod.invoke(metrics); + period = (long) getCpuPeriodMethod.invoke(metrics); + } else { + quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH)); + period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH)); + } + if (quota > 0) { return 100.0 * quota / period; } - } catch (IOException e) { - log.warn("[LinuxInfo] Failed to read CPU quotas from cgroups", e); + } catch (Exception e) { + log.warn("[LinuxInfo] Failed to read CPU quotas from cgroup", e); // Fallback to availableProcessors } } @@ -99,11 +139,14 @@ public static double getTotalCpuLimit(boolean isCGroupsEnabled) { * Get CGroup cpu usage. * @return Cpu usage */ - public static double getCpuUsageForCGroup() { + public static long getCpuUsageForCGroup() { try { + if (metrics != null && getCpuUsageMethod != null) { + return (long) getCpuUsageMethod.invoke(metrics); + } return readLongFromFile(Paths.get(CGROUPS_CPU_USAGE_PATH)); - } catch (IOException e) { - log.error("[LinuxInfo] Failed to read CPU usage from {}", CGROUPS_CPU_USAGE_PATH, e); + } catch (Exception e) { + log.error("[LinuxInfo] Failed to read CPU usage from cgroup", e); return -1; } } @@ -291,6 +334,11 @@ enum Operstate { UP } + @VisibleForTesting + public static Object getMetrics() { + return metrics; + } + @AllArgsConstructor public enum NICUsageType { // transport diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 318f37f7f7a97..5d2b7bdd09adf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -140,7 +140,7 @@ private double getTotalCpuUsage(double elapsedTimeSeconds) { } private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) { - double usage = getCpuUsageForCGroup(); + double usage = (double) getCpuUsageForCGroup(); double currentUsage = usage - lastCpuUsage; lastCpuUsage = usage; return 100 * currentUsage / elapsedTimeSeconds / TimeUnit.SECONDS.toNanos(1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index 6de2eb90f12ef..28dde8b7f559d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -20,6 +20,8 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -28,6 +30,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.testng.Assert; import org.testng.annotations.Test; @Slf4j @@ -96,4 +99,27 @@ public void testNoNICSpeed() throws Exception { } + @Test + public void testCGroupMetrics() { + if (!LinuxInfoUtils.isLinux()) { + return; + } + + boolean existsCGroup = Files.exists(Paths.get("/sys/fs/cgroup")); + boolean cGroupEnabled = LinuxInfoUtils.isCGroupEnabled(); + Assert.assertEquals(cGroupEnabled, existsCGroup); + + double totalCpuLimit = LinuxInfoUtils.getTotalCpuLimit(cGroupEnabled); + log.info("totalCpuLimit: {}", totalCpuLimit); + Assert.assertTrue(totalCpuLimit > 0.0); + + if (cGroupEnabled) { + Assert.assertNotNull(LinuxInfoUtils.getMetrics()); + + long cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); + log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup); + Assert.assertTrue(cpuUsageForCGroup > 0); + } + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java index 39298dce0b16a..563f707c445b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java @@ -18,15 +18,19 @@ */ package org.apache.pulsar.broker.loadbalance.impl; -import lombok.Cleanup; -import org.testng.Assert; -import org.testng.annotations.Test; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils; +import org.testng.Assert; +import org.testng.annotations.Test; +@Slf4j public class LinuxBrokerHostUsageImplTest { @Test @@ -42,4 +46,31 @@ public void checkOverrideBrokerNicSpeedGbps() { double totalLimit = linuxBrokerHostUsage.getTotalNicLimitWithConfiguration(nics); Assert.assertEquals(totalLimit, 3.0 * 1000 * 1000 * 3); } + + @Test + public void testCpuUsage() throws InterruptedException { + if (!LinuxInfoUtils.isLinux()) { + return; + } + + @Cleanup("shutdown") + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + LinuxBrokerHostUsageImpl linuxBrokerHostUsage = + new LinuxBrokerHostUsageImpl(Integer.MAX_VALUE, Optional.empty(), executorService); + + linuxBrokerHostUsage.calculateBrokerHostUsage(); + TimeUnit.SECONDS.sleep(1); + linuxBrokerHostUsage.calculateBrokerHostUsage(); + + double usage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().usage; + double limit = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().limit; + float percentUsage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().percentUsage(); + + Assert.assertTrue(usage > 0); + Assert.assertTrue(limit > 0); + Assert.assertTrue(limit >= usage); + Assert.assertTrue(percentUsage > 0); + + log.info("usage: {}, limit: {}, percentUsage: {}", usage, limit, percentUsage); + } } From d135c4a115038dc61f8fe2d230cb1f0c02239f92 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 29 Apr 2023 01:54:05 +0800 Subject: [PATCH 158/280] [improve] [broker] Skip split boundle if only one broker (#20190) Co-authored-by: Zixuan Liu --- .../impl/ModularLoadManagerImpl.java | 2 +- .../broker/stats/PrometheusMetricsTest.java | 9 +++ .../client/api/BrokerServiceLookupTest.java | 58 +++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index c6f406a7c8a5c..30a2ef5cdf250 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -744,7 +744,7 @@ public boolean shouldAntiAffinityNamespaceUnload(String namespace, String bundle public void checkNamespaceBundleSplit() { if (!conf.isLoadBalancerAutoBundleSplitEnabled() || pulsar.getLeaderElectionService() == null - || !pulsar.getLeaderElectionService().isLeader()) { + || !pulsar.getLeaderElectionService().isLeader() || knownBrokers.size() <= 1) { return; } final boolean unloadSplitBundles = pulsar.getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index f5ae8459f1803..bf141e10aa1b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -80,6 +81,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.compaction.Compactor; +import org.apache.zookeeper.CreateMode; import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; @@ -772,6 +774,10 @@ public void testBundlesMetrics() throws Exception { c1.acknowledge(c1.receive()); } + // Mock another broker to make split task work. + String mockedBroker = "/loadbalance/brokers/127.0.0.1:0"; + mockZooKeeper.create(mockedBroker, new byte[]{0}, Collections.emptyList(), CreateMode.EPHEMERAL); + pulsar.getBrokerService().updateRates(); Awaitility.await().untilAsserted(() -> assertTrue(pulsar.getBrokerService().getBundleStats().size() > 0)); ModularLoadManagerWrapper loadManager = (ModularLoadManagerWrapper)pulsar.getLoadManager().get(); @@ -796,6 +802,9 @@ public void testBundlesMetrics() throws Exception { assertTrue(metrics.containsKey("pulsar_lb_bandwidth_out_usage")); assertTrue(metrics.containsKey("pulsar_lb_bundles_split_total")); + + // cleanup. + mockZooKeeper.delete(mockedBroker, 0); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 2af9f450dd3b1..8597e0a87997a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -67,6 +67,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; @@ -80,6 +81,7 @@ import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; @@ -781,6 +783,62 @@ public void testModularLoadManagerSplitBundle() throws Exception { } } + @Test(timeOut = 20000) + public void testSkipSplitBundleIfOnlyOneBroker() throws Exception { + + log.info("-- Starting {} test --", methodName); + final String loadBalancerName = conf.getLoadManagerClassName(); + final int defaultNumberOfNamespaceBundles = conf.getDefaultNumberOfNamespaceBundles(); + final int loadBalancerNamespaceBundleMaxTopics = conf.getLoadBalancerNamespaceBundleMaxTopics(); + + final String namespace = "my-property/my-ns"; + final String topicName1 = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_"); + final String topicName2 = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_"); + try { + // configure broker with ModularLoadManager. + stopBroker(); + conf.setDefaultNumberOfNamespaceBundles(1); + conf.setLoadBalancerNamespaceBundleMaxTopics(1); + conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + startBroker(); + final ModularLoadManagerWrapper modularLoadManagerWrapper = + (ModularLoadManagerWrapper) pulsar.getLoadManager().get(); + final ModularLoadManagerImpl modularLoadManager = + (ModularLoadManagerImpl) modularLoadManagerWrapper.getLoadManager(); + + // Create one topic and trigger tasks, then verify there is only one bundle now. + Consumer consumer1 = pulsarClient.newConsumer().topic(topicName1) + .subscriptionName("my-subscriber-name").subscribe(); + List bounldes1 = pulsar.getNamespaceService().getNamespaceBundleFactory() + .getBundles(NamespaceName.get(namespace)).getBundles(); + pulsar.getBrokerService().updateRates(); + pulsar.getLoadManager().get().writeLoadReportOnZookeeper(); + pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper(); + modularLoadManager.updateAll(); + assertEquals(bounldes1.size(), 1); + + // Create the second topic and trigger tasks, then verify the split task will be skipped. + Consumer consumer2 = pulsarClient.newConsumer().topic(topicName2) + .subscriptionName("my-subscriber-name").subscribe(); + pulsar.getBrokerService().updateRates(); + pulsar.getLoadManager().get().writeLoadReportOnZookeeper(); + pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper(); + modularLoadManager.updateAll(); + List bounldes2 = pulsar.getNamespaceService().getNamespaceBundleFactory() + .getBundles(NamespaceName.get(namespace)).getBundles(); + assertEquals(bounldes2.size(), 1); + + consumer1.close(); + consumer2.close(); + admin.topics().delete(topicName1, false); + admin.topics().delete(topicName2, false); + } finally { + conf.setDefaultNumberOfNamespaceBundles(defaultNumberOfNamespaceBundles); + conf.setLoadBalancerNamespaceBundleMaxTopics(loadBalancerNamespaceBundleMaxTopics); + conf.setLoadManagerClassName(loadBalancerName); + } + } + @Test(timeOut = 10000) public void testPartitionedMetadataWithDeprecatedVersion() throws Exception { From 7ccd1ba119b1349de4f865b0bc0e1065c6769950 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 28 Apr 2023 20:55:19 +0300 Subject: [PATCH 159/280] [improve][ci] Disable Maven http connection pooling on CI also for newer Maven versions (#20198) --- .github/workflows/ci-go-functions.yaml | 2 +- .github/workflows/ci-maven-cache-update.yaml | 2 +- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- .github/workflows/pulsar-ci-flaky.yaml | 2 +- .github/workflows/pulsar-ci.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index a34cb1a90908e..f96a6d6586e6e 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -32,7 +32,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: preconditions: diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index a5a88e2e0b8a6..0bf1c4148b2c9 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -42,7 +42,7 @@ on: - cron: '30 */12 * * *' env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: update-maven-dependencies-cache: diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 8aec501f8d868..9f1ba2cc996f8 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: run-owasp-dependency-check: diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index ea19bc3307819..fdc25ca1d93a2 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -36,7 +36,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 4d4f2902d8f27..fca0928477925 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -36,7 +36,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR From a362eaa8bd8caf13795aba5ddbdf4edd8cec7bd0 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Sat, 29 Apr 2023 01:57:43 +0800 Subject: [PATCH 160/280] [improve][broker] Improve knownBrokers update in ModularLoadManagerImpl (#20196) --- .../loadbalance/impl/ModularLoadManagerImpl.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 30a2ef5cdf250..73b4f318f3a36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -195,7 +195,7 @@ public class ModularLoadManagerImpl implements ModularLoadManager { private long unloadBundleCount = 0; private final Lock lock = new ReentrantLock(); - private Set knownBrokers = ConcurrentHashMap.newKeySet(); + private final Set knownBrokers = new HashSet<>(); private Map bundleBrokerAffinityMap; /** @@ -480,13 +480,11 @@ public void updateAll() { checkNamespaceBundleSplit(); } - private void cleanupDeadBrokersData() { + private synchronized void cleanupDeadBrokersData() { final Set activeBrokers = getAvailableBrokers(); - final Set knownBrokersCopy = new HashSet<>(this.knownBrokers); - Collection newBrokers = CollectionUtils.subtract(activeBrokers, knownBrokersCopy); - this.knownBrokers.addAll(newBrokers); - Collection deadBrokers = CollectionUtils.subtract(knownBrokersCopy, activeBrokers); - this.knownBrokers.removeAll(deadBrokers); + Collection deadBrokers = CollectionUtils.subtract(knownBrokers, activeBrokers); + this.knownBrokers.clear(); + this.knownBrokers.addAll(activeBrokers); if (pulsar.getLeaderElectionService() != null && pulsar.getLeaderElectionService().isLeader()) { deadBrokers.forEach(this::deleteTimeAverageDataFromMetadataStoreAsync); From 8c963e3ae3359fee824294c590a9c045f4800228 Mon Sep 17 00:00:00 2001 From: Clay Johnson Date: Sat, 29 Apr 2023 20:26:10 -0500 Subject: [PATCH 161/280] [improve][build] Capture local build scans on ge.apache.org to benefit from deep build insights (#20187) --- .github/actions/gradle-enterprise/action.yml | 1 - .gitignore | 1 - .mvn/{ge-extensions.xml => extensions.xml} | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) rename .mvn/{ge-extensions.xml => extensions.xml} (97%) diff --git a/.github/actions/gradle-enterprise/action.yml b/.github/actions/gradle-enterprise/action.yml index 935e76d3cd645..a11e59899c756 100644 --- a/.github/actions/gradle-enterprise/action.yml +++ b/.github/actions/gradle-enterprise/action.yml @@ -29,7 +29,6 @@ runs: - run: | if [[ -n "${{ inputs.token }}" ]]; then echo "::group::Configuring Gradle Enterprise for build" - cp .mvn/ge-extensions.xml .mvn/extensions.xml echo "GRADLE_ENTERPRISE_ACCESS_KEY=${{ inputs.token }}" >> $GITHUB_ENV echo "::endgroup::" fi diff --git a/.gitignore b/.gitignore index c584baaa0a0b8..cd00c44200059 100644 --- a/.gitignore +++ b/.gitignore @@ -97,4 +97,3 @@ test-reports/ # Gradle Enterprise .mvn/.gradle-enterprise/ -.mvn/extensions.xml diff --git a/.mvn/ge-extensions.xml b/.mvn/extensions.xml similarity index 97% rename from .mvn/ge-extensions.xml rename to .mvn/extensions.xml index 1c7a1611c1bcc..3c03b32a23775 100644 --- a/.mvn/ge-extensions.xml +++ b/.mvn/extensions.xml @@ -24,7 +24,7 @@ com.gradle gradle-enterprise-maven-extension - 1.16.3 + 1.17 com.gradle From e8d63952a40b79499eb3fd524f80db6bc986ed34 Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Sun, 30 Apr 2023 12:58:53 +0300 Subject: [PATCH 162/280] [fix][docs] Remove old template inlined (#20208) --- wiki/proposals/PIP.md | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index 159c9199895d1..fc68905726eb8 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -116,38 +116,5 @@ Some examples: ## Template for a PIP design doc -``` -## Motivation +Read [the template file](/.github/ISSUE_TEMPLATE/pip.md). -Explain why this change is needed, what benefits it would bring to Apache Pulsar -and what problem it's trying to solve. - -## Goal - -Define the scope of this proposal. Given the motivation stated above, what are -the problems that this proposal is addressing and what other items will be -considering out of scope, perhaps to be left to a different PIP. - -## API Changes - -Illustrate all the proposed changes to the API or wire protocol, with examples -of all the newly added classes/methods, including Javadoc. - -## Implementation - -This should be a detailed description of all the changes that are -expected to be made. It should be detailed enough that any developer that is -familiar with Pulsar internals would be able to understand all the parts of the -code changes for this proposal. - -This should also serve as documentation for any person that is trying to -understand or debug the behavior of a certain feature. - - -## Reject Alternatives - -If there are alternatives that were already considered by the authors or, -after the discussion, by the community, and were rejected, please list them -here along with the reason why they were rejected. - -``` From a22700df665a9ea0a8db98334b6d4337e5e490f3 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 1 May 2023 16:47:36 +0800 Subject: [PATCH 163/280] [fix][client] SchemaDefinition handle JSR310_CONVERSION_ENABLED property (#20201) Signed-off-by: tison --- .../schema/SchemaDefinitionBuilderImpl.java | 2 +- .../impl/schema/KeyValueSchemaTest.java | 34 +++++++++++++++++-- .../pulsar/functions/source/TopicSchema.java | 2 -- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java index c50aac5c39821..93869a3bb9e38 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java @@ -129,7 +129,7 @@ public SchemaDefinitionBuilder withProperties(Map properties) if (properties.containsKey(ALWAYS_ALLOW_NULL)) { alwaysAllowNull = Boolean.parseBoolean(properties.get(ALWAYS_ALLOW_NULL)); } - if (properties.containsKey(ALWAYS_ALLOW_NULL)) { + if (properties.containsKey(JSR310_CONVERSION_ENABLED)) { jsr310ConversionEnabled = Boolean.parseBoolean(properties.get(JSR310_CONVERSION_ENABLED)); } return this; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index b40fa86e8bf95..078c52c5f2e11 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -69,7 +69,7 @@ public void testAllowNullAvroSchemaCreate() { } @Test - public void testFillParametersToSchemainfo() { + public void testFillParametersToSchemaInfo() { Map keyProperties = new TreeMap<>(); keyProperties.put("foo.key1", "value"); keyProperties.put("foo.key2", "value"); @@ -89,7 +89,6 @@ public void testFillParametersToSchemainfo() { .build()); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); - assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.type"), String.valueOf(SchemaType.AVRO)); assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.properties"), "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\",\"foo.key1\":\"value\",\"foo.key2\":\"value\"}"); @@ -98,6 +97,37 @@ public void testFillParametersToSchemainfo() { "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\",\"bar.key\":\"key\"}"); } + @Test + public void testOverwriteSchemaDefaultProperties() { + Map keyProperties = new TreeMap<>(); + keyProperties.put("foo.key1", "value"); + keyProperties.put("foo.key2", "value"); + keyProperties.put(SchemaDefinitionBuilderImpl.ALWAYS_ALLOW_NULL, "false"); + keyProperties.put(SchemaDefinitionBuilderImpl.JSR310_CONVERSION_ENABLED, "true"); + + Map valueProperties = new TreeMap<>(); + valueProperties.put("bar.key", "key"); + + AvroSchema fooSchema = AvroSchema.of( + SchemaDefinition.builder() + .withPojo(Foo.class) + .withProperties(keyProperties) + .build()); + AvroSchema barSchema = AvroSchema.of( + SchemaDefinition.builder() + .withPojo(Bar.class) + .withProperties(valueProperties) + .build()); + + Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); + assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.type"), String.valueOf(SchemaType.AVRO)); + assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.properties"), + "{\"__alwaysAllowNull\":\"false\",\"__jsr310ConversionEnabled\":\"true\",\"foo.key1\":\"value\",\"foo.key2\":\"value\"}"); + assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("value.schema.type"), String.valueOf(SchemaType.AVRO)); + assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("value.schema.properties"), + "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\",\"bar.key\":\"key\"}"); + } + @Test public void testNotAllowNullAvroSchemaCreate() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).withAlwaysAllowNull(false).build()); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index 57f49fed0cac9..8ed177cba3b19 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -44,8 +44,6 @@ @Slf4j public class TopicSchema { - public static final String JSR_310_CONVERSION_ENABLED = "jsr310ConversionEnabled"; - public static final String ALWAYS_ALLOW_NULL = "alwaysAllowNull"; private final Map> cachedSchemas = new HashMap<>(); private final PulsarClient client; From 010bd508cecae501fa812be6df4b3f7d33ec4be6 Mon Sep 17 00:00:00 2001 From: Clay Johnson Date: Thu, 4 May 2023 06:06:13 -0500 Subject: [PATCH 164/280] [improve][ci] Replace handmade action to configure Gradle Enterprise (#20218) --- .github/actions/gradle-enterprise/action.yml | 35 ---------- .github/workflows/ci-maven-cache-update.yaml | 6 +- .../workflows/ci-owasp-dependency-check.yaml | 7 +- .github/workflows/pulsar-ci-flaky.yaml | 6 +- .github/workflows/pulsar-ci.yaml | 68 ++++--------------- 5 files changed, 16 insertions(+), 106 deletions(-) delete mode 100644 .github/actions/gradle-enterprise/action.yml diff --git a/.github/actions/gradle-enterprise/action.yml b/.github/actions/gradle-enterprise/action.yml deleted file mode 100644 index a11e59899c756..0000000000000 --- a/.github/actions/gradle-enterprise/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Configure Gradle Enterprise integration -description: Configure Gradle Enterprise when secret GE_ACCESS_TOKEN is available -inputs: - token: - description: 'The token for accessing Gradle Enterprise' - required: true -runs: - using: composite - steps: - - run: | - if [[ -n "${{ inputs.token }}" ]]; then - echo "::group::Configuring Gradle Enterprise for build" - echo "GRADLE_ENTERPRISE_ACCESS_KEY=${{ inputs.token }}" >> $GITHUB_ENV - echo "::endgroup::" - fi - shell: bash \ No newline at end of file diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index 0bf1c4148b2c9..15fefaf3f1645 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -49,6 +49,7 @@ jobs: name: Update Maven dependency cache for ${{ matrix.name }} env: JOB_NAME: Update Maven dependency cache for ${{ matrix.name }} + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 45 @@ -77,11 +78,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Detect changed files if: ${{ github.event_name != 'schedule' }} id: changes diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 9f1ba2cc996f8..06edbae51adde 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -32,6 +32,7 @@ jobs: name: Check ${{ matrix.branch }} env: JOB_NAME: Check ${{ matrix.branch }} + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: 45 strategy: @@ -57,12 +58,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - if: ${{ matrix.branch == 'master' }} - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Cache local Maven repository uses: actions/cache@v3 timeout-minutes: 5 diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index fdc25ca1d93a2..555ebdb17292f 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -94,6 +94,7 @@ jobs: env: JOB_NAME: Flaky tests suite COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: 100 if: ${{ needs.preconditions.outputs.docs_only != 'true' }} @@ -104,11 +105,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index fca0928477925..b92599581cd83 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -95,6 +95,7 @@ jobs: name: Build and License check env: JOB_NAME: Build and License check + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: 60 if: ${{ needs.preconditions.outputs.docs_only != 'true' }} @@ -105,11 +106,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -175,6 +171,7 @@ jobs: env: JOB_NAME: CI - Unit - ${{ matrix.name }} COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['preconditions', 'build-and-license-check'] @@ -211,11 +208,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -391,6 +383,8 @@ jobs: timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true'}} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -398,11 +392,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -469,6 +458,7 @@ jobs: env: JOB_NAME: CI - Integration - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} strategy: fail-fast: false matrix: @@ -513,11 +503,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -721,11 +706,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -739,6 +719,8 @@ jobs: timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -746,11 +728,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -851,6 +828,7 @@ jobs: env: JOB_NAME: CI - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} strategy: fail-fast: false matrix: @@ -886,11 +864,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -1081,6 +1054,7 @@ jobs: env: JOB_NAME: CI Flaky - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} strategy: fail-fast: false matrix: @@ -1098,11 +1072,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -1208,11 +1177,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -1226,6 +1190,8 @@ jobs: timeout-minutes: 120 needs: ['preconditions', 'integration-tests'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -1233,11 +1199,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 @@ -1264,6 +1225,8 @@ jobs: timeout-minutes: 120 needs: [ 'preconditions', 'integration-tests' ] if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -1271,11 +1234,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} From 2b515ffb389c4b4fe3cb5a9c5f3d7eb0a4c9ef99 Mon Sep 17 00:00:00 2001 From: ran Date: Fri, 5 May 2023 08:48:59 +0800 Subject: [PATCH 165/280] [fix][broker] Fix the reason label of authentication metrics (#20030) --- .../AuthenticationProviderAthenz.java | 20 ++++++++++-- .../oidc/AuthenticationProviderOpenID.java | 2 +- .../AuthenticationProvider.java | 5 +++ .../AuthenticationProviderBasic.java | 21 +++++++++++-- .../AuthenticationProviderList.java | 20 +++++++----- .../AuthenticationProviderTls.java | 12 +++++-- .../AuthenticationProviderToken.java | 31 ++++++++++++------- .../metrics/AuthenticationMetrics.java | 16 ++++++++++ .../broker/stats/PrometheusMetricsTest.java | 6 ++-- 9 files changed, 103 insertions(+), 30 deletions(-) diff --git a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java index 2e062b87a8325..652a922b9a5ad 100644 --- a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java +++ b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java @@ -43,6 +43,15 @@ public class AuthenticationProviderAthenz implements AuthenticationProvider { private List domainNameList = null; private int allowedOffset = 30; + public enum ErrorCode { + UNKNOWN, + NO_CLIENT, + NO_TOKEN, + NO_PUBLIC_KEY, + DOMAIN_MISMATCH, + INVALID_TOKEN, + } + @Override public void initialize(ServiceConfiguration config) throws IOException { String domainNames; @@ -81,11 +90,13 @@ public String getAuthMethodName() { public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { SocketAddress clientAddress; String roleToken; + ErrorCode errorCode = ErrorCode.UNKNOWN; try { if (authData.hasDataFromPeer()) { clientAddress = authData.getPeerAddress(); } else { + errorCode = ErrorCode.NO_CLIENT; throw new AuthenticationException("Authentication data source does not have a client address"); } @@ -94,13 +105,16 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat } else if (authData.hasDataFromHttp()) { roleToken = authData.getHttpHeader(AuthZpeClient.ZPE_TOKEN_HDR); } else { + errorCode = ErrorCode.NO_TOKEN; throw new AuthenticationException("Authentication data source does not have a role token"); } if (roleToken == null) { + errorCode = ErrorCode.NO_TOKEN; throw new AuthenticationException("Athenz token is null, can't authenticate"); } if (roleToken.isEmpty()) { + errorCode = ErrorCode.NO_TOKEN; throw new AuthenticationException("Athenz RoleToken is empty, Server is Using Athenz Authentication"); } if (log.isDebugEnabled()) { @@ -110,6 +124,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat RoleToken token = new RoleToken(roleToken); if (!domainNameList.contains(token.getDomain())) { + errorCode = ErrorCode.DOMAIN_MISMATCH; throw new AuthenticationException( String.format("Athenz RoleToken Domain mismatch, Expected: %s, Found: %s", domainNameList.toString(), token.getDomain())); @@ -120,6 +135,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat PublicKey ztsPublicKey = AuthZpeClient.getZtsPublicKey(token.getKeyId()); if (ztsPublicKey == null) { + errorCode = ErrorCode.NO_PUBLIC_KEY; throw new AuthenticationException("Unable to retrieve ZTS Public Key"); } @@ -128,13 +144,13 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); return token.getPrincipal(); } else { + errorCode = ErrorCode.INVALID_TOKEN; throw new AuthenticationException( String.format("Athenz Role Token Not Authenticated from Client: %s", clientAddress)); } } } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(errorCode); throw exception; } } diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java index ae4774b6f6b6b..00ec09bd1817f 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java @@ -447,7 +447,7 @@ DecodedJWT verifyJWT(PublicKey publicKey, } static void incrementFailureMetric(AuthenticationExceptionCode code) { - AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, "token", code.toString()); + AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, AUTH_METHOD_NAME, code); } /** diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java index 109259537a494..7862a35b5e871 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java @@ -30,6 +30,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.util.FutureUtil; @@ -143,6 +144,10 @@ default CompletableFuture authenticateHttpRequestAsync(HttpServletReque } } + default void incrementFailureMetric(Enum errorCode) { + AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), errorCode); + } + /** * Set response, according to passed in request. * and return whether we should do following chain.doFilter or not. diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java index 3c4759fec7da9..ca5150c9bdb60 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java @@ -46,6 +46,14 @@ public class AuthenticationProviderBasic implements AuthenticationProvider { private static final String CONF_PULSAR_PROPERTY_KEY = "basicAuthConf"; private Map users; + private enum ErrorCode { + UNKNOWN, + EMPTY_AUTH_DATA, + INVALID_HEADER, + INVALID_AUTH_DATA, + INVALID_TOKEN, + } + @Override public void close() throws IOException { // noop @@ -104,9 +112,10 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat String userId = authParams.getUserId(); String password = authParams.getPassword(); String msg = "Unknown user or invalid password"; - + ErrorCode errorCode = ErrorCode.UNKNOWN; try { if (users.get(userId) == null) { + errorCode = ErrorCode.INVALID_AUTH_DATA; throw new AuthenticationException(msg); } @@ -117,15 +126,16 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat List splitEncryptedPassword = Arrays.asList(encryptedPassword.split("\\$")); if (splitEncryptedPassword.size() != 4 || !encryptedPassword .equals(Md5Crypt.apr1Crypt(password.getBytes(), splitEncryptedPassword.get(2)))) { + errorCode = ErrorCode.INVALID_TOKEN; throw new AuthenticationException(msg); } // For crypt algorithm } else if (!encryptedPassword.equals(Crypt.crypt(password.getBytes(), encryptedPassword.substring(0, 2)))) { + errorCode = ErrorCode.INVALID_TOKEN; throw new AuthenticationException(msg); } } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(errorCode); throw exception; } AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); @@ -144,24 +154,29 @@ public AuthParams(AuthenticationDataSource authData) throws AuthenticationExcept String rawAuthToken = authData.getHttpHeader(HTTP_HEADER_NAME); // parsing and validation if (StringUtils.isBlank(rawAuthToken) || !rawAuthToken.toUpperCase().startsWith("BASIC ")) { + incrementFailureMetric(ErrorCode.INVALID_HEADER); throw new AuthenticationException("Authentication token has to be started with \"Basic \""); } String[] splitRawAuthToken = rawAuthToken.split(" "); if (splitRawAuthToken.length != 2) { + incrementFailureMetric(ErrorCode.INVALID_HEADER); throw new AuthenticationException("Base64 encoded token is not found"); } try { authParams = new String(Base64.getDecoder().decode(splitRawAuthToken[1])); } catch (Exception e) { + incrementFailureMetric(ErrorCode.INVALID_HEADER); throw new AuthenticationException("Base64 decoding is failure: " + e.getMessage()); } } else { + incrementFailureMetric(ErrorCode.EMPTY_AUTH_DATA); throw new AuthenticationException("Authentication data source does not have data"); } String[] parsedAuthParams = authParams.split(":"); if (parsedAuthParams.length != 2) { + incrementFailureMetric(ErrorCode.INVALID_AUTH_DATA); throw new AuthenticationException("Base64 decoded params are invalid"); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java index 16d6f9859a0b4..663a6253f4460 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java @@ -44,9 +44,15 @@ private interface AuthProcessor { } + private enum ErrorCode { + UNKNOWN, + AUTH_REQUIRED, + } + static T applyAuthProcessor(List processors, AuthProcessor authFunc) throws AuthenticationException { AuthenticationException authenticationException = null; + String errorCode = ErrorCode.UNKNOWN.name(); for (W ap : processors) { try { return authFunc.apply(ap); @@ -56,19 +62,19 @@ static T applyAuthProcessor(List processors, AuthProcessor authF } // Store the exception so we can throw it later instead of a generic one authenticationException = ae; + errorCode = ap.getClass().getSimpleName() + "-INVALID-AUTH"; } } if (null == authenticationException) { AuthenticationMetrics.authenticateFailure( AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", "Authentication required"); + "authentication-provider-list", ErrorCode.AUTH_REQUIRED); throw new AuthenticationException("Authentication required"); } else { - AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", - authenticationException.getMessage() != null - ? authenticationException.getMessage() : "Authentication required"); + AuthenticationMetrics.authenticateFailure( + AuthenticationProviderList.class.getSimpleName(), + "authentication-provider-list", errorCode); throw authenticationException; } @@ -129,7 +135,7 @@ private void authenticateRemainingAuthStates(CompletableFuture authCha previousException = new AuthenticationException("Authentication required"); } AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", "Authentication required"); + "authentication-provider-list", ErrorCode.AUTH_REQUIRED); authChallengeFuture.completeExceptionally(previousException); return; } @@ -235,7 +241,7 @@ private void authenticateRemainingAuthProviders(CompletableFuture roleFu previousException = new AuthenticationException("Authentication required"); } AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", "Authentication required"); + "authentication-provider-list", ErrorCode.AUTH_REQUIRED); roleFuture.completeExceptionally(previousException); return; } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java index 47e5316bce9ce..a4c44121b4b96 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java @@ -27,6 +27,12 @@ public class AuthenticationProviderTls implements AuthenticationProvider { + private enum ErrorCode { + UNKNOWN, + INVALID_CERTS, + INVALID_CN, // invalid common name + } + @Override public void close() throws IOException { // noop @@ -45,6 +51,7 @@ public String getAuthMethodName() { @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { String commonName = null; + ErrorCode errorCode = ErrorCode.UNKNOWN; try { if (authData.hasDataFromTls()) { /** @@ -72,6 +79,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat // CN=Steve Kille,O=Isode Limited,C=GB Certificate[] certs = authData.getTlsCertificates(); if (null == certs) { + errorCode = ErrorCode.INVALID_CERTS; throw new AuthenticationException("Failed to get TLS certificates from client"); } String distinguishedName = ((X509Certificate) certs[0]).getSubjectX500Principal().getName(); @@ -85,12 +93,12 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat } if (commonName == null) { + errorCode = ErrorCode.INVALID_CN; throw new AuthenticationException("Client unable to authenticate with TLS certificate"); } AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(errorCode); throw exception; } return commonName; diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java index 67e1f39a38924..f8992b21ff49f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java @@ -106,6 +106,12 @@ public class AuthenticationProviderToken implements AuthenticationProvider { private String confTokenAudienceSettingName; private String confTokenAllowedClockSkewSecondsSettingName; + public enum ErrorCode { + INVALID_AUTH_DATA, + INVALID_TOKEN, + INVALID_AUDIENCES, + } + @Override public void close() throws IOException { // noop @@ -158,19 +164,18 @@ public String getAuthMethodName() { @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { + String token; try { // Get Token - String token; token = getToken(authData); - // Parse Token by validating - String role = getPrincipal(authenticateToken(token)); - AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); - return role; } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(ErrorCode.INVALID_AUTH_DATA); throw exception; } + // Parse Token by validating + String role = getPrincipal(authenticateToken(token)); + AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + return role; } @Override @@ -241,16 +246,19 @@ private Jwt authenticateToken(final String token) throws Authenticati List audiences = (List) object; // audience not contains this broker, throw exception. if (audiences.stream().noneMatch(audienceInToken -> audienceInToken.equals(audience))) { - throw new AuthenticationException("Audiences in token: [" + String.join(", ", audiences) - + "] not contains this broker: " + audience); + incrementFailureMetric(ErrorCode.INVALID_AUDIENCES); + throw new AuthenticationException("Audiences in token: [" + + String.join(", ", audiences) + "] not contains this broker: " + audience); } } else if (object instanceof String) { if (!object.equals(audience)) { - throw new AuthenticationException("Audiences in token: [" + object - + "] not contains this broker: " + audience); + incrementFailureMetric(ErrorCode.INVALID_AUDIENCES); + throw new AuthenticationException( + "Audiences in token: [" + object + "] not contains this broker: " + audience); } } else { // should not reach here. + incrementFailureMetric(ErrorCode.INVALID_AUDIENCES); throw new AuthenticationException("Audiences in token is not in expected format: " + object); } } @@ -264,6 +272,7 @@ private Jwt authenticateToken(final String token) throws Authenticati if (e instanceof ExpiredJwtException) { expiredTokenMetrics.inc(); } + incrementFailureMetric(ErrorCode.INVALID_TOKEN); throw new AuthenticationException("Failed to authentication token: " + e.getMessage()); } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java index 0a095235f3cb3..5faaccbe15716 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java @@ -43,11 +43,27 @@ public static void authenticateSuccess(String providerName, String authMethod) { /** * Log authenticate failure event to the authentication metrics. + * + * This method is deprecated due to the label "reason" is a potential infinite value. + * @deprecated See {@link #authenticateFailure(String, String, Enum)} ()} + * * @param providerName The short class name of the provider * @param authMethod Authentication method name. * @param reason Failure reason. */ + @Deprecated public static void authenticateFailure(String providerName, String authMethod, String reason) { authFailuresMetrics.labels(providerName, authMethod, reason).inc(); } + + /** + * Log authenticate failure event to the authentication metrics. + * @param providerName The short class name of the provider + * @param authMethod Authentication method name. + * @param errorCode Error code. + */ + public static void authenticateFailure(String providerName, String authMethod, Enum errorCode) { + authFailuresMetrics.labels(providerName, authMethod, errorCode.name()).inc(); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index bf141e10aa1b9..7a23e56fe0b41 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1379,15 +1379,12 @@ public void testAuthMetrics() throws IOException, AuthenticationException { conf.setProperties(properties); provider.initialize(conf); - String authExceptionMessage = ""; - try { provider.authenticate(new AuthenticationDataSource() { }); fail("Should have failed"); } catch (AuthenticationException e) { // expected, no credential passed - authExceptionMessage = e.getMessage(); } String token = AuthTokenUtils.createToken(secretKey, "subject", Optional.empty()); @@ -1424,7 +1421,8 @@ public String getCommandData() { boolean haveFailed = false; for (Metric metric : cm) { if (Objects.equals(metric.tags.get("auth_method"), "token") - && Objects.equals(metric.tags.get("reason"), authExceptionMessage) + && Objects.equals(metric.tags.get("reason"), + AuthenticationProviderToken.ErrorCode.INVALID_AUTH_DATA.name()) && Objects.equals(metric.tags.get("provider_name"), provider.getClass().getSimpleName())) { haveFailed = true; } From 0dd238abf3798ba1f5b83182c1c4fb36e6ba6511 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Fri, 5 May 2023 15:31:03 +0800 Subject: [PATCH 166/280] [cleanup][build] Bumped version to 3.1.0-SNAPSHOT (#20216) Signed-off-by: Zike Yang --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 132 files changed, 134 insertions(+), 134 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index cc7e7952b69f0..d5882b4659528 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 44f0ada4630d7..3b8c6754c3fa9 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index bd5d64bd84191..a07e5e19907f2 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 9f8ace79b77c3..daefeb83b5371 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index a21618004e31a..46099df4d39ff 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,12 +31,12 @@ org.apache.pulsar buildtools - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT jar Pulsar Build Tools - ${maven.build.timestamp} + 2023-05-03T02:53:27Z 1.8 1.8 3.0.0-M3 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 99105bef950d5..568d76922bf4e 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 1e86758ed5a6a..d23ebec2ef26d 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 9782f269284bf..36a3fa1c5835a 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 6b225bfc00c0b..f804c9c54b9cd 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index b38baee4257ba..9e3134a75e5bf 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/docker/pom.xml b/docker/pom.xml index 4d3b05fe33a76..882240925ef24 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 2edd6c776dcf5..7a2f492632135 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 8c3b868315535..e1c1503a3f381 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index d4138ea041317..dfb155c2d5a7d 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 2a7b9c576d318..a8cb560b7b376 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index c5d7092c7b4ec..5e19823ffea77 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - ${maven.build.timestamp} + 2023-05-03T02:53:27Z true 3.2.13 1.1.1 - 7.7.0 + 7.7.1 3.12.4 3.25.0-GA 1.5.0 @@ -273,14 +273,13 @@ flexible messaging model and an intuitive client API. 3.0.0 4.1 1.0 - 3.1.0 + 3.3.0 - - 3.0.0-M3 - 3.4.2 - 3.10.1 - 3.4.0 + 3.1.0 + 3.5.0 + 3.11.0 + 3.5.0 2.3.0 3.4.1 3.1.0 @@ -288,7 +287,7 @@ flexible messaging model and an intuitive client API. 1.3.4 3.1.2 4.0.2 - 3.4.3 + 3.5.3 1.7.0 0.8.8 4.7.3.0 @@ -1489,7 +1488,6 @@ flexible messaging model and an intuitive client API. UTF-8 true true - true false @@ -1542,6 +1540,13 @@ flexible messaging model and an intuitive client API. + + + org.apache.maven.surefire + surefire-testng + ${surefire.version} + + diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index 3aba6f55d99e1..a3a3de19202c2 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -64,13 +64,6 @@ true - - org.sonatype.plugins - nexus-staging-maven-plugin - - true - - From ea56197d18d8d1f5b06c26246d920cb736d4e6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 10 May 2023 13:26:16 +0200 Subject: [PATCH 181/280] [fix][monitor] topic with double quote breaks the prometheus format (#20230) --- .../prometheus/PrometheusMetricStreams.java | 6 ++++- .../broker/stats/PrometheusMetricsTest.java | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java index df107b6d7d9c4..93cbad4e19503 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java @@ -41,7 +41,11 @@ void writeSample(String metricName, Number value, String... labelsAndValuesArray SimpleTextOutputStream stream = initGaugeType(metricName); stream.write(metricName).write('{'); for (int i = 0; i < labelsAndValuesArray.length; i += 2) { - stream.write(labelsAndValuesArray[i]).write("=\"").write(labelsAndValuesArray[i + 1]).write('\"'); + String labelValue = labelsAndValuesArray[i + 1]; + if (labelValue != null) { + labelValue = labelValue.replace("\"", "\\\""); + } + stream.write(labelsAndValuesArray[i]).write("=\"").write(labelValue).write('\"'); if (i + 2 != labelsAndValuesArray.length) { stream.write(','); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 7a23e56fe0b41..a7a28afd8ac64 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1953,4 +1953,30 @@ public String toString() { } } + @Test + public void testEscapeLabelValue() throws Exception { + String ns1 = "prop/ns-abc1"; + admin.namespaces().createNamespace(ns1); + String topic = "persistent://" + ns1 + "/\"mytopic"; + admin.topics().createNonPartitionedTopic(topic); + + @Cleanup + final Consumer consumer = pulsarClient.newConsumer() + .subscriptionName("sub") + .topic(topic) + .subscribe(); + @Cleanup + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, false, + false, statsOut); + String metricsStr = statsOut.toString(); + final List subCountLines = metricsStr.lines() + .filter(line -> line.startsWith("pulsar_subscriptions_count")) + .collect(Collectors.toList()); + System.out.println(subCountLines); + assertEquals(subCountLines.size(), 1); + assertEquals(subCountLines.get(0), + "pulsar_subscriptions_count{cluster=\"test\",namespace=\"prop/ns-abc1\",topic=\"persistent://prop/ns-abc1/\\\"mytopic\"} 1"); + } + } From c32caba54a5ab2c01c092ab952343f4ac405da35 Mon Sep 17 00:00:00 2001 From: Raghavender Mittapalli <14803749+syk-coder@users.noreply.github.com> Date: Wed, 10 May 2023 18:05:17 +0530 Subject: [PATCH 182/280] [fix][broker] Fix default bundle size used while setting bookie affinity (#20250) --- .../org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 2 +- .../pulsar/broker/service/BrokerBookieIsolationTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 19f8d7c437ded..8c72d0b0286d3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -883,7 +883,7 @@ protected void internalSetBookieAffinityGroup(BookieAffinityGroupData bookieAffi policies -> new LocalPolicies(policies.bundles, bookieAffinityGroup, policies.namespaceAntiAffinityGroup)) - .orElseGet(() -> new LocalPolicies(defaultBundle(), + .orElseGet(() -> new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()), bookieAffinityGroup, null)); log.info("[{}] Successfully updated local-policies configuration: namespace={}, map={}", clientAppId(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java index 36d4053ae497f..951892f4ebfbc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java @@ -156,6 +156,7 @@ public void testBookieIsolation() throws Exception { config.setBrokerServicePort(Optional.of(0)); config.setAdvertisedAddress("localhost"); config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups); + config.setDefaultNumberOfNamespaceBundles(8); config.setManagedLedgerDefaultEnsembleSize(2); config.setManagedLedgerDefaultWriteQuorum(2); @@ -207,6 +208,9 @@ public void testBookieIsolation() throws Exception { .bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups) .build()); + //Checks the namespace bundles after setting the bookie affinity + assertEquals(admin.namespaces().getBundles(ns2).getNumBundles(), config.getDefaultNumberOfNamespaceBundles()); + try { admin.namespaces().getBookieAffinityGroup(ns1); fail("ns1 should have no bookie affinity group set"); From f04252b84a8b594ea371645e367c491aa8e2dd80 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 10 May 2023 21:15:37 +0800 Subject: [PATCH 183/280] [fix][fn] Support multiple input topics for Go runtime (#20000) --- pulsar-function-go/conf/conf.go | 18 +- pulsar-function-go/conf/conf.yaml | 6 +- pulsar-function-go/pf/instance.go | 4 +- pulsar-function-go/pf/instanceConf.go | 31 ++-- .../instance/go/GoInstanceConfig.java | 5 + .../src/main/resources/findbugsExclude.xml | 10 ++ .../functions/runtime/RuntimeUtils.java | 13 +- .../functions/PulsarFunctionsTest.java | 160 +++++++++++------- .../functions/PulsarFunctionsTestBase.java | 2 + .../functions/go/PulsarFunctionsGoTest.java | 5 + .../java/PulsarFunctionsJavaTest.java | 4 +- .../python/PulsarFunctionsPythonTest.java | 8 +- 12 files changed, 179 insertions(+), 87 deletions(-) diff --git a/pulsar-function-go/conf/conf.go b/pulsar-function-go/conf/conf.go index c2ff443fcc40c..d52b886b540f9 100644 --- a/pulsar-function-go/conf/conf.go +++ b/pulsar-function-go/conf/conf.go @@ -50,8 +50,8 @@ type Conf struct { SecretsMap string `json:"secretsMap" yaml:"secretsMap"` Runtime int32 `json:"runtime" yaml:"runtime"` //Deprecated - AutoACK bool `json:"autoAck" yaml:"autoAck"` - Parallelism int32 `json:"parallelism" yaml:"parallelism"` + AutoACK bool `json:"autoAck" yaml:"autoAck"` + Parallelism int32 `json:"parallelism" yaml:"parallelism"` //source config SubscriptionType int32 `json:"subscriptionType" yaml:"subscriptionType"` TimeoutMs uint64 `json:"timeoutMs" yaml:"timeoutMs"` @@ -59,10 +59,16 @@ type Conf struct { CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"` SubscriptionPosition int32 `json:"subscriptionPosition" yaml:"subscriptionPosition"` //source input specs - SourceSpecTopic string `json:"sourceSpecsTopic" yaml:"sourceSpecsTopic"` - SourceSchemaType string `json:"sourceSchemaType" yaml:"sourceSchemaType"` - IsRegexPatternSubscription bool `json:"isRegexPatternSubscription" yaml:"isRegexPatternSubscription"` - ReceiverQueueSize int32 `json:"receiverQueueSize" yaml:"receiverQueueSize"` + SourceInputSpecs map[string]string `json:"sourceInputSpecs" yaml:"sourceInputSpecs"` + // for backward compatibility + // Deprecated + SourceSpecTopic string `json:"sourceSpecsTopic" yaml:"sourceSpecsTopic"` + // Deprecated + SourceSchemaType string `json:"sourceSchemaType" yaml:"sourceSchemaType"` + // Deprecated + IsRegexPatternSubscription bool `json:"isRegexPatternSubscription" yaml:"isRegexPatternSubscription"` + // Deprecated + ReceiverQueueSize int32 `json:"receiverQueueSize" yaml:"receiverQueueSize"` //sink spec config SinkSpecTopic string `json:"sinkSpecsTopic" yaml:"sinkSpecsTopic"` SinkSchemaType string `json:"sinkSchemaType" yaml:"sinkSchemaType"` diff --git a/pulsar-function-go/conf/conf.yaml b/pulsar-function-go/conf/conf.yaml index 59ac9bbd51308..098e33cda1f57 100644 --- a/pulsar-function-go/conf/conf.yaml +++ b/pulsar-function-go/conf/conf.yaml @@ -43,10 +43,8 @@ subscriptionName: "" cleanupSubscription: false subscriptionPosition: 1 # source input specs -sourceSpecsTopic: persistent://public/default/topic-01 -sourceSchemaType: "" -isRegexPatternSubscription: false -receiverQueueSize: 10 +sourceInputSpecs: + persistent://public/default/topic-01: "{\"schemaType\": \"\", \"isRegexPattern\": false, \"receiverQueueSize\": {\"value\": 10}}" # sink specs config sinkSpecsTopic: persistent://public/default/topic-02 sinkSchemaType: "" diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index a82273031ecfb..138489444d160 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -439,7 +439,6 @@ func (gi *goInstance) addLogTopicHandler() { }() if gi.context.logAppender == nil { - log.Error("the logAppender is nil, if you want to use it, please specify `--log-topic` at startup.") return } @@ -571,6 +570,9 @@ func (gi *goInstance) getMatchingMetricFunc() func(lbl *prometheus_client.LabelP func (gi *goInstance) getMatchingMetricFromRegistry(metricName string) prometheus_client.Metric { filteredMetricFamilies := gi.getFilteredMetricFamilies(metricName) + if len(filteredMetricFamilies) == 0 { + return prometheus_client.Metric{} + } metricFunc := gi.getMatchingMetricFunc() matchingMetric := getFirstMatch(filteredMetricFamilies[0].Metric, metricFunc) return *matchingMetric diff --git a/pulsar-function-go/pf/instanceConf.go b/pulsar-function-go/pf/instanceConf.go index d60beef29e8d3..9d4cabfae5a9b 100644 --- a/pulsar-function-go/pf/instanceConf.go +++ b/pulsar-function-go/pf/instanceConf.go @@ -20,6 +20,7 @@ package pf import ( + "encoding/json" "fmt" "time" @@ -44,6 +45,24 @@ type instanceConf struct { } func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { + inputSpecs := make(map[string]*pb.ConsumerSpec) + // for backward compatibility + if cfg.SourceSpecTopic != "" { + inputSpecs[cfg.SourceSpecTopic] = &pb.ConsumerSpec{ + SchemaType: cfg.SourceSchemaType, + IsRegexPattern: cfg.IsRegexPatternSubscription, + ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{ + Value: cfg.ReceiverQueueSize, + }, + } + } + for topic, value := range cfg.SourceInputSpecs { + spec := &pb.ConsumerSpec{} + if err := json.Unmarshal([]byte(value), spec); err != nil { + panic(fmt.Sprintf("Failed to unmarshal consume specs: %v", err)) + } + inputSpecs[topic] = spec + } instanceConf := &instanceConf{ instanceID: cfg.InstanceID, funcID: cfg.FuncID, @@ -66,16 +85,8 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { AutoAck: cfg.AutoACK, Parallelism: cfg.Parallelism, Source: &pb.SourceSpec{ - SubscriptionType: pb.SubscriptionType(cfg.SubscriptionType), - InputSpecs: map[string]*pb.ConsumerSpec{ - cfg.SourceSpecTopic: { - SchemaType: cfg.SourceSchemaType, - IsRegexPattern: cfg.IsRegexPatternSubscription, - ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{ - Value: cfg.ReceiverQueueSize, - }, - }, - }, + SubscriptionType: pb.SubscriptionType(cfg.SubscriptionType), + InputSpecs: inputSpecs, TimeoutMs: cfg.TimeoutMs, SubscriptionName: cfg.SubscriptionName, CleanupSubscription: cfg.CleanupSubscription, diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java index abf17bdb1656d..67fe2a41d553d 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.instance.go; +import java.util.Map; import lombok.Getter; import lombok.Setter; import org.apache.pulsar.functions.proto.Function; @@ -53,6 +54,10 @@ public class GoInstanceConfig { private boolean cleanupSubscription; private int subscriptionPosition = Function.SubscriptionPosition.LATEST.getNumber(); + // value is the json string of ConsumerSpec + private Map sourceInputSpecs; + + // for backward compatibility private String sourceSpecsTopic = ""; private String sourceSchemaType = ""; private boolean isRegexPatternSubscription; diff --git a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml index 027affbfeb2ec..7fe247d2ab20a 100644 --- a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml @@ -542,4 +542,14 @@ + + + + + + + + + + diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 5392697e9283b..1d2eacbd77b12 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -38,6 +38,7 @@ import java.net.InetAddress; import java.net.URL; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -204,9 +205,15 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig, instanceConfig.getFunctionDetails().getSource().getSubscriptionPosition().getNumber()); if (instanceConfig.getFunctionDetails().getSource().getInputSpecsMap() != null) { - for (String inputTopic : instanceConfig.getFunctionDetails().getSource().getInputSpecsMap().keySet()) { - goInstanceConfig.setSourceSpecsTopic(inputTopic); + Map sourceInputSpecs = new HashMap<>(); + for (Map.Entry entry : + instanceConfig.getFunctionDetails().getSource().getInputSpecsMap().entrySet()) { + String topic = entry.getKey(); + Function.ConsumerSpec spec = entry.getValue(); + sourceInputSpecs.put(topic, JsonFormat.printer().omittingInsignificantWhitespace().print(spec)); + goInstanceConfig.setSourceSpecsTopic(topic); } + goInstanceConfig.setSourceInputSpecs(sourceInputSpecs); } if (instanceConfig.getFunctionDetails().getSource().getTimeoutMs() != 0) { @@ -304,7 +311,7 @@ public static List getCmd(InstanceConfig instanceConfig, } if (StringUtils.isNotEmpty(functionInstanceClassPath)) { - args.add(String.format("-D%s=%s", FUNCTIONS_INSTANCE_CLASSPATH, functionInstanceClassPath)); + args.add(String.format("-D%s=%s", FUNCTIONS_INSTANCE_CLASSPATH, functionInstanceClassPath)); } else { // add complete classpath for broker/worker so that the function instance can load // the functions instance dependencies separately from user code dependencies diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index eaf66974b982a..6088628aac52b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -29,10 +29,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -126,13 +129,14 @@ protected Map produceMessagesToInputTopic(String inputTopicName, return kvs; } - protected void testFunctionLocalRun(Runtime runtime) throws Exception { + protected void testFunctionLocalRun(Runtime runtime) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD) { return; } - String inputTopicName = "persistent://public/default/test-function-local-run-" + runtime + "-input-" + randomName(8); + String inputTopicName = + "persistent://public/default/test-function-local-run-" + runtime + "-input-" + randomName(8); String outputTopicName = "test-function-local-run-" + runtime + "-output-" + randomName(8); final int numMessages = 10; @@ -377,7 +381,8 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { if (runtime == Runtime.PYTHON) { submitFunction( - runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, EXCEPTION_PYTHON_CLASS, schema); + runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, + EXCEPTION_PYTHON_CLASS, schema); } else { submitFunction( runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema); @@ -550,7 +555,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { final int numMessages = 10; // submit the exclamation function - switch (runtime){ + switch (runtime) { case JAVA: submitFunction( runtime, @@ -622,7 +627,8 @@ protected void testPublishFunction(Runtime runtime) throws Exception { .create(); for (int i = 0; i < numMessages; i++) { - producer.newMessage().key(String.valueOf(i)).property("count", String.valueOf(i)).value(("message-" + i).getBytes(UTF_8)).send(); + producer.newMessage().key(String.valueOf(i)).property("count", String.valueOf(i)) + .value(("message-" + i).getBytes(UTF_8)).send(); } Set expectedMessages = new HashSet<>(); @@ -662,9 +668,10 @@ protected void testPublishFunction(Runtime runtime) throws Exception { protected void testExclamationFunction(Runtime runtime, boolean isTopicPattern, boolean pyZip, + boolean multipleInput, boolean withExtraDeps) throws Exception { - if (functionRuntimeType == FunctionRuntimeType.THREAD && runtime == Runtime.PYTHON) { - // python can only run on process mode + if (functionRuntimeType == FunctionRuntimeType.THREAD && (runtime == Runtime.PYTHON || runtime == Runtime.GO)) { + // python&go can only run on process mode return; } @@ -683,22 +690,9 @@ protected void testExclamationFunction(Runtime runtime, admin.topics().createNonPartitionedTopic(outputTopicName); } if (isTopicPattern) { - @Cleanup PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup Consumer consumer1 = client.newConsumer(schema) - .topic(inputTopicName + "1") - .subscriptionType(SubscriptionType.Exclusive) - .subscriptionName("test-sub") - .subscribe(); - - @Cleanup Consumer consumer2 = client.newConsumer(schema) - .topic(inputTopicName + "2") - .subscriptionType(SubscriptionType.Exclusive) - .subscriptionName("test-sub") - .subscribe(); inputTopicName = inputTopicName + ".*"; + } else if (multipleInput) { + inputTopicName = inputTopicName + "1," + inputTopicName + "2"; } String functionName = "test-exclamation-fn-" + randomName(8); final int numMessages = 10; @@ -725,8 +719,11 @@ protected void testExclamationFunction(Runtime runtime, // get function status getFunctionStatus(functionName, numMessages, true); - // get function stats - getFunctionStats(functionName, numMessages); + if (Runtime.GO != runtime) { + // TODO: Go runtime doesn't collect `process_latency_ms_1min` metric + // get function stats + getFunctionStats(functionName, numMessages); + } // update parallelism updateFunctionParallelism(functionName, 2); @@ -787,6 +784,12 @@ private void submitFunction(Runtime runtime, } else { file = EXCLAMATION_PYTHON_FILE; } + } else if (Runtime.GO == runtime) { + if (isPublishFunction) { + file = PUBLISH_FUNCTION_GO_FILE; + } else { + file = EXCLAMATION_GO_FILE; + } } submitFunction(runtime, inputTopicName, outputTopicName, functionName, file, functionClass, inputTopicSchema); @@ -819,7 +822,7 @@ private void submitFunction(Runtime runtime, if (StringUtils.isNotEmpty(inputTopicName)) { ensureSubscriptionCreated( - inputTopicName, String.format("public/default/%s", functionName), inputTopicSchema); + inputTopicName, String.format("public/default/%s", functionName), inputTopicSchema); } CommandGenerator generator; @@ -853,7 +856,7 @@ private void submitFunction(Runtime runtime, } String command = ""; - switch (runtime){ + switch (runtime) { case JAVA: command = generator.generateCreateFunctionCommand(); break; @@ -945,8 +948,17 @@ private void ensureSubscriptionCreated(String inputTopicName, try (PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) .build()) { + List topics = new ArrayList<>(); + if (inputTopicName.endsWith(".*")) { + topics.add(inputTopicName.substring(0, inputTopicName.length() - 2) + "1"); + topics.add(inputTopicName.substring(0, inputTopicName.length() - 2) + "2"); + } else if (inputTopicName.contains(",")) { + topics.addAll(Arrays.asList(inputTopicName.split(","))); + } else { + topics.add(inputTopicName); + } try (Consumer ignored = client.newConsumer(inputTopicSchema) - .topic(inputTopicName) + .topic(topics.toArray(new String[0])) .subscriptionType(SubscriptionType.Shared) .subscriptionName(subscriptionName) .subscribe()) { @@ -1042,7 +1054,8 @@ private void getFunctionStats(String functionName, int numMessages) throws Excep assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0); assertTrue(functionStats.instances.get(0).getMetrics().getAvgProcessLatency() > 0); assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getReceivedTotal(), numMessages); - assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), numMessages); + assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), + numMessages); assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getSystemExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getUserExceptionsTotal(), 0); assertTrue(functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency() > 0); @@ -1071,19 +1084,29 @@ private void getFunctionInfoNotFound(String functionName) throws Exception { } private void checkSubscriptionsCleanup(String topic) throws Exception { - try { - ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd( - PulsarCluster.ADMIN_SCRIPT, - "topics", - "stats", - topic); - TopicStats topicStats = ObjectMapperFactory.getMapper().reader() - .readValue(result.getStdout(), TopicStats.class); - assertEquals(topicStats.getSubscriptions().size(), 0); - - } catch (ContainerExecException e) { - fail("Command should have exited with non-zero"); + List topics = new ArrayList<>(); + if (topic.endsWith(".*")) { + topics.add(topic.substring(0, topic.length() - 2) + "1"); + topics.add(topic.substring(0, topic.length() - 2) + "2"); + } else if (topic.contains(",")) { + topics.addAll(Arrays.asList(topic.split(","))); + } else { + topics.add(topic); } + topics.stream().forEach(t -> { + try { + ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd( + PulsarCluster.ADMIN_SCRIPT, + "topics", + "stats", + t); + TopicStats topicStats = ObjectMapperFactory.getMapper().reader() + .readValue(result.getStdout(), TopicStats.class); + assertEquals(topicStats.getSubscriptions().size(), 0); + } catch (Exception e) { + fail("Command should have exited with non-zero"); + } + }); } private void checkPublisherCleanup(String topic) throws Exception { @@ -1145,7 +1168,8 @@ private void doGetFunctionStatus(String functionName, int numMessages, boolean c lastInvocationTimeGreaterThanZero = lastInvocationTimeGreaterThanZero || functionStatus.getInstances().get(i).getStatus().getLastInvocationTime() > 0; totalMessagesProcessed += functionStatus.getInstances().get(i).getStatus().getNumReceived(); - totalMessagesSuccessfullyProcessed += functionStatus.getInstances().get(i).getStatus().getNumSuccessfullyProcessed(); + totalMessagesSuccessfullyProcessed += + functionStatus.getInstances().get(i).getStatus().getNumSuccessfullyProcessed(); if (checkRestarts) { assertEquals(functionStatus.getInstances().get(i).getStatus().getNumRestarts(), 0); } @@ -1238,6 +1262,23 @@ private void publishAndConsumeMessagesBytes(String inputTopic, producer1.send(("message-" + i).getBytes(UTF_8)); } + for (int i = numMessages / 2; i < numMessages; i++) { + producer2.send(("message-" + i).getBytes(UTF_8)); + } + } else if (inputTopic.contains(",")) { + String[] topics = inputTopic.split(","); + @Cleanup Producer producer1 = client.newProducer(Schema.BYTES) + .topic(topics[0]) + .create(); + + @Cleanup Producer producer2 = client.newProducer(Schema.BYTES) + .topic(topics[1]) + .create(); + + for (int i = 0; i < numMessages / 2; i++) { + producer1.send(("message-" + i).getBytes(UTF_8)); + } + for (int i = numMessages / 2; i < numMessages; i++) { producer2.send(("message-" + i).getBytes(UTF_8)); } @@ -1420,7 +1461,8 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { AVRO_SCHEMA_FUNCTION_PYTHON_FILE, AVRO_SCHEMA_PYTHON_CLASS, Schema.AVRO(AvroTestObject.class), - null, objectMapper.writeValueAsString(inputSpecs), "avro", null, "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject"); + null, objectMapper.writeValueAsString(inputSpecs), "avro", null, + "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject"); } log.info("pulsar submitFunction"); @@ -1430,7 +1472,7 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { Set expectedSet = new HashSet<>(); log.info("test-avro-schema producer connected: " + producer.isConnected()); - for (int i = 0 ; i < numMessages ; i++) { + for (int i = 0; i < numMessages; i++) { AvroTestObject inputObject = new AvroTestObject(); inputObject.setBaseValue(i); MessageId messageId = producer.send(inputObject); @@ -1457,7 +1499,7 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { }); log.info("test-avro-schema consumer connected: " + consumer.isConnected()); - for (int i = 0 ; i < numMessages ; i++) { + for (int i = 0; i < numMessages; i++) { log.info("test-avro-schema consumer receive [{}] start", i); Message message = consumer.receive(); log.info("test-avro-schema consumer receive [{}] over", i); @@ -1495,7 +1537,8 @@ protected void testInitFunction(Runtime runtime) throws Exception { final int numMessages = 10; // submit the exclamation function - submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, InitializableFunction.class.getName(), schema, + submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, + InitializableFunction.class.getName(), schema, Collections.singletonMap("publish-topic", outputTopicName), null, null, null, null, null); // publish and consume result @@ -1650,7 +1693,8 @@ private void publishAndConsumeMessages(String inputTopic, } - protected void testGenericObjectFunction(String function, boolean removeAgeField, boolean keyValue) throws Exception { + protected void testGenericObjectFunction(String function, boolean removeAgeField, boolean keyValue) + throws Exception { log.info("start {} function test ...", function); String ns = "public/ns-genericobject-" + randomName(8); @@ -1721,13 +1765,14 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField GenericRecord genericRecord = message.getValue(); if (keyValue) { @SuppressWarnings("unchecked") - KeyValue keyValueObject = (KeyValue) genericRecord.getNativeObject(); + KeyValue keyValueObject = + (KeyValue) genericRecord.getNativeObject(); GenericRecord key = keyValueObject.getKey(); GenericRecord value = keyValueObject.getValue(); - key.getFields().forEach(f-> { + key.getFields().forEach(f -> { log.info("key field {} value {}", f.getName(), key.getField(f.getName())); }); - value.getFields().forEach(f-> { + value.getFields().forEach(f -> { log.info("value field {} value {}", f.getName(), value.getField(f.getName())); }); assertEquals(i, key.getField("age")); @@ -1743,7 +1788,7 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField } else { GenericRecord value = genericRecord; log.info("received value {}", value); - value.getFields().forEach(f-> { + value.getFields().forEach(f -> { log.info("value field {} value {}", f.getName(), value.getField(f.getName())); }); @@ -1939,13 +1984,14 @@ private void prepareDataForMergeFunction(String ns, } private void generateDataByDifferentSchema(String ns, - String baseTopic, - PulsarClient pulsarClient, - Schema schema, - T data, - int messageCnt, - ObjectNode inputSpecNode, - Map topicMsgCntMap) throws PulsarClientException { + String baseTopic, + PulsarClient pulsarClient, + Schema schema, + T data, + int messageCnt, + ObjectNode inputSpecNode, + Map topicMsgCntMap) + throws PulsarClientException { String topic = ns + "/" + baseTopic; Producer producer = pulsarClient.newProducer(schema) .topic(topic) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 033e590d6dd4e..62aa36da1b6be 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -151,6 +151,8 @@ protected static String getExclamationClass(Runtime runtime, } else { return EXCLAMATION_PYTHON_CLASS; } + } else if (Runtime.GO == runtime) { + return null; } else { throw new IllegalArgumentException("Unsupported runtime : " + runtime); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java index 9ef2398c55c00..ad8dc475aac20 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java @@ -34,4 +34,9 @@ public void testGoFunctionLocalRun() throws Exception { testFunctionLocalRun(Runtime.GO); } + @Test(groups = {"go_function", "function"}) + public void testGoExclamationMultiInputsFunction() throws Exception { + testExclamationFunction(Runtime.GO, false, false, true, false); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java index e6f3c67ebcdb7..097a452937d27 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java @@ -98,12 +98,12 @@ private void testCustomSerdeFunction() throws Exception { @Test(groups = {"java_function", "function"}) public void testJavaExclamationFunction() throws Exception { - testExclamationFunction(Runtime.JAVA, false, false, false); + testExclamationFunction(Runtime.JAVA, false, false, false, false); } @Test(groups = {"java_function", "function"}) public void testJavaExclamationTopicPatternFunction() throws Exception { - testExclamationFunction(Runtime.JAVA, true, false, false); + testExclamationFunction(Runtime.JAVA, true, false, false, false); } @Test(groups = {"java_function", "function"}) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java index 1c75d70498609..87a52d27e8927 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java @@ -46,22 +46,22 @@ public void testPythonPublishFunction() throws Exception { @Test(groups = {"python_function", "function"}) public void testPythonExclamationFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, false, false); + testExclamationFunction(Runtime.PYTHON, false, false, false, false); } @Test(groups = {"python_function", "function"}) public void testPythonExclamationFunctionWithExtraDeps() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, false, true); + testExclamationFunction(Runtime.PYTHON, false, false, false, true); } @Test(groups = {"python_function", "function"}) public void testPythonExclamationZipFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, true, false); + testExclamationFunction(Runtime.PYTHON, false, true, false, false); } @Test(groups = {"python_function", "function"}) public void testPythonExclamationTopicPatternFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, true, false, false); + testExclamationFunction(Runtime.PYTHON, true, false, false, false); } @Test(groups = {"python_function", "function"}) From fb9c4d034bfdcfdf256720ec2490c737cae53839 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 10 May 2023 21:16:17 +0800 Subject: [PATCH 184/280] [fix][fn] Make pulsar-admin support update py/go with package url (#19897) --- .../main/java/org/apache/pulsar/admin/cli/CmdFunctions.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 91ec7183ff13c..9b30d59f1679c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -1037,6 +1037,10 @@ void runCmd() throws Exception { updateOptions.setUpdateAuthData(updateAuthData); if (Utils.isFunctionPackageUrlSupported(functionConfig.getJar())) { getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getJar(), updateOptions); + } else if (Utils.isFunctionPackageUrlSupported(functionConfig.getPy())) { + getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getPy(), updateOptions); + } else if (Utils.isFunctionPackageUrlSupported(functionConfig.getGo())) { + getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getGo(), updateOptions); } else { getAdmin().functions().updateFunction(functionConfig, userCodeFile, updateOptions); } From 96367e1c2bfd64ec0f72602321ae6ede4bed96cd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 10 May 2023 09:52:45 -0500 Subject: [PATCH 185/280] [cleanup] Deduplicate test certificates to simplify management (#20289) --- .../broker/admin/AdminApiTlsAuthTest.java | 38 +++++++++---------- .../admin/BrokerAdminClientTlsAuthTest.java | 18 ++++----- .../auth/MockedPulsarServiceBaseTest.java | 15 ++++++++ .../authentication/tls-http/admin.cert.pem | 26 ------------- .../authentication/tls-http/admin.key-pk8.pem | 28 -------------- .../authentication/tls-http/broker.cert.pem | 27 ------------- .../tls-http/broker.key-pk8.pem | 28 -------------- .../authentication/tls-http/ca.cert.pem | 29 -------------- .../authentication/tls-http/proxy.cert.pem | 26 ------------- .../authentication/tls-http/proxy.key-pk8.pem | 28 -------------- .../tls-http/superproxy.cert.pem | 26 ------------- .../tls-http/superproxy.key-pk8.pem | 28 -------------- .../authentication/tls-http/user1.cert.pem | 26 ------------- .../authentication/tls-http/user1.key-pk8.pem | 28 -------------- .../server/AuthedAdminProxyHandlerTest.java | 32 +++++++--------- .../SuperUserAuthedAdminProxyHandlerTest.java | 26 ++++++------- .../tls-admin-proxy/admin.cert.pem | 26 ------------- .../tls-admin-proxy/admin.key-pk8.pem | 28 -------------- .../tls-admin-proxy/broker.cert.pem | 27 ------------- .../tls-admin-proxy/broker.key-pk8.pem | 28 -------------- .../tls-admin-proxy/ca.cert.pem | 29 -------------- .../tls-admin-proxy/proxy.cert.pem | 26 ------------- .../tls-admin-proxy/proxy.key-pk8.pem | 28 -------------- .../tls-admin-proxy/randouser.cert.pem | 19 ---------- .../tls-admin-proxy/randouser.key-pk8.pem | 28 -------------- .../tls-admin-proxy/superproxy.cert.pem | 26 ------------- .../tls-admin-proxy/superproxy.key-pk8.pem | 28 -------------- .../tls-admin-proxy/user1.cert.pem | 26 ------------- .../tls-admin-proxy/user1.key-pk8.pem | 28 -------------- 29 files changed, 64 insertions(+), 712 deletions(-) delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java index bd23e0d7e4e85..2b7b9101a81f2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java @@ -65,19 +65,15 @@ @Test(groups = "broker-admin") public class AdminApiTlsAuthTest extends MockedPulsarServiceBaseTest { - private static String getTLSFile(String name) { - return String.format("./src/test/resources/authentication/tls-http/%s.pem", name); - } - @BeforeMethod @Override public void setup() throws Exception { conf.setLoadBalancerEnabled(true); conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsCertificateFilePath(getTLSFile("broker.cert")); - conf.setTlsKeyFilePath(getTLSFile("broker.key-pk8")); - conf.setTlsTrustCertsFilePath(getTLSFile("ca.cert")); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders( Set.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls")); @@ -87,8 +83,8 @@ public void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.AuthenticationTls"); conf.setBrokerClientAuthenticationParameters( - String.format("tlsCertFile:%s,tlsKeyFile:%s", getTLSFile("admin.cert"), getTLSFile("admin.key-pk8"))); - conf.setBrokerClientTrustCertsFilePath(getTLSFile("ca.cert")); + String.format("tlsCertFile:%s,tlsKeyFile:%s", getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"))); + conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setBrokerClientTlsEnabled(true); conf.setNumExecutorThreadPoolSize(5); @@ -115,11 +111,11 @@ WebTarget buildWebClient(String user) throws Exception { .register(JacksonConfigurator.class).register(JacksonFeature.class); X509Certificate trustCertificates[] = SecurityUtility.loadCertificatesFromPemFile( - getTLSFile("ca.cert")); + CA_CERT_FILE_PATH); SSLContext sslCtx = SecurityUtility.createSslContext( false, trustCertificates, - SecurityUtility.loadCertificatesFromPemFile(getTLSFile(user + ".cert")), - SecurityUtility.loadPrivateKeyFromPemFile(getTLSFile(user + ".key-pk8"))); + SecurityUtility.loadCertificatesFromPemFile(getTlsFileForClient(user + ".cert")), + SecurityUtility.loadPrivateKeyFromPemFile(getTlsFileForClient(user + ".key-pk8"))); clientBuilder.sslContext(sslCtx).hostnameVerifier(NoopHostnameVerifier.INSTANCE); Client client = clientBuilder.build(); @@ -133,8 +129,8 @@ PulsarAdmin buildAdminClient(String user) throws Exception { .serviceHttpUrl(brokerUrlTls.toString()) .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls", String.format("tlsCertFile:%s,tlsKeyFile:%s", - getTLSFile(user + ".cert"), getTLSFile(user + ".key-pk8"))) - .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build(); + getTlsFileForClient(user + ".cert"), getTlsFileForClient(user + ".key-pk8"))) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build(); } PulsarClient buildClient(String user) throws Exception { @@ -143,8 +139,8 @@ PulsarClient buildClient(String user) throws Exception { .enableTlsHostnameVerification(false) .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls", String.format("tlsCertFile:%s,tlsKeyFile:%s", - getTLSFile(user + ".cert"), getTLSFile(user + ".key-pk8"))) - .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build(); + getTlsFileForClient(user + ".cert"), getTlsFileForClient(user + ".key-pk8"))) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build(); } @Test @@ -471,11 +467,11 @@ public void testDeleteNamespace() throws Exception { public void testCertRefreshForPulsarAdmin() throws Exception { String adminUser = "admin"; String user2 = "user1"; - File keyFile = new File(getTLSFile("temp" + ".key-pk8")); + File keyFile = File.createTempFile("temp", ".key-pk8"); Path keyFilePath = Paths.get(keyFile.getAbsolutePath()); int autoCertRefreshTimeSec = 1; try { - Files.copy(Paths.get(getTLSFile(user2 + ".key-pk8")), keyFilePath, StandardCopyOption.REPLACE_EXISTING); + Files.copy(Paths.get(getTlsFileForClient(user2 + ".key-pk8")), keyFilePath, StandardCopyOption.REPLACE_EXISTING); PulsarAdmin admin = PulsarAdmin.builder() .allowTlsInsecureConnection(false) .enableTlsHostnameVerification(false) @@ -483,8 +479,8 @@ public void testCertRefreshForPulsarAdmin() throws Exception { .autoCertRefreshTime(autoCertRefreshTimeSec, TimeUnit.SECONDS) .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls", String.format("tlsCertFile:%s,tlsKeyFile:%s", - getTLSFile(adminUser + ".cert"), keyFile)) - .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build(); + getTlsFileForClient(adminUser + ".cert"), keyFile)) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build(); // try to call admin-api which should fail due to incorrect key-cert try { admin.tenants().createTenant("tenantX", @@ -496,7 +492,7 @@ public void testCertRefreshForPulsarAdmin() throws Exception { // replace correct key file Files.delete(keyFile.toPath()); Thread.sleep(2 * autoCertRefreshTimeSec * 1000); - Files.copy(Paths.get(getTLSFile(adminUser + ".key-pk8")), keyFilePath); + Files.copy(Paths.get(getTlsFileForClient(adminUser + ".key-pk8")), keyFilePath); MutableBoolean success = new MutableBoolean(false); retryStrategically((test) -> { try { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java index 54164c3d40ef5..19a550457a4dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java @@ -48,10 +48,6 @@ public void beforeMethod(Method m) throws Exception { methodName = m.getName(); } - private static String getTLSFile(String name) { - return String.format("./src/test/resources/authentication/tls-http/%s.pem", name); - } - @BeforeMethod @Override public void setup() throws Exception { @@ -63,19 +59,19 @@ public void setup() throws Exception { private void buildConf(ServiceConfiguration conf) { conf.setLoadBalancerEnabled(true); - conf.setTlsCertificateFilePath(getTLSFile("broker.cert")); - conf.setTlsKeyFilePath(getTLSFile("broker.key-pk8")); - conf.setTlsTrustCertsFilePath(getTLSFile("ca.cert")); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setAuthenticationEnabled(true); conf.setSuperUserRoles(Set.of("superproxy", "broker.pulsar.apache.org")); conf.setAuthenticationProviders( Set.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls")); conf.setAuthorizationEnabled(true); conf.setBrokerClientTlsEnabled(true); - String str = String.format("tlsCertFile:%s,tlsKeyFile:%s", getTLSFile("broker.cert"), getTLSFile("broker.key-pk8")); + String str = String.format("tlsCertFile:%s,tlsKeyFile:%s", BROKER_CERT_FILE_PATH, BROKER_KEY_FILE_PATH); conf.setBrokerClientAuthenticationParameters(str); conf.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.AuthenticationTls"); - conf.setBrokerClientTrustCertsFilePath(getTLSFile("ca.cert")); + conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setTlsAllowInsecureConnection(true); conf.setNumExecutorThreadPoolSize(5); } @@ -93,8 +89,8 @@ PulsarAdmin buildAdminClient(String user) throws Exception { .serviceHttpUrl(brokerUrlTls.toString()) .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls", String.format("tlsCertFile:%s,tlsKeyFile:%s", - getTLSFile(user + ".cert"), getTLSFile(user + ".key-pk8"))) - .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build(); + getTlsFileForClient(user + ".cert"), getTlsFileForClient(user + ".key-pk8"))) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build(); } /** diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 3fe8c22b1c4de..b688d5fbf24d8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -66,6 +66,21 @@ * Base class for all tests that need a Pulsar instance without a ZK and BK cluster. */ public abstract class MockedPulsarServiceBaseTest extends TestRetrySupport { + // All certificate-authority files are copied from the tests/certificate-authority directory and all share the same + // root CA. + protected static String getTlsFileForClient(String name) { + return ResourceUtils.getAbsolutePath(String.format("certificate-authority/client-keys/%s.pem", name)); + } + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + public final static String PROXY_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/proxy.cert.pem"); + public final static String PROXY_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/proxy.key-pk8.pem"); public final static String BROKER_KEYSTORE_FILE_PATH = ResourceUtils.getAbsolutePath("certificate-authority/jks/broker.keystore.jks"); public final static String BROKER_TRUSTSTORE_FILE_PATH = diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem deleted file mode 100644 index 0665edbdc126c..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEZTCCAk2gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyMjA4NTcwNloYDzIyOTIwNDA2MDg1NzA2WjAQMQ4wDAYDVQQD -DAVhZG1pbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJw3Jfbn0xkW -36kqQjES6Hn+YTZ2jXS5Co2MzsGBsIY0qJ2BbWHSSaMrja4IERUaCQp16SWxPmZ0 -srMm6ErDoap+O70CWXLT3ybYMV5aVwv3ca4uxsedzaw9MpFXfUDsJJ3yre1JpO+t -A/QzJEGq1d6NN49InUP5kB1Rpay3vaxx8hduzqTO+E/Lptv92p6GjOpXi2icSjiA -pgaan2ldGGKEKv2Sc2bfdIDkTq1yDyNmuPET0yD2dci106EW/mPyj81umPKG/o4K -5W18yG/IhXw5W1zlgO1fWCuqva8NCBdu7s1c7hUX8DBx7km4/I7dllz/nYHIfCEQ -Dmj38oQjYk8CAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF -oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp -ZmljYXRlMB0GA1UdDgQWBBQTMzAAOJ9gXvQSS7Be3+qmrb1kVDAfBgNVHSMEGDAW -gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw -FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQAwy8f6hsG0 -85e3SOIztbUnaVxS7wzDeDzR3vCjpXpm4vTToYzN9zx3JHKSdJrB12emVxItwW/7 -bXqBk0n2EdQjRHCuebnY05eFMNGagMEEMVmSLOproQD7VsyALNxCss1JAyRikh70 -W7wgOVeAhqE53UqqrkzTE7Q+8Bag9t3FytHxApY17XglbWkiVcFpQwSURe9Emi3E -aCJCryGJXrBNuCFXGzetSygDEy27+2FeH8S2XsMwUEGLqDDehzvMenVz1xjXtq+s -KPkofAde52NHd4lLkSeBMSFnKe3V7Xxax2OEUsoQRF3bkbpcJSWsKS9ZAA2yrtuy -Nz/aA1F42LuSFPAYQr1kcZ8eSS918RWz+BiJYU2JuUOPd1XUmJXVvZ4CJurWaC7+ -ZD51YdD8E245xd55fsA6/qLx3eE/Kp0dVq+Hxuz6b4yLET0zkGunOe4A3hnRgkOA -XolXCL+VthhWtFGXn8CjpxDnzjahq69Io+dINehqd5aJEgvnHZIK2s7FTqqBBodU -HhyAE94f64z7ziuRhEG54bmBF+MoGyPf6dVn1Mp3+o+YeQ5q6XlKgh+u8jMgmqRO -ikdsVdMqopt/FXh9eFzvQrwOZFLK6JE/edUgb6xvS1jMF5zi2lIlIkq1RBQOr4HT -XDwX4vRtfpDxpsetGVpeq9O07fbMvp+mkw== ------END CERTIFICATE----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem deleted file mode 100644 index 6aaa22c44d746..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCcNyX259MZFt+p -KkIxEuh5/mE2do10uQqNjM7BgbCGNKidgW1h0kmjK42uCBEVGgkKdeklsT5mdLKz -JuhKw6Gqfju9Ally098m2DFeWlcL93GuLsbHnc2sPTKRV31A7CSd8q3tSaTvrQP0 -MyRBqtXejTePSJ1D+ZAdUaWst72scfIXbs6kzvhPy6bb/dqehozqV4tonEo4gKYG -mp9pXRhihCr9knNm33SA5E6tcg8jZrjxE9Mg9nXItdOhFv5j8o/Nbpjyhv6OCuVt -fMhvyIV8OVtc5YDtX1grqr2vDQgXbu7NXO4VF/Awce5JuPyO3ZZc/52ByHwhEA5o -9/KEI2JPAgMBAAECggEAP+Ipq2Q4puz8wGBgu1LhMWp+9Nfcl1xI3YQ01VulBe0o -+2h/g96McKcSBJaV7cw84ENB+kEWpK2amrsRiemhBmkjIvOAAv50Jp2I6u4E5Qbn -PXUxo1Z8UrCgKmHd/hvUCafByuUwBzf5AvebHyOu3JlhnD302mSHtAW8u/pUHd3D -sxJaw1zwQvmlD5ryM2IVYSji7NYCXF0H6V7HfyohTCrQFEWEAdqEDcFR1BUwbPCE -raq7sAiEy+cBUnfV3IOEAffOZy0vSR90/WwERcwrzCdZmpWpTqtbcqtdBPqsSQzX -shDvXd0e43+FSJzCtQsSQ8WzIrp3rgKJUDA1pJQW4QKBgQDJdTcB6qZz8r+4q9gc -q1KAJyMy01Vio1yaqYzXr0C9Z5FW1GhL+4fwer2y9JyD45sb/lP4reFj9S193BNR -C8cdxM5GrWEpzaQ0Dt1s8P1UbU8G6r4NqwI6ORF3CxXZKfXivQcgqBurJGrBdjIC -NwqAzSkX5flBbhfTlJuUH87k3wKBgQDGgjzdIWXab7FZfdUzzrtEwNooBiSEFixm -UAwP5sxL8VkM9wzAKPEVDDQBBoKIga9OESif4S9UUo4tu65AYfxF9Om26K4QrVj8 -HT/U+lfT7xFmPd/GINIbeTSmW0w7Ehpj8SbcQI4Sb2lVE562FlHh7QbHZd0/X/2J -nbgT9MRAkQKBgQCVPAN3o/+SPOzRPFtnQXJoBJYKfIrv+twKpjbzP5vRsvrzO33X -a4kUF5iXDKU0/lJUtl42BXjFt0Xvyit1CiiCYNv9d0pW0UMmXSyiGxNOi3rTQOlw -7pFD2Cqb6NZSfMbtI+I3ytBUQzHiBlCdW3CoYVJjpbSzR37W+WsWm0mEOQKBgQCq -ANObFYUjA1DBMZCrY7rhcL/kUw5myI6RuK/71k7UIwd+oP0cfHOq8N6AmlCkE1xM -4UkHU1SzRFhbNkZPARuJ1etqJ+8afTqd/3axMQyShkVCaG8CQQ1vVegPKFUqqaBM -QzRioC6L/zoYEEt16buKXvHVRpmqMszxVE9XV+HS4QKBgDvs5qloOowS5kcWInrj -yecu5MJvFf2IZpMw7EiKV8VUPeKaiUlqgFj9d9cotUIMauXgBq6f5NBRg7Ike60t -/JJrPtqXY+gdFLjxcKMUVYhomFlQYYg/RJZUBrtkyKBP68abopCYmb59r3ixeNNf -qA1F36mmFtzdjSdtH/dTTecN ------END PRIVATE KEY----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem deleted file mode 100644 index b5c7a5dc709a1..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEkDCCAnigAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyMjA4NTUzMloYDzIyOTIwNDA2MDg1NTMyWjAjMSEwHwYDVQQD -DBhicm9rZXIucHVsc2FyLmFwYWNoZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB -DwAwggEKAoIBAQDQouKhZah4hMCqmg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63 -EvNjEK4L/ZWSEV45L/wc6YV14RmM6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0O -OQXVicqZLOc6igZQhRg8ANDYdTJUTF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01 -+ARO9OotMJtBY+vIU5bV6JydfgkhQH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+X -aqTN3/tF8+MBcFB3G04s1qc2CJPJM3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00s -bxa4FGbKgfDobbkJ+GgblWCkAcLN95sKTqtHAgMBAAGjgd0wgdowCQYDVR0TBAIw -ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu -ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUaxFvJrkEGqk8azTA -DyVyTyTbJAIwQQYDVR0jBDowOIAUVwvpyyPov0c+UHo/RX6hGEOdFSehFaQTMBEx -DzANBgNVBAMMBmZvb2JhcoIJANfih0+geeIMMA4GA1UdDwEB/wQEAwIFoDATBgNV -HSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA35QDGclHzQtHs3yQ -ZzNOSKisg5srTiIoQgRzfHrXfkthNFCnBzhKjBxqk3EIasVtvyGuk0ThneC1ai3y -ZK3BivnMZfm1SfyvieFoqWetsxohWfcpOSVkpvO37P6v/NmmaTIGkBN3gxKCx0QN -zqApLQyNTM++X3wxetYH/afAGUrRmBGWZuJheQpB9yZ+FB6BRp8YuYIYBzANJyW9 -spvXW03TpqX2AIoRBoGMLzK72vbhAbLWiCIfEYREhbZVRkP+yvD338cWrILlOEur -x/n8L/FTmbf7mXzHg4xaQ3zg/5+0OCPMDPUBE4xWDBAbZ82hgOcTqfVjwoPgo2V0 -fbbx6redq44J3Vn5d9Xhi59fkpqEjHpX4xebr5iMikZsNTJMeLh0h3uf7DstuO9d -mfnF5j+yDXCKb9XzCsTSvGCN+spmUh6RfSrbkw8/LrRvBUpKVEM0GfKSnaFpOaSS -efM4UEi72FRjszzHEkdvpiLhYvihINLJmDXszhc3fCi42be/DGmUhuhTZWynOPmp -0N0V/8/sGT5gh4fGEtGzS/8xEvZwO9uDlccJiG8Pi+aO0/K9urB9nppd/xKWXv3C -cib/QrW0Qow4TADWC1fnGYCpFzzaZ2esPL2MvzOYXnW4/AbEqmb6Weatluai64ZK -3N2cGJWRyvpvvmbP2hKCa4eLgEc= ------END CERTIFICATE----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem deleted file mode 100644 index 2b51d015b8ace..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDQouKhZah4hMCq -mg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63EvNjEK4L/ZWSEV45L/wc6YV14RmM -6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0OOQXVicqZLOc6igZQhRg8ANDYdTJU -TF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01+ARO9OotMJtBY+vIU5bV6Jydfgkh -QH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+XaqTN3/tF8+MBcFB3G04s1qc2CJPJ -M3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00sbxa4FGbKgfDobbkJ+GgblWCkAcLN -95sKTqtHAgMBAAECggEBALE1eMtfnk3nbAI74bih84D7C0Ug14p8jJv/qqBnsx4j -WrgbWDMVrJa7Rym2FQHBMMfgIwKnso0iSeJvaPz683j1lk833YKe0VQOPgD1m0IN -wV1J6mQ3OOZcKDIcerY1IBHqSmBEzR7dxIbnaxlCAX9gb0hdBK6zCwA5TMG5OQ5Y -3cGOmevK5i2PiejhpruA8h7E48P1ATaGHUZif9YD724oi6AcilQ8H/DlOjZTvlmK -r4aJ30f72NwGM8Ecet5CE2wyflAGtY0k+nChYkPRfy54u64Z/T9B53AvneFaj8jv -yFepZgRTs2cWhEl0KQGuBHQ4+IeOfMt2LebhvjWW8YkCgYEA7BXVsnqPHKRDd8wP -eNkolY4Fjdq4wu9ad+DaFiZcJuv7ugr+Kplltq6e4aU36zEdBYdPp/6KM/HGE/Xj -bo0CELNUKs/Ny9H/UJc8DDbVEmoF3XGiIbKKq1T8NTXTETFnwrGkBFD8nl7YTsOF -M4FZmSok0MhhkpEULAqxBS6YpQsCgYEA4jxM1egTVSWjTreg2UdYo2507jKa7maP -PRtoPsNJzWNbOpfj26l3/8pd6oYKWck6se6RxIUxUrk3ywhNJIIOvWEC7TaOH1c9 -T4NQNcweqBW9+A1x5gyzT14gDaBfl45gs82vI+kcpVv/w2N3HZOQZX3yAUqWpfw2 -yw1uQDXtgDUCgYEAiYPWbBXTkp1j5z3nrT7g0uxc89n5USLWkYlZvxktCEbg4+dP -UUT06EoipdD1F3wOKZA9p98uZT9pX2sUxOpBz7SFTEKq3xQ9IZZWFc9CoW08aVat -V++FsnLYTa5CeXtLsy6CGTmLTDx2xrpAtlWb+QmBVFPD8fmrxFOd9STFKS0CgYAt -6ztVN3OlFqyc75yQPXD6SxMkvdTAisSMDKIOCylRrNb5f5baIP2gR3zkeyxiqPtm -3htsHfSy67EtXpP50wQW4Dft2eLi7ZweJXMEWFfomfEjBeeWYAGNHHe5DFIauuVZ -2WexDEGqNpAlIm0s7aSjVPrn1DHbouOkNyenlMqN+QKBgQDVYVhk9widShSnCmUA -G30moXDgj3eRqCf5T7NEr9GXD1QBD/rQSPh5agnDV7IYLpV7/wkYLI7l9x7mDwu+ -I9mRXkyAmTVEctLTdXQHt0jdJa5SfUaVEDUzQbr0fUjkmythTvqZ809+d3ELPeLI -5qJ7jxgksHWji4lYfL4r4J6Zaw== ------END PRIVATE KEY----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem deleted file mode 100644 index 0446700135d39..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFCDCCAvCgAwIBAgIJANfih0+geeIMMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV -BAMMBmZvb2JhcjAeFw0xODA2MjIwODQ2MjFaFw0zODA2MTcwODQ2MjFaMBExDzAN -BgNVBAMMBmZvb2JhcjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOVU -UpTPeXCeyfUiQS824l9s9krZd4R6TA4D97eQ9EWm2D7ppV4gPApHO8j5f+joo/b6 -Iso4aFlHpJ8VV2a5Ol7rjQw43MJHaBgwDxB1XWgsNdfoI7ebtp/BWg2nM3r8wm+Z -gKenf9d1/1Ol+6yFUehkLkIXUvldiVegmmje8FnwhcDNE1eTrh66XqSJXEXqgBKu -NqsoYcVak72OyOO1/N8CESoSdyBkbSiH5vJyo0AUCjn7tULga7fxojmqBZDog9Pg -e5Fi/hbCrdinbxBrMgIxQ7wqXw2sw6iOWu4FU8Ih/CuF4xaQy2YP7MEk4Ff0LCY0 -KMhFMWU7550r/fz/C2l7fKhREyCQPa/bVE+dfxgZ/gCZ+p7vQ154hCCjpd+5bECv -SN1bcVIPG6ngQu4vMXa7QRBi/Od40jSVGVJXYY6kXvrYatad7035w2GGGGkvMsQm -y53yh4tqQfH7ulHqB0J5LebTQRp6nRizWigVCLjNkxJYI+Dj51qvT1zdyWEegKr1 -CthBfYzXlfjeH3xri1f0UABeC12n24Wkacd9af7zs7S3rYntEK444w/3fB0F62Lh -SESfMLAmUH0dF5plRShrFUXz23nUeS8EYgWmnGkpf/HDzB67vdfAK0tfJEtmmY78 -q06OSgMr+AOOqaomh4Ez2ZQG592bS71G8MrE7r2/AgMBAAGjYzBhMB0GA1UdDgQW -BBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAfBgNVHSMEGDAWgBRXC+nLI+i/Rz5Qej9F -fqEYQ50VJzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG -9w0BAQsFAAOCAgEAYd2PxdV+YOaWcmMG1fK7CGwSzDOGsgC7hi4gWPiNsVbz6fwQ -m5Ac7Zw76dzin8gzOPKST7B8WIoc7ZWrMnyh3G6A3u29Ec8iWahqGa91NPA3bOIl -0ldXnXfa416+JL/Q5utpiV6W2XDaB53v9GqpMk4rOTS9kCFOiuH5ZU8P69jp9mq6 -7pI/+hWFr+21ibmXH6ANxRLd/5+AqojRUYowAu2997Z+xmbpwx/2Svciq3LNY/Vz -s9DudUHCBHj/DPgNxsEUt8QNohjQkRbFTY0a1aXodJ/pm0Ehk2kf9KwYYYduR7ak -6UmPIPrZg6FePNahxwMZ0RtgX7EXmpiiIH1q9BsulddWkrFQclevsWO3ONQVrDs2 -gwY0HQuCRCJ+xgS2cyGiGohW5MkIsg1aI0i0j5GIUSppCIYgirAGCairARbCjhcx -pbMe8RTuBhCqO3R2wZ0wXu7P7/ArI/Ltm1dU6IeHUAUmeneVj5ie0SdA19mHTS2o -lG77N0jy6eq2zyEwJE6tuS/tyP1xrxdzXCYY7f6X9aNfsuPVQTcnrFajvDv8R6uD -YnRStVCdS6fZEP0JzsLrqp9bgLIRRsiqsVVBCgJdK1I/X59qk2EyCLXWSgk8T9XZ -iux8LlPpskt30YYt1KhlWB9zVz7k0uYAwits5foU6RfCRDPAyOa1q/QOXk0= ------END CERTIFICATE----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem deleted file mode 100644 index 6c2f4295c9dbd..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEZTCCAk2gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyNTEzNTYwNFoYDzIyOTIwNDA5MTM1NjA0WjAQMQ4wDAYDVQQD -DAVwcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMieX78Yj8OW -eBHAneRaCCoO8Qrpj8zGo7h9lCdmi1lBDh1uR2sDbotiHGfJzQn836WcYmyeAvfn -qvgr9HCXmXdLgmJ3GT/LVu5GEm6msSDiZQPr9so5lQVioisK4UwJROQsE/J52cyR -9o3H6M4FKb6QpoobKa62fSfTumwwulaYaDJuRRGoGIkcRuUQ59EWAaDkD3IcDpAn -9mTbnE4Iz+JxSrsZ5DJ3X/m/AqyLWtj6GAfyK9a1dhNdlf2x4JZT1QNtojiBXt95 -OIZyRBNbHMFniq5gel6wdBkmJWutfcTct7wKa2LCxLpKoDIc1HWoL3+RUzOKxYIP -0qXEQ3bmONkCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF -oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp -ZmljYXRlMB0GA1UdDgQWBBSgsgfmDbXEkrrpHUC9GnDDjxaKizAfBgNVHSMEGDAW -gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw -FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQB+uZ2OWR+G -sRqYHEeVqUI8G2p8Np8eC/onGpBL8Gj9SGxIQcIPtqHPUlCe9fd/96JOptOGRYEB -BmUCaCmQ4IgMW6e6fArka5IB4XXIgHFXyQ6ImTvjDavzlVw06zn9S4dLwVzsRBg+ -GS9svtq23W+f5rEN5N+7LhtcbclfiG4VCCqDG5VhkzEok+SRamDI8rDRZtodMw0O -/+L+xaaQPUPjX8KUlKn4uVpCDbxUzHonlCPzbkHHm5su0D4ysjJIy3/y3yow6JE/ -02L7PZkmkmw3/V+84T3X8/GD15sVUv/3v1gXEBxYwAs+RNTJ0APvMEMSvCq0AMfF -bPMZBuAGNBG7lv7TovzHgGFKXT7du5OFF/qjAsEffhbo224CB96fgwvvndwHHBFh -J06BvHZG1i9dDVhUKoB1owkWrE4RZv2ZKEtZYgizzSmzZRHtARo0t1Byc5djx1tX -TkJOHshNqJZOY1ER0DPaVQgKI+PRTbEdj/xPGRX3ebSqDmilAfPXshqgElfch6Yl -f2V58TyCnjXOibvkG9D5OyCdWLEECumOZgYar0KZgNfrvTOi1OKXnX1fsbh29fWA -ICZRcdmjkz79zQXY2SuzCWlskuXPKAmW1AMqs+l6ormmKfzIUx3Yriy4LqIIYY1v -uQD5vghZmd9HUg2KaXfSGD9stGCD8KhntA== ------END CERTIFICATE----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem deleted file mode 100644 index 70e0107f2741f..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDInl+/GI/DlngR -wJ3kWggqDvEK6Y/MxqO4fZQnZotZQQ4dbkdrA26LYhxnyc0J/N+lnGJsngL356r4 -K/Rwl5l3S4Jidxk/y1buRhJuprEg4mUD6/bKOZUFYqIrCuFMCUTkLBPyednMkfaN -x+jOBSm+kKaKGymutn0n07psMLpWmGgybkURqBiJHEblEOfRFgGg5A9yHA6QJ/Zk -25xOCM/icUq7GeQyd1/5vwKsi1rY+hgH8ivWtXYTXZX9seCWU9UDbaI4gV7feTiG -ckQTWxzBZ4quYHpesHQZJiVrrX3E3Le8CmtiwsS6SqAyHNR1qC9/kVMzisWCD9Kl -xEN25jjZAgMBAAECggEAJJyCbKVW1yLGlrbIGbw0cTh41Lz6+SvnBOwl9WrJU2iD -4usVLXpa2iT1ehthx8jWJ6r6a0gK0qL8mH2tBj8kSpkFGmMRwIqjOqifBIJ3IMEw -Hh8Z0p3fjDQL1D8QDohCgkFpAn8qOCMLE6S/35khnR1Yxytd1/yFqpcBFm1uFA85 -dYisjPqWm/IZIU5rH0zgKAIhtvl9abnoi93443EHsKpRAW1gwRXx9Aak7TV768bZ -tELBsaTnXnNzamDiaimmxEOlqR9O0W8JE/31KFL26JcVmsTRG7sMpoUxCEMjOuGZ -J30bXFZUW6NrDpFsQ7uTqD6TNn2971N8KFCLnC/JYQKBgQDo82axC7L7n7BYTu18 -dupeT7n5dTBD/I3l0KtT05xiZA8GZr2i+pt+/aWzCzK4/Ee4jb4/o8CRQRB5v5mo -c9lc+BaoAIQiwiw+aufT+UojrcijrOMEL5Zk3zdZ2rcEoAsVvqtejNnwLCGI9Rnl -gp7n9oRhwDIv9Fu09snUougE3wKBgQDceAGKUB8pGd3eEya/0jU9J60LsKbcJSsN -4v1S5LiPtHOyhr0g4x/LibMP2PJhG3tJ1bgpaGmn9du2D20M6ukRhIYyn/7G+N+A -oqryyvO1MMYnhc4IEQvWrzDnBM0hV2bdjp4s/1ASVHVRwk8+orqxysIJ1D75nnRX -Tyfl6HgBRwKBgQCfVlWIhiMPv6OkU6BXgRNAHTJs8f5okmgQqNF3jgequRwZ2c6e -muIfU6myNNel9lGsZ6+Y4g4GjMWTMT4OHeewkrUUhv3atIwEyaT2tc5DZ0wUwF2r -cE1jg9bdbB/BVyMd5YRcMOWlRNpPTq8+8EB3E4RrREZPzMmplyBohGFFawKBgGjc -P0dM8nU3E1rj2wNTdPUAYQL1Y3fDyeWR+BEsLkhTeNAJ2/y/akkB1oQMGMRtMMee -ejhfrBkyC+1dCu4g8PffA4EirihvCMcDF7HhK+cbKrRzpNobWXkj3GuU0ggwrQFm -Kv+V87y0JRTdCZnuBkQ3/vBz3fwWDJnWUVC9sA5TAoGBAN4t2gJCZax5yInUgyWi -Tgumb2qVWsGBBLMTKIsrkK/KphzgAhHcVhDCybA79TmIM2FfIlpf6TE7Mv4NI055 -ZJzHX+GMT5Czy2Ku9MJD3PLTFN5MjYb6g92fKViLDMI6fwzTHB8xPJ7Ob9bV2srS -ZlmKNXTkZFk3/orK4WdCZufz ------END PRIVATE KEY----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem deleted file mode 100644 index 9656e2c8ba0d2..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEajCCAlKgAwIBAgICEAQwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyNjEzMjUxN1oYDzIyOTIwNDEwMTMyNTE3WjAVMRMwEQYDVQQD -DApzdXBlcnByb3h5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA43Sn -ys/h3wxJ/IBEzbJdQAKCxU4of5KgTDzFoOaS8C63Nwbjgy0qcWYdRDceP4lJIKSA -1+JZ+R1opyrfVIC2D9oDJFIJTFfXy9G9VYDccwAONPgAamvRzBnYcEu8fqM+Ohle -kZltKktAHnX3WtG3RyxEL5nYzlMlGwUXJu3Rxc9SlkYSxERzjWHikGqGmXLX2qB0 -k6oyxTrK+4+EHk3khbEIqyQZSOFbD7NMfnRy0CWFv/9T1shgjAIBCake6jY7lwaT -S7JvbLfG6ABf5xHMxoWLXa2qwb+Ar43Ff9g8kKZwFOxMeGcwkzyJPBbWytFxaWn+ -R2RHhTaVCGc22CjdfQIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB -BAQDAgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQg -Q2VydGlmaWNhdGUwHQYDVR0OBBYEFHEgLhfH0Z1QlLxeQbG7YZyBlz1MMB8GA1Ud -IwQYMBaAFFcL6csj6L9HPlB6P0V+oRhDnRUnMA4GA1UdDwEB/wQEAwIF4DAdBgNV -HSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBAK9q -3YnEa99Cq3Vo3g6PE/A/xOE97seVNuavqyBcVv9PrTydb5XG4jPkYU3xOYXIc8rA -A4gzd+AsGO9rjGMPGGjQJI3JO/3BCeBJMkn50C/rM1yWVMHnVyFcJlg16xaWtj1e -2Jk8egJW2gSYyF+N2TdzI7tOb002GNr36posnqO+IOoLapyHFBxxUjsPDRoo8fJn -myWsV1Y9oRUZyJlfIAJsu85ew7gDBY2jaiEiopzour3uU3C0N7gYni2OmVwfr6J8 -R2/Jp43BSD5sYOW9RAJIEEXef+InYtz9HTJvKu2LsWwIBkaztk29tJcDE+1La6Sw -0dF0YkUwnXoGQFjiV+8pXX3TF5glXKj1rU8WfNazF6lqslB6DmdgR3/FQ6Z2sE86 -d9hVtayZIGlzU0rWmBBtr++7Wo88nmzAtd/xbZMFG8U//+Q2AvJT2oVGtqM48+al -rnsN/gYrLDr7RC14bHIuO1v6ZL/rAi7SPKrKYAyQVTAcRuW516SxxR6S1Xa1ITnh -rwgKg13eQuwu3iigguIS9XL6nAXabBIxBxMl6o2YlyIPekKYIcQmpqhkavJ6VOgX -iq9VdY6fIJVfmxNZuwM3/28k7UeUAfhI2SSVH4ZURbPiGGH2wukc1QkmtZ2cNa35 -C1y79aqJbIa3ErqLFPj/fM+34x8L7QHPq6RfaODa ------END CERTIFICATE----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem deleted file mode 100644 index 2e4140b8fb392..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDjdKfKz+HfDEn8 -gETNsl1AAoLFTih/kqBMPMWg5pLwLrc3BuODLSpxZh1ENx4/iUkgpIDX4ln5HWin -Kt9UgLYP2gMkUglMV9fL0b1VgNxzAA40+ABqa9HMGdhwS7x+oz46GV6RmW0qS0Ae -dfda0bdHLEQvmdjOUyUbBRcm7dHFz1KWRhLERHONYeKQaoaZctfaoHSTqjLFOsr7 -j4QeTeSFsQirJBlI4VsPs0x+dHLQJYW//1PWyGCMAgEJqR7qNjuXBpNLsm9st8bo -AF/nEczGhYtdrarBv4CvjcV/2DyQpnAU7Ex4ZzCTPIk8FtbK0XFpaf5HZEeFNpUI -ZzbYKN19AgMBAAECggEBAM1JAwuD1drmj3wKFI8F1S2pVndXFCwXnP9RthiDIckO -kKNkX0CMKgtQ20cu6+jyMgL5FaRCkWvJxCNkCU6OIENsQ3urYuL5QTWeZeBevhg4 -y5m43z8tcptgFD09zbEKCmaLcRO9wo3yfrs/QvE/58efxyajFs8YsZuSW5Px/msl -AFN8qGY0q0lQbQPK8cPYT4OnW9isqEjkvHq1Q+ryv4cxsGWtBZYmJVeLVHzClihi -lODdN6YsDcu9jLwsA8o2WSBRsbLHK6caW4FdEwgOLckc0EGGEU64VEB9ZOwgcwYQ -M9mZDs2w7qk63YnDH4KmpoP0Oc8N+bG7Pkifd7f8HqECgYEA8esMQya6Qln4O7Qo -aI8X9t5gCpq+bh/P+7vYdBR/9St5VtE/P0yuaQdT2e5t+Ei4ujqHvJ0GTPmqYIJf -2DNvJMu8EnXv+nqKQCsyMc/nqQN1CcRGqZssH1V/ZIQLRUA0cN4hzE0eHITwp6U9 -vD/WaE3mKX/XHthIjc8hnuoajOkCgYEA8LIYMgyD3wClWOKe5TkBumI0KvA9tP90 -IFHu1wLNl0tKBpsXkzzxiav1FMX3K7B29vxukrs946KRESHzRg4c5ULPnMmwcQyx -AortKJWrGVsna/QDWPRutXSN1XmnjKWsHmTpQQqLfaRvLW+Hd34+EVdVh9sVt/y9 -RucnBvxcX3UCgYBebm/U7pMaP2BkfcigN+sU1G0M9qaK+iQHkaXGehIQs62js/5K -STZzjQawNR/8IPbqytodR/YjqflVvs6G6FzkMhrx4dORJLA+qB3pz8wP72eKLnGe -1xF8EbWumNSFbbCKtkrfIuM0IriF2Dym9QxOnsnPPTXNtoNrx4TKMXu3sQKBgBq9 -noSI8WmoF7adTsvmnnOHj4YptKFUNCGXGLLYg+DII4xCVMct4SPLb+oD6Gb5Lu5X -sy0oEkMk/3roy68/yCQMXSZtHeYhY9UFfD2jCyRBBUswC+MpHNeaAFv0LRIqIcoq -qeNo+YBW8WcZ2fIDm3+vtTfntiz/rkOfUK2tAdI1AoGAL2LVDF5e1meSRocEoy8e -gqrshrhg+KBqYmcjtd4Iup4WvR6uyH4qE9yFLLHFZ04pZzXLHcPfqgOSP8qTxgH3 -h0uqtcYmejS/yl6PbC9OPXcSkMgntRkTbU9Ug05ijfN9NRnW+8WXvi8Z6DKed1P1 -9PxwA75P9o0h00mMk8PQitQ= ------END PRIVATE KEY----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem deleted file mode 100644 index 072f2867fe62b..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEZTCCAk2gAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyNzA4NDAyNVoYDzIyOTIwNDExMDg0MDI1WjAQMQ4wDAYDVQQD -DAV1c2VyMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM5iqgr4PUUZ -AW9MDGP5cBaSJALSPV63m6M/IoovrMWJ9CGtcQfZTUHwDorIlXgQ6H/KufmsHW0Y -OQbChLSTDB14D0jSMtyv6+ibSoE1ZEl2SbB1miLd0P5AS9YmzzEW2+bx0zJORLYD -PzJ1Nh3/kQlRs04IECki291WZiVRzX2JRoL7kMtOAoKJqQfsT14Oi9EAw39VhLeB -uc/Mx6Jsutq/YdXakoZtQbfZka2MMfLXgMDLIPDqbU+09q7au2dq8RjGzrWnxnOX -o/XQssrIbwzJJYASBsgAAtnAw7bPzCX6+cL6PZVRyiEZov0HKXyRyvrbQ5hyEMuS -3dHqoKt0fKMCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF -oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp -ZmljYXRlMB0GA1UdDgQWBBQ7NSD6lx6Vq38cEoD5l7FHs1Ej8DAfBgNVHSMEGDAW -gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw -FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQC+cU7ctY7o -eTW+oHTq9EGJUMwW1fOww4QDrHtgZT4OYkO88zxQV2Cr050p8eaV5dHXZBf9/bRV -7hPNV5+HpQhb9TZ9xK2WRZ2QV7a/UDyUnksVKGSK9tNZMZPueOEB19e4bIBcgnQa -5i9sgZr93na7pFOY7lBQy6gfaOcnejYHmvVqIGaZBVH8rkEsGhhkJxy7qkpFNKgf -PGiIRo9L0WYqDCSiaICeCiteJwIfjsUFJKF0YnpXZq1kFfQscnleg60MWZAXvacp -tAciE51Ow60cqQWER66iwqnBSPD4l91SxAaGQAmalgCioGsYSbojXcOvRidhYJ2T -3YwCpqlC0qC9D2ZmNoukb1a0Pi03MuSJwD/8v9eqwEW9dFAzdnWDzTZMN9CfdjVh -2qiO5o5Si/X1Dmjdk2F/EM62YJQBAlkZBetFJ0o2QPGTSD+zrpfITIW8Pu+/5zcC -MZdzyUf0p1GO2Kn7wmqPQjz59zABagmxCNks8HeqPnzmWuADMaggb0nOmrBACE2x -b9XR6/xaXpwTRf0h5N3evivzUHo6XVw8A3gVUNoBm9Of3PlAsjM4I4SWFb6nrwYv -RnI04c+R95Su1fMc2wky0PmW+iWRTaEN/cUdX1SF6jo1nRLELGcbMSGUI6UI8kff -crvCz7uLu7Lr5/CKnEm2bSCZ4eIQpOs4nQ== ------END CERTIFICATE----- diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem deleted file mode 100644 index eec5c94c0665c..0000000000000 --- a/pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOYqoK+D1FGQFv -TAxj+XAWkiQC0j1et5ujPyKKL6zFifQhrXEH2U1B8A6KyJV4EOh/yrn5rB1tGDkG -woS0kwwdeA9I0jLcr+vom0qBNWRJdkmwdZoi3dD+QEvWJs8xFtvm8dMyTkS2Az8y -dTYd/5EJUbNOCBApItvdVmYlUc19iUaC+5DLTgKCiakH7E9eDovRAMN/VYS3gbnP -zMeibLrav2HV2pKGbUG32ZGtjDHy14DAyyDw6m1PtPau2rtnavEYxs61p8Zzl6P1 -0LLKyG8MySWAEgbIAALZwMO2z8wl+vnC+j2VUcohGaL9Byl8kcr620OYchDLkt3R -6qCrdHyjAgMBAAECggEBAK8J8QvytAxJg/T/+7ZC1PTfp1kZNGGDuaV/o2ytuIul -T//MGQQ+IY8d6Ud9jX9SX84agxalCiP/mkYIbgK0gF7x94yccfTH433ZTxw8yzye -7SqS41JU7K7mmysaqTkKGSFK0gNlbFMud8f0rxxMJ5dOypMQtZwd63lSkLlwIqcn -YhKcAdXM4WGv07MFdwAXvrK2trhBBkCA08ZzyVy4lWE+cVfEkwH6O8EdGchl7g4U -LqIkYzufQWKdH9frt6N3qEV/qs0s1qipVzV07GaPpEdK/G5exJ7NjZaXJkHOuMbt -7ae1zJBexW41/++fjzsWSYljvSEphjqwrGYrr+tMGNECgYEA+Sg1sHcYO4Bg16qG -c8XM9g8DFL396X4MnMh+AFrDV8dbdz39x9fFtpKEfqaAZrG6I8fW3RkMLsJvJ4GI -KDJNz2ezEMbNKlXE9UzvAsrdvWNWDGJvrDu0CbJg2wtBeVEBIUYetSdNHoEwY7ZJ -QHnwbxYYrUFIJ34rI0SQ17/Z4vsCgYEA1A27jYTWr86JmvjQDUiJcJsbhpa/6OzZ -n3KTu0n0oCvl7uvvjUfhlzmcq7kyY0oO1C5TQHLiqBaI5hXw+iYwnESLIn7BwhMX -dcxglvXx95h1NqHYX9GnURl2QtrjmuY5yx+itfmZCRkRPO3c5e9uZyf01CeE4uwl -hDNh0RrSXHkCgYBvlQhmXQ+nJhk4vI+2LXFbCOISWfvqo562YDu9oOg22Xsm7cZH -x2QuHXPk3GBInXOFLqwVHHCOSFlLUgFOLykVp5VUABRFz1+Dk86+a2fetywEI9lr -QtmgNhiWQHY0BIkDA8ogytcIwEaRgUNQ8sswlK68eK39sc1T4BMV7D+CHQKBgG3g -+8lWBwSsKgOCYBQx/P27caTo4mJosE99yG0o4jhI5ulJmiSEFbINqVAWM7TdQBfU -NVFU9nuQybknr2l/dnrSzaG/Otk8mVBx6a7vnETm2/3GGV91PJS6c9wqnfu6xkGp -j99piVH8ikEfI/KFgZi0TJnOLH6FTN9W3J3EnzJJAoGAE4ZLLi+2RP4ZYXI11CJC -BSM1AEpqemgTUSidZyTiJJMGGrC7tyEba+4TIqeGgvD74p57XFbN7LOJWmnAiK8b -BLhQqPgOCXqrna00GnNKKx7AzgYHqlq03tGlX858aH18rkSRVk3UhkTkGTSr3iQn -aJeN/0HbpGr67bimf4bqlCY= ------END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java index de168284ab47f..af70276aed95e 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java @@ -55,10 +55,6 @@ public class AuthedAdminProxyHandlerTest extends MockedPulsarServiceBaseTest { private BrokerDiscoveryProvider discoveryProvider; private PulsarResources resource; - static String getTlsFile(String name) { - return String.format("./src/test/resources/authentication/tls-admin-proxy/%s.pem", name); - } - @BeforeMethod @Override protected void setup() throws Exception { @@ -67,9 +63,9 @@ protected void setup() throws Exception { conf.setAuthorizationEnabled(true); conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsTrustCertsFilePath(getTlsFile("ca.cert")); - conf.setTlsCertificateFilePath(getTlsFile("broker.cert")); - conf.setTlsKeyFilePath(getTlsFile("broker.key-pk8")); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); conf.setTlsAllowInsecureConnection(false); conf.setSuperUserRoles(ImmutableSet.of("admin")); conf.setProxyRoles(ImmutableSet.of("proxy")); @@ -91,15 +87,15 @@ protected void setup() throws Exception { proxyConfig.setHttpMaxRequestHeaderSize(20000); // enable tls and auth&auth at proxy - proxyConfig.setTlsCertificateFilePath(getTlsFile("broker.cert")); - proxyConfig.setTlsKeyFilePath(getTlsFile("broker.key-pk8")); - proxyConfig.setTlsTrustCertsFilePath(getTlsFile("ca.cert")); + proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH); + proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH); + proxyConfig.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( String.format("tlsCertFile:%s,tlsKeyFile:%s", - getTlsFile("proxy.cert"), getTlsFile("proxy.key-pk8"))); - proxyConfig.setBrokerClientTrustCertsFilePath(getTlsFile("ca.cert")); + getTlsFileForClient("proxy.cert"), getTlsFileForClient("proxy.key-pk8"))); + proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName())); resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), @@ -128,22 +124,22 @@ protected void cleanup() throws Exception { PulsarAdmin getDirectToBrokerAdminClient(String user) throws Exception { return PulsarAdmin.builder() .serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(getTlsFile("ca.cert")) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), - ImmutableMap.of("tlsCertFile", getTlsFile(user + ".cert"), - "tlsKeyFile", getTlsFile(user + ".key-pk8"))) + ImmutableMap.of("tlsCertFile", getTlsFileForClient(user + ".cert"), + "tlsKeyFile", getTlsFileForClient(user + ".key-pk8"))) .build(); } PulsarAdmin getAdminClient(String user) throws Exception { return PulsarAdmin.builder() .serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get()) - .tlsTrustCertsFilePath(getTlsFile("ca.cert")) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), - ImmutableMap.of("tlsCertFile", getTlsFile(user + ".cert"), - "tlsKeyFile", getTlsFile(user + ".key-pk8"))) + ImmutableMap.of("tlsCertFile", getTlsFileForClient(user + ".cert"), + "tlsKeyFile", getTlsFileForClient(user + ".key-pk8"))) .build(); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java index 5b7cb88f98280..d3291c8fb910d 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java @@ -51,10 +51,6 @@ public class SuperUserAuthedAdminProxyHandlerTest extends MockedPulsarServiceBas private BrokerDiscoveryProvider discoveryProvider; private PulsarResources resource; - static String getTlsFile(String name) { - return String.format("./src/test/resources/authentication/tls-admin-proxy/%s.pem", name); - } - @BeforeMethod @Override protected void setup() throws Exception { @@ -64,9 +60,9 @@ protected void setup() throws Exception { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsTrustCertsFilePath(getTlsFile("ca.cert")); - conf.setTlsCertificateFilePath(getTlsFile("broker.cert")); - conf.setTlsKeyFilePath(getTlsFile("broker.key-pk8")); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); conf.setTlsAllowInsecureConnection(false); conf.setSuperUserRoles(ImmutableSet.of("admin", "superproxy")); conf.setProxyRoles(ImmutableSet.of("superproxy")); @@ -86,15 +82,15 @@ protected void setup() throws Exception { proxyConfig.setTlsEnabledWithBroker(true); // enable tls and auth&auth at proxy - proxyConfig.setTlsCertificateFilePath(getTlsFile("broker.cert")); - proxyConfig.setTlsKeyFilePath(getTlsFile("broker.key-pk8")); - proxyConfig.setTlsTrustCertsFilePath(getTlsFile("ca.cert")); + proxyConfig.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + proxyConfig.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + proxyConfig.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( String.format("tlsCertFile:%s,tlsKeyFile:%s", - getTlsFile("superproxy.cert"), getTlsFile("superproxy.key-pk8"))); - proxyConfig.setBrokerClientTrustCertsFilePath(getTlsFile("ca.cert")); + getTlsFileForClient("superproxy.cert"), getTlsFileForClient("superproxy.key-pk8"))); + proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName())); resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), @@ -123,11 +119,11 @@ protected void cleanup() throws Exception { PulsarAdmin getAdminClient(String user) throws Exception { return PulsarAdmin.builder() .serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get()) - .tlsTrustCertsFilePath(getTlsFile("ca.cert")) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), - ImmutableMap.of("tlsCertFile", getTlsFile(user + ".cert"), - "tlsKeyFile", getTlsFile(user + ".key-pk8"))) + ImmutableMap.of("tlsCertFile", getTlsFileForClient(user + ".cert"), + "tlsKeyFile", getTlsFileForClient(user + ".key-pk8"))) .build(); } diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem deleted file mode 100644 index 0665edbdc126c..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEZTCCAk2gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyMjA4NTcwNloYDzIyOTIwNDA2MDg1NzA2WjAQMQ4wDAYDVQQD -DAVhZG1pbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJw3Jfbn0xkW -36kqQjES6Hn+YTZ2jXS5Co2MzsGBsIY0qJ2BbWHSSaMrja4IERUaCQp16SWxPmZ0 -srMm6ErDoap+O70CWXLT3ybYMV5aVwv3ca4uxsedzaw9MpFXfUDsJJ3yre1JpO+t -A/QzJEGq1d6NN49InUP5kB1Rpay3vaxx8hduzqTO+E/Lptv92p6GjOpXi2icSjiA -pgaan2ldGGKEKv2Sc2bfdIDkTq1yDyNmuPET0yD2dci106EW/mPyj81umPKG/o4K -5W18yG/IhXw5W1zlgO1fWCuqva8NCBdu7s1c7hUX8DBx7km4/I7dllz/nYHIfCEQ -Dmj38oQjYk8CAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF -oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp -ZmljYXRlMB0GA1UdDgQWBBQTMzAAOJ9gXvQSS7Be3+qmrb1kVDAfBgNVHSMEGDAW -gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw -FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQAwy8f6hsG0 -85e3SOIztbUnaVxS7wzDeDzR3vCjpXpm4vTToYzN9zx3JHKSdJrB12emVxItwW/7 -bXqBk0n2EdQjRHCuebnY05eFMNGagMEEMVmSLOproQD7VsyALNxCss1JAyRikh70 -W7wgOVeAhqE53UqqrkzTE7Q+8Bag9t3FytHxApY17XglbWkiVcFpQwSURe9Emi3E -aCJCryGJXrBNuCFXGzetSygDEy27+2FeH8S2XsMwUEGLqDDehzvMenVz1xjXtq+s -KPkofAde52NHd4lLkSeBMSFnKe3V7Xxax2OEUsoQRF3bkbpcJSWsKS9ZAA2yrtuy -Nz/aA1F42LuSFPAYQr1kcZ8eSS918RWz+BiJYU2JuUOPd1XUmJXVvZ4CJurWaC7+ -ZD51YdD8E245xd55fsA6/qLx3eE/Kp0dVq+Hxuz6b4yLET0zkGunOe4A3hnRgkOA -XolXCL+VthhWtFGXn8CjpxDnzjahq69Io+dINehqd5aJEgvnHZIK2s7FTqqBBodU -HhyAE94f64z7ziuRhEG54bmBF+MoGyPf6dVn1Mp3+o+YeQ5q6XlKgh+u8jMgmqRO -ikdsVdMqopt/FXh9eFzvQrwOZFLK6JE/edUgb6xvS1jMF5zi2lIlIkq1RBQOr4HT -XDwX4vRtfpDxpsetGVpeq9O07fbMvp+mkw== ------END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem deleted file mode 100644 index 6aaa22c44d746..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCcNyX259MZFt+p -KkIxEuh5/mE2do10uQqNjM7BgbCGNKidgW1h0kmjK42uCBEVGgkKdeklsT5mdLKz -JuhKw6Gqfju9Ally098m2DFeWlcL93GuLsbHnc2sPTKRV31A7CSd8q3tSaTvrQP0 -MyRBqtXejTePSJ1D+ZAdUaWst72scfIXbs6kzvhPy6bb/dqehozqV4tonEo4gKYG -mp9pXRhihCr9knNm33SA5E6tcg8jZrjxE9Mg9nXItdOhFv5j8o/Nbpjyhv6OCuVt -fMhvyIV8OVtc5YDtX1grqr2vDQgXbu7NXO4VF/Awce5JuPyO3ZZc/52ByHwhEA5o -9/KEI2JPAgMBAAECggEAP+Ipq2Q4puz8wGBgu1LhMWp+9Nfcl1xI3YQ01VulBe0o -+2h/g96McKcSBJaV7cw84ENB+kEWpK2amrsRiemhBmkjIvOAAv50Jp2I6u4E5Qbn -PXUxo1Z8UrCgKmHd/hvUCafByuUwBzf5AvebHyOu3JlhnD302mSHtAW8u/pUHd3D -sxJaw1zwQvmlD5ryM2IVYSji7NYCXF0H6V7HfyohTCrQFEWEAdqEDcFR1BUwbPCE -raq7sAiEy+cBUnfV3IOEAffOZy0vSR90/WwERcwrzCdZmpWpTqtbcqtdBPqsSQzX -shDvXd0e43+FSJzCtQsSQ8WzIrp3rgKJUDA1pJQW4QKBgQDJdTcB6qZz8r+4q9gc -q1KAJyMy01Vio1yaqYzXr0C9Z5FW1GhL+4fwer2y9JyD45sb/lP4reFj9S193BNR -C8cdxM5GrWEpzaQ0Dt1s8P1UbU8G6r4NqwI6ORF3CxXZKfXivQcgqBurJGrBdjIC -NwqAzSkX5flBbhfTlJuUH87k3wKBgQDGgjzdIWXab7FZfdUzzrtEwNooBiSEFixm -UAwP5sxL8VkM9wzAKPEVDDQBBoKIga9OESif4S9UUo4tu65AYfxF9Om26K4QrVj8 -HT/U+lfT7xFmPd/GINIbeTSmW0w7Ehpj8SbcQI4Sb2lVE562FlHh7QbHZd0/X/2J -nbgT9MRAkQKBgQCVPAN3o/+SPOzRPFtnQXJoBJYKfIrv+twKpjbzP5vRsvrzO33X -a4kUF5iXDKU0/lJUtl42BXjFt0Xvyit1CiiCYNv9d0pW0UMmXSyiGxNOi3rTQOlw -7pFD2Cqb6NZSfMbtI+I3ytBUQzHiBlCdW3CoYVJjpbSzR37W+WsWm0mEOQKBgQCq -ANObFYUjA1DBMZCrY7rhcL/kUw5myI6RuK/71k7UIwd+oP0cfHOq8N6AmlCkE1xM -4UkHU1SzRFhbNkZPARuJ1etqJ+8afTqd/3axMQyShkVCaG8CQQ1vVegPKFUqqaBM -QzRioC6L/zoYEEt16buKXvHVRpmqMszxVE9XV+HS4QKBgDvs5qloOowS5kcWInrj -yecu5MJvFf2IZpMw7EiKV8VUPeKaiUlqgFj9d9cotUIMauXgBq6f5NBRg7Ike60t -/JJrPtqXY+gdFLjxcKMUVYhomFlQYYg/RJZUBrtkyKBP68abopCYmb59r3ixeNNf -qA1F36mmFtzdjSdtH/dTTecN ------END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem deleted file mode 100644 index b5c7a5dc709a1..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEkDCCAnigAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyMjA4NTUzMloYDzIyOTIwNDA2MDg1NTMyWjAjMSEwHwYDVQQD -DBhicm9rZXIucHVsc2FyLmFwYWNoZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB -DwAwggEKAoIBAQDQouKhZah4hMCqmg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63 -EvNjEK4L/ZWSEV45L/wc6YV14RmM6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0O -OQXVicqZLOc6igZQhRg8ANDYdTJUTF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01 -+ARO9OotMJtBY+vIU5bV6JydfgkhQH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+X -aqTN3/tF8+MBcFB3G04s1qc2CJPJM3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00s -bxa4FGbKgfDobbkJ+GgblWCkAcLN95sKTqtHAgMBAAGjgd0wgdowCQYDVR0TBAIw -ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu -ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUaxFvJrkEGqk8azTA -DyVyTyTbJAIwQQYDVR0jBDowOIAUVwvpyyPov0c+UHo/RX6hGEOdFSehFaQTMBEx -DzANBgNVBAMMBmZvb2JhcoIJANfih0+geeIMMA4GA1UdDwEB/wQEAwIFoDATBgNV -HSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA35QDGclHzQtHs3yQ -ZzNOSKisg5srTiIoQgRzfHrXfkthNFCnBzhKjBxqk3EIasVtvyGuk0ThneC1ai3y -ZK3BivnMZfm1SfyvieFoqWetsxohWfcpOSVkpvO37P6v/NmmaTIGkBN3gxKCx0QN -zqApLQyNTM++X3wxetYH/afAGUrRmBGWZuJheQpB9yZ+FB6BRp8YuYIYBzANJyW9 -spvXW03TpqX2AIoRBoGMLzK72vbhAbLWiCIfEYREhbZVRkP+yvD338cWrILlOEur -x/n8L/FTmbf7mXzHg4xaQ3zg/5+0OCPMDPUBE4xWDBAbZ82hgOcTqfVjwoPgo2V0 -fbbx6redq44J3Vn5d9Xhi59fkpqEjHpX4xebr5iMikZsNTJMeLh0h3uf7DstuO9d -mfnF5j+yDXCKb9XzCsTSvGCN+spmUh6RfSrbkw8/LrRvBUpKVEM0GfKSnaFpOaSS -efM4UEi72FRjszzHEkdvpiLhYvihINLJmDXszhc3fCi42be/DGmUhuhTZWynOPmp -0N0V/8/sGT5gh4fGEtGzS/8xEvZwO9uDlccJiG8Pi+aO0/K9urB9nppd/xKWXv3C -cib/QrW0Qow4TADWC1fnGYCpFzzaZ2esPL2MvzOYXnW4/AbEqmb6Weatluai64ZK -3N2cGJWRyvpvvmbP2hKCa4eLgEc= ------END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem deleted file mode 100644 index 2b51d015b8ace..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDQouKhZah4hMCq -mg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63EvNjEK4L/ZWSEV45L/wc6YV14RmM -6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0OOQXVicqZLOc6igZQhRg8ANDYdTJU -TF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01+ARO9OotMJtBY+vIU5bV6Jydfgkh -QH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+XaqTN3/tF8+MBcFB3G04s1qc2CJPJ -M3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00sbxa4FGbKgfDobbkJ+GgblWCkAcLN -95sKTqtHAgMBAAECggEBALE1eMtfnk3nbAI74bih84D7C0Ug14p8jJv/qqBnsx4j -WrgbWDMVrJa7Rym2FQHBMMfgIwKnso0iSeJvaPz683j1lk833YKe0VQOPgD1m0IN -wV1J6mQ3OOZcKDIcerY1IBHqSmBEzR7dxIbnaxlCAX9gb0hdBK6zCwA5TMG5OQ5Y -3cGOmevK5i2PiejhpruA8h7E48P1ATaGHUZif9YD724oi6AcilQ8H/DlOjZTvlmK -r4aJ30f72NwGM8Ecet5CE2wyflAGtY0k+nChYkPRfy54u64Z/T9B53AvneFaj8jv -yFepZgRTs2cWhEl0KQGuBHQ4+IeOfMt2LebhvjWW8YkCgYEA7BXVsnqPHKRDd8wP -eNkolY4Fjdq4wu9ad+DaFiZcJuv7ugr+Kplltq6e4aU36zEdBYdPp/6KM/HGE/Xj -bo0CELNUKs/Ny9H/UJc8DDbVEmoF3XGiIbKKq1T8NTXTETFnwrGkBFD8nl7YTsOF -M4FZmSok0MhhkpEULAqxBS6YpQsCgYEA4jxM1egTVSWjTreg2UdYo2507jKa7maP -PRtoPsNJzWNbOpfj26l3/8pd6oYKWck6se6RxIUxUrk3ywhNJIIOvWEC7TaOH1c9 -T4NQNcweqBW9+A1x5gyzT14gDaBfl45gs82vI+kcpVv/w2N3HZOQZX3yAUqWpfw2 -yw1uQDXtgDUCgYEAiYPWbBXTkp1j5z3nrT7g0uxc89n5USLWkYlZvxktCEbg4+dP -UUT06EoipdD1F3wOKZA9p98uZT9pX2sUxOpBz7SFTEKq3xQ9IZZWFc9CoW08aVat -V++FsnLYTa5CeXtLsy6CGTmLTDx2xrpAtlWb+QmBVFPD8fmrxFOd9STFKS0CgYAt -6ztVN3OlFqyc75yQPXD6SxMkvdTAisSMDKIOCylRrNb5f5baIP2gR3zkeyxiqPtm -3htsHfSy67EtXpP50wQW4Dft2eLi7ZweJXMEWFfomfEjBeeWYAGNHHe5DFIauuVZ -2WexDEGqNpAlIm0s7aSjVPrn1DHbouOkNyenlMqN+QKBgQDVYVhk9widShSnCmUA -G30moXDgj3eRqCf5T7NEr9GXD1QBD/rQSPh5agnDV7IYLpV7/wkYLI7l9x7mDwu+ -I9mRXkyAmTVEctLTdXQHt0jdJa5SfUaVEDUzQbr0fUjkmythTvqZ809+d3ELPeLI -5qJ7jxgksHWji4lYfL4r4J6Zaw== ------END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem deleted file mode 100644 index 0446700135d39..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFCDCCAvCgAwIBAgIJANfih0+geeIMMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV -BAMMBmZvb2JhcjAeFw0xODA2MjIwODQ2MjFaFw0zODA2MTcwODQ2MjFaMBExDzAN -BgNVBAMMBmZvb2JhcjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOVU -UpTPeXCeyfUiQS824l9s9krZd4R6TA4D97eQ9EWm2D7ppV4gPApHO8j5f+joo/b6 -Iso4aFlHpJ8VV2a5Ol7rjQw43MJHaBgwDxB1XWgsNdfoI7ebtp/BWg2nM3r8wm+Z -gKenf9d1/1Ol+6yFUehkLkIXUvldiVegmmje8FnwhcDNE1eTrh66XqSJXEXqgBKu -NqsoYcVak72OyOO1/N8CESoSdyBkbSiH5vJyo0AUCjn7tULga7fxojmqBZDog9Pg -e5Fi/hbCrdinbxBrMgIxQ7wqXw2sw6iOWu4FU8Ih/CuF4xaQy2YP7MEk4Ff0LCY0 -KMhFMWU7550r/fz/C2l7fKhREyCQPa/bVE+dfxgZ/gCZ+p7vQ154hCCjpd+5bECv -SN1bcVIPG6ngQu4vMXa7QRBi/Od40jSVGVJXYY6kXvrYatad7035w2GGGGkvMsQm -y53yh4tqQfH7ulHqB0J5LebTQRp6nRizWigVCLjNkxJYI+Dj51qvT1zdyWEegKr1 -CthBfYzXlfjeH3xri1f0UABeC12n24Wkacd9af7zs7S3rYntEK444w/3fB0F62Lh -SESfMLAmUH0dF5plRShrFUXz23nUeS8EYgWmnGkpf/HDzB67vdfAK0tfJEtmmY78 -q06OSgMr+AOOqaomh4Ez2ZQG592bS71G8MrE7r2/AgMBAAGjYzBhMB0GA1UdDgQW -BBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAfBgNVHSMEGDAWgBRXC+nLI+i/Rz5Qej9F -fqEYQ50VJzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG -9w0BAQsFAAOCAgEAYd2PxdV+YOaWcmMG1fK7CGwSzDOGsgC7hi4gWPiNsVbz6fwQ -m5Ac7Zw76dzin8gzOPKST7B8WIoc7ZWrMnyh3G6A3u29Ec8iWahqGa91NPA3bOIl -0ldXnXfa416+JL/Q5utpiV6W2XDaB53v9GqpMk4rOTS9kCFOiuH5ZU8P69jp9mq6 -7pI/+hWFr+21ibmXH6ANxRLd/5+AqojRUYowAu2997Z+xmbpwx/2Svciq3LNY/Vz -s9DudUHCBHj/DPgNxsEUt8QNohjQkRbFTY0a1aXodJ/pm0Ehk2kf9KwYYYduR7ak -6UmPIPrZg6FePNahxwMZ0RtgX7EXmpiiIH1q9BsulddWkrFQclevsWO3ONQVrDs2 -gwY0HQuCRCJ+xgS2cyGiGohW5MkIsg1aI0i0j5GIUSppCIYgirAGCairARbCjhcx -pbMe8RTuBhCqO3R2wZ0wXu7P7/ArI/Ltm1dU6IeHUAUmeneVj5ie0SdA19mHTS2o -lG77N0jy6eq2zyEwJE6tuS/tyP1xrxdzXCYY7f6X9aNfsuPVQTcnrFajvDv8R6uD -YnRStVCdS6fZEP0JzsLrqp9bgLIRRsiqsVVBCgJdK1I/X59qk2EyCLXWSgk8T9XZ -iux8LlPpskt30YYt1KhlWB9zVz7k0uYAwits5foU6RfCRDPAyOa1q/QOXk0= ------END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem deleted file mode 100644 index 6c2f4295c9dbd..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEZTCCAk2gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyNTEzNTYwNFoYDzIyOTIwNDA5MTM1NjA0WjAQMQ4wDAYDVQQD -DAVwcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMieX78Yj8OW -eBHAneRaCCoO8Qrpj8zGo7h9lCdmi1lBDh1uR2sDbotiHGfJzQn836WcYmyeAvfn -qvgr9HCXmXdLgmJ3GT/LVu5GEm6msSDiZQPr9so5lQVioisK4UwJROQsE/J52cyR -9o3H6M4FKb6QpoobKa62fSfTumwwulaYaDJuRRGoGIkcRuUQ59EWAaDkD3IcDpAn -9mTbnE4Iz+JxSrsZ5DJ3X/m/AqyLWtj6GAfyK9a1dhNdlf2x4JZT1QNtojiBXt95 -OIZyRBNbHMFniq5gel6wdBkmJWutfcTct7wKa2LCxLpKoDIc1HWoL3+RUzOKxYIP -0qXEQ3bmONkCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF -oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp -ZmljYXRlMB0GA1UdDgQWBBSgsgfmDbXEkrrpHUC9GnDDjxaKizAfBgNVHSMEGDAW -gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw -FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQB+uZ2OWR+G -sRqYHEeVqUI8G2p8Np8eC/onGpBL8Gj9SGxIQcIPtqHPUlCe9fd/96JOptOGRYEB -BmUCaCmQ4IgMW6e6fArka5IB4XXIgHFXyQ6ImTvjDavzlVw06zn9S4dLwVzsRBg+ -GS9svtq23W+f5rEN5N+7LhtcbclfiG4VCCqDG5VhkzEok+SRamDI8rDRZtodMw0O -/+L+xaaQPUPjX8KUlKn4uVpCDbxUzHonlCPzbkHHm5su0D4ysjJIy3/y3yow6JE/ -02L7PZkmkmw3/V+84T3X8/GD15sVUv/3v1gXEBxYwAs+RNTJ0APvMEMSvCq0AMfF -bPMZBuAGNBG7lv7TovzHgGFKXT7du5OFF/qjAsEffhbo224CB96fgwvvndwHHBFh -J06BvHZG1i9dDVhUKoB1owkWrE4RZv2ZKEtZYgizzSmzZRHtARo0t1Byc5djx1tX -TkJOHshNqJZOY1ER0DPaVQgKI+PRTbEdj/xPGRX3ebSqDmilAfPXshqgElfch6Yl -f2V58TyCnjXOibvkG9D5OyCdWLEECumOZgYar0KZgNfrvTOi1OKXnX1fsbh29fWA -ICZRcdmjkz79zQXY2SuzCWlskuXPKAmW1AMqs+l6ormmKfzIUx3Yriy4LqIIYY1v -uQD5vghZmd9HUg2KaXfSGD9stGCD8KhntA== ------END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem deleted file mode 100644 index 70e0107f2741f..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDInl+/GI/DlngR -wJ3kWggqDvEK6Y/MxqO4fZQnZotZQQ4dbkdrA26LYhxnyc0J/N+lnGJsngL356r4 -K/Rwl5l3S4Jidxk/y1buRhJuprEg4mUD6/bKOZUFYqIrCuFMCUTkLBPyednMkfaN -x+jOBSm+kKaKGymutn0n07psMLpWmGgybkURqBiJHEblEOfRFgGg5A9yHA6QJ/Zk -25xOCM/icUq7GeQyd1/5vwKsi1rY+hgH8ivWtXYTXZX9seCWU9UDbaI4gV7feTiG -ckQTWxzBZ4quYHpesHQZJiVrrX3E3Le8CmtiwsS6SqAyHNR1qC9/kVMzisWCD9Kl -xEN25jjZAgMBAAECggEAJJyCbKVW1yLGlrbIGbw0cTh41Lz6+SvnBOwl9WrJU2iD -4usVLXpa2iT1ehthx8jWJ6r6a0gK0qL8mH2tBj8kSpkFGmMRwIqjOqifBIJ3IMEw -Hh8Z0p3fjDQL1D8QDohCgkFpAn8qOCMLE6S/35khnR1Yxytd1/yFqpcBFm1uFA85 -dYisjPqWm/IZIU5rH0zgKAIhtvl9abnoi93443EHsKpRAW1gwRXx9Aak7TV768bZ -tELBsaTnXnNzamDiaimmxEOlqR9O0W8JE/31KFL26JcVmsTRG7sMpoUxCEMjOuGZ -J30bXFZUW6NrDpFsQ7uTqD6TNn2971N8KFCLnC/JYQKBgQDo82axC7L7n7BYTu18 -dupeT7n5dTBD/I3l0KtT05xiZA8GZr2i+pt+/aWzCzK4/Ee4jb4/o8CRQRB5v5mo -c9lc+BaoAIQiwiw+aufT+UojrcijrOMEL5Zk3zdZ2rcEoAsVvqtejNnwLCGI9Rnl -gp7n9oRhwDIv9Fu09snUougE3wKBgQDceAGKUB8pGd3eEya/0jU9J60LsKbcJSsN -4v1S5LiPtHOyhr0g4x/LibMP2PJhG3tJ1bgpaGmn9du2D20M6ukRhIYyn/7G+N+A -oqryyvO1MMYnhc4IEQvWrzDnBM0hV2bdjp4s/1ASVHVRwk8+orqxysIJ1D75nnRX -Tyfl6HgBRwKBgQCfVlWIhiMPv6OkU6BXgRNAHTJs8f5okmgQqNF3jgequRwZ2c6e -muIfU6myNNel9lGsZ6+Y4g4GjMWTMT4OHeewkrUUhv3atIwEyaT2tc5DZ0wUwF2r -cE1jg9bdbB/BVyMd5YRcMOWlRNpPTq8+8EB3E4RrREZPzMmplyBohGFFawKBgGjc -P0dM8nU3E1rj2wNTdPUAYQL1Y3fDyeWR+BEsLkhTeNAJ2/y/akkB1oQMGMRtMMee -ejhfrBkyC+1dCu4g8PffA4EirihvCMcDF7HhK+cbKrRzpNobWXkj3GuU0ggwrQFm -Kv+V87y0JRTdCZnuBkQ3/vBz3fwWDJnWUVC9sA5TAoGBAN4t2gJCZax5yInUgyWi -Tgumb2qVWsGBBLMTKIsrkK/KphzgAhHcVhDCybA79TmIM2FfIlpf6TE7Mv4NI055 -ZJzHX+GMT5Czy2Ku9MJD3PLTFN5MjYb6g92fKViLDMI6fwzTHB8xPJ7Ob9bV2srS -ZlmKNXTkZFk3/orK4WdCZufz ------END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem deleted file mode 100644 index 01b69eacc32cb..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDCDCCAfCgAwIBAgIJAMPzPm3QygzRMA0GCSqGSIb3DQEBCwUAMBAxDjAMBgNV -BAMMBXJhbmRvMCAXDTE4MDYyNzA5MDA0OVoYDzIyOTIwNDExMDkwMDQ5WjAQMQ4w -DAYDVQQDDAVyYW5kbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKR -nN1y/aBrhbNM6kDfBmWTaKFuANEZ+VST++AVImssN+FyOL2ukxuT74JW0b9g9scl -aTYzUBbKuTrMIf77RPIJY6R5E6JWn12UhDpnLa9eYGOkyPyoSp5hdyJj3spElWkl -eGRY8g9zsWJO6QDpTL4g2VAtWTjFIVI3l3enNgNp3hJnJkdvxyBuRN7Szg9hGPmQ -Tp9rsKkZQD4SJ9/WMAlqSqeB6rSn20rydMJCnI92TT8hlv1wZcVXSVrJypzZb27t -aAi5o9GEtag3aXalR8zL9DiUk64338A0fYgf2/GSjomZehcg17h8tElYMFV6hJqS -95gM18nfwldttbihjKMCAwEAAaNjMGEwHQYDVR0OBBYEFAzmiWmqQIB10sSoSAzb -claQvBPtMB8GA1UdIwQYMBaAFAzmiWmqQIB10sSoSAzbclaQvBPtMA8GA1UdEwEB -/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IBAQCN7nup -rGd4sVuP2vvcRbhGo1JCWDS4Uwe7nbSR/4QTAlyUYK8KIwTvfxeEeO4yBGmiDzOE -+W2y0i22qVnShNCBZ50Evm+FkIEcwlnLtAJt6zIrE4cWdqW1a6CMp9MJkONjACOx -3Y560SlqQuoKa41PtXUnk8tFT8Asz7b3NtEfmrBlZJpSSM5exBOCH/Enwzac890u -zJuweTz4aO8aefunkPMKlBPliJ6EkEYclrjJqxi+4VrEMtAIRxwKXPvnHnXESaF7 -yUE4qrjQa4B+Bn1DWqfLcHJCVDxqYrXiEAN/YtX0UOqxZvGd49eNIs2vQkET0+55 -J3hW25OLSnPjAO/j ------END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem deleted file mode 100644 index 86a7fa387a8c7..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCykZzdcv2ga4Wz -TOpA3wZlk2ihbgDRGflUk/vgFSJrLDfhcji9rpMbk++CVtG/YPbHJWk2M1AWyrk6 -zCH++0TyCWOkeROiVp9dlIQ6Zy2vXmBjpMj8qEqeYXciY97KRJVpJXhkWPIPc7Fi -TukA6Uy+INlQLVk4xSFSN5d3pzYDad4SZyZHb8cgbkTe0s4PYRj5kE6fa7CpGUA+ -Eiff1jAJakqngeq0p9tK8nTCQpyPdk0/IZb9cGXFV0laycqc2W9u7WgIuaPRhLWo -N2l2pUfMy/Q4lJOuN9/ANH2IH9vxko6JmXoXINe4fLRJWDBVeoSakveYDNfJ38JX -bbW4oYyjAgMBAAECggEAf8cSqKQQOSq3kYYIWkM9IJJK3LkKfJZJg+wg4Eg/SNFr -azeAwrqZKbLCQFI/5OJNtFNg5hfxx11pDlnkOcEzpL5zPs4k7pVtlFkiBWivmD3A -W40fBSynuI2l4kX0tmg9QfA+JhA/pi7zT5WHxc8ryyFWX7kTjzwAjASbrlNIo0d7 -e5SMGcZUeH7m/+pCw626nhpexAJ5Fm9/uc0Zwh4xrzS6j2xoqhOqvQb175V/Iv6/ -8nBq/VjE5LqoP57gqw/Cs4zwwBFayr/WaitGVikga2eZI3q/GFMtez27cdMxEZhI -9ZP1y8kzlzs1QPjdJQUxIE+/zIm3w8IL4iO0SQE5AQKBgQDmSYazZBkmixUzvBhg -8tGTWJOAG0NhgBamkbaR6s1L8ou9A5MGbFbRBbc5zRmiyMC2Q4D6Xu0ut2N93tUr -DkxZa35dt3HWMRmNByjHtpMzapYnmDpuF4k7dy+81GkUbd2ZDYadnknjiaFaDZ7j -9JxUENIwzzLhFMx7emBQ1+3zewKBgQDGgcewYthAsCTAcBtGKw6G39jAYCpr2N7X -/B5W5n4VXVHKq35wK1bHRnHj7dkm4sH572XB1tPoMrnQ7j9i+XSkc1Ixo0iHr9H0 -QauVQKQqzkvI5cSaWS5N6ZRk6GOGxI0SwYcNNdqJzEEJuYHCTUpff4d6utMgpTZ6 -SQJw8u0O+QKBgE2VEcNYAr0geDkgslnfFEn+ulqbVL0BSSA+0PIh154xjXBVRvAQ -CcOLmGnptixIU9xTq50t49wsPmGGc+x4ebJaa40pIznU+tWvRsbZtIfK7eFTAMRc -O4iEI9oK+Ye/Z7uLegGZ9SyqDmjnU9NaclxD+nwlIfAAcM9csBwsUucHAoGABjJ7 -B3iug6Z8Hz3gvBoQBAns/GSELoXAv0FxuQjNGuGk8gzUj6/qr6H1YEZGpz4hDCp7 -JMgOKYub3XfypqZfC9tFz6LnWsUUaum575jrByMVnpn9v0vVdD08ksHmiYiNVu6P -xsvNnMuxpBoUgPpkvgJ/Okem27gMsViiKOCMohECgYEAt3MwlVILN0t6RPBCpTKB -nrh6cxx8kQ2TI1UP44WciBkGVvIQOEbl+1955lt/LAECzooiBJOkKubcPK22sJPS -MZauEwdAqRQj/uUs3oRISUjo0H/IKGI3O23jhIAI2XC/zkQTE6dKjhXxJIl+5a8f -v9VhnIlgSfaxu3R0l7SvQlo= ------END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem deleted file mode 100644 index 9656e2c8ba0d2..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEajCCAlKgAwIBAgICEAQwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyNjEzMjUxN1oYDzIyOTIwNDEwMTMyNTE3WjAVMRMwEQYDVQQD -DApzdXBlcnByb3h5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA43Sn -ys/h3wxJ/IBEzbJdQAKCxU4of5KgTDzFoOaS8C63Nwbjgy0qcWYdRDceP4lJIKSA -1+JZ+R1opyrfVIC2D9oDJFIJTFfXy9G9VYDccwAONPgAamvRzBnYcEu8fqM+Ohle -kZltKktAHnX3WtG3RyxEL5nYzlMlGwUXJu3Rxc9SlkYSxERzjWHikGqGmXLX2qB0 -k6oyxTrK+4+EHk3khbEIqyQZSOFbD7NMfnRy0CWFv/9T1shgjAIBCake6jY7lwaT -S7JvbLfG6ABf5xHMxoWLXa2qwb+Ar43Ff9g8kKZwFOxMeGcwkzyJPBbWytFxaWn+ -R2RHhTaVCGc22CjdfQIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB -BAQDAgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQg -Q2VydGlmaWNhdGUwHQYDVR0OBBYEFHEgLhfH0Z1QlLxeQbG7YZyBlz1MMB8GA1Ud -IwQYMBaAFFcL6csj6L9HPlB6P0V+oRhDnRUnMA4GA1UdDwEB/wQEAwIF4DAdBgNV -HSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBAK9q -3YnEa99Cq3Vo3g6PE/A/xOE97seVNuavqyBcVv9PrTydb5XG4jPkYU3xOYXIc8rA -A4gzd+AsGO9rjGMPGGjQJI3JO/3BCeBJMkn50C/rM1yWVMHnVyFcJlg16xaWtj1e -2Jk8egJW2gSYyF+N2TdzI7tOb002GNr36posnqO+IOoLapyHFBxxUjsPDRoo8fJn -myWsV1Y9oRUZyJlfIAJsu85ew7gDBY2jaiEiopzour3uU3C0N7gYni2OmVwfr6J8 -R2/Jp43BSD5sYOW9RAJIEEXef+InYtz9HTJvKu2LsWwIBkaztk29tJcDE+1La6Sw -0dF0YkUwnXoGQFjiV+8pXX3TF5glXKj1rU8WfNazF6lqslB6DmdgR3/FQ6Z2sE86 -d9hVtayZIGlzU0rWmBBtr++7Wo88nmzAtd/xbZMFG8U//+Q2AvJT2oVGtqM48+al -rnsN/gYrLDr7RC14bHIuO1v6ZL/rAi7SPKrKYAyQVTAcRuW516SxxR6S1Xa1ITnh -rwgKg13eQuwu3iigguIS9XL6nAXabBIxBxMl6o2YlyIPekKYIcQmpqhkavJ6VOgX -iq9VdY6fIJVfmxNZuwM3/28k7UeUAfhI2SSVH4ZURbPiGGH2wukc1QkmtZ2cNa35 -C1y79aqJbIa3ErqLFPj/fM+34x8L7QHPq6RfaODa ------END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem deleted file mode 100644 index 2e4140b8fb392..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDjdKfKz+HfDEn8 -gETNsl1AAoLFTih/kqBMPMWg5pLwLrc3BuODLSpxZh1ENx4/iUkgpIDX4ln5HWin -Kt9UgLYP2gMkUglMV9fL0b1VgNxzAA40+ABqa9HMGdhwS7x+oz46GV6RmW0qS0Ae -dfda0bdHLEQvmdjOUyUbBRcm7dHFz1KWRhLERHONYeKQaoaZctfaoHSTqjLFOsr7 -j4QeTeSFsQirJBlI4VsPs0x+dHLQJYW//1PWyGCMAgEJqR7qNjuXBpNLsm9st8bo -AF/nEczGhYtdrarBv4CvjcV/2DyQpnAU7Ex4ZzCTPIk8FtbK0XFpaf5HZEeFNpUI -ZzbYKN19AgMBAAECggEBAM1JAwuD1drmj3wKFI8F1S2pVndXFCwXnP9RthiDIckO -kKNkX0CMKgtQ20cu6+jyMgL5FaRCkWvJxCNkCU6OIENsQ3urYuL5QTWeZeBevhg4 -y5m43z8tcptgFD09zbEKCmaLcRO9wo3yfrs/QvE/58efxyajFs8YsZuSW5Px/msl -AFN8qGY0q0lQbQPK8cPYT4OnW9isqEjkvHq1Q+ryv4cxsGWtBZYmJVeLVHzClihi -lODdN6YsDcu9jLwsA8o2WSBRsbLHK6caW4FdEwgOLckc0EGGEU64VEB9ZOwgcwYQ -M9mZDs2w7qk63YnDH4KmpoP0Oc8N+bG7Pkifd7f8HqECgYEA8esMQya6Qln4O7Qo -aI8X9t5gCpq+bh/P+7vYdBR/9St5VtE/P0yuaQdT2e5t+Ei4ujqHvJ0GTPmqYIJf -2DNvJMu8EnXv+nqKQCsyMc/nqQN1CcRGqZssH1V/ZIQLRUA0cN4hzE0eHITwp6U9 -vD/WaE3mKX/XHthIjc8hnuoajOkCgYEA8LIYMgyD3wClWOKe5TkBumI0KvA9tP90 -IFHu1wLNl0tKBpsXkzzxiav1FMX3K7B29vxukrs946KRESHzRg4c5ULPnMmwcQyx -AortKJWrGVsna/QDWPRutXSN1XmnjKWsHmTpQQqLfaRvLW+Hd34+EVdVh9sVt/y9 -RucnBvxcX3UCgYBebm/U7pMaP2BkfcigN+sU1G0M9qaK+iQHkaXGehIQs62js/5K -STZzjQawNR/8IPbqytodR/YjqflVvs6G6FzkMhrx4dORJLA+qB3pz8wP72eKLnGe -1xF8EbWumNSFbbCKtkrfIuM0IriF2Dym9QxOnsnPPTXNtoNrx4TKMXu3sQKBgBq9 -noSI8WmoF7adTsvmnnOHj4YptKFUNCGXGLLYg+DII4xCVMct4SPLb+oD6Gb5Lu5X -sy0oEkMk/3roy68/yCQMXSZtHeYhY9UFfD2jCyRBBUswC+MpHNeaAFv0LRIqIcoq -qeNo+YBW8WcZ2fIDm3+vtTfntiz/rkOfUK2tAdI1AoGAL2LVDF5e1meSRocEoy8e -gqrshrhg+KBqYmcjtd4Iup4WvR6uyH4qE9yFLLHFZ04pZzXLHcPfqgOSP8qTxgH3 -h0uqtcYmejS/yl6PbC9OPXcSkMgntRkTbU9Ug05ijfN9NRnW+8WXvi8Z6DKed1P1 -9PxwA75P9o0h00mMk8PQitQ= ------END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem deleted file mode 100644 index 072f2867fe62b..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEZTCCAk2gAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyNzA4NDAyNVoYDzIyOTIwNDExMDg0MDI1WjAQMQ4wDAYDVQQD -DAV1c2VyMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM5iqgr4PUUZ -AW9MDGP5cBaSJALSPV63m6M/IoovrMWJ9CGtcQfZTUHwDorIlXgQ6H/KufmsHW0Y -OQbChLSTDB14D0jSMtyv6+ibSoE1ZEl2SbB1miLd0P5AS9YmzzEW2+bx0zJORLYD -PzJ1Nh3/kQlRs04IECki291WZiVRzX2JRoL7kMtOAoKJqQfsT14Oi9EAw39VhLeB -uc/Mx6Jsutq/YdXakoZtQbfZka2MMfLXgMDLIPDqbU+09q7au2dq8RjGzrWnxnOX -o/XQssrIbwzJJYASBsgAAtnAw7bPzCX6+cL6PZVRyiEZov0HKXyRyvrbQ5hyEMuS -3dHqoKt0fKMCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF -oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp -ZmljYXRlMB0GA1UdDgQWBBQ7NSD6lx6Vq38cEoD5l7FHs1Ej8DAfBgNVHSMEGDAW -gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw -FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQC+cU7ctY7o -eTW+oHTq9EGJUMwW1fOww4QDrHtgZT4OYkO88zxQV2Cr050p8eaV5dHXZBf9/bRV -7hPNV5+HpQhb9TZ9xK2WRZ2QV7a/UDyUnksVKGSK9tNZMZPueOEB19e4bIBcgnQa -5i9sgZr93na7pFOY7lBQy6gfaOcnejYHmvVqIGaZBVH8rkEsGhhkJxy7qkpFNKgf -PGiIRo9L0WYqDCSiaICeCiteJwIfjsUFJKF0YnpXZq1kFfQscnleg60MWZAXvacp -tAciE51Ow60cqQWER66iwqnBSPD4l91SxAaGQAmalgCioGsYSbojXcOvRidhYJ2T -3YwCpqlC0qC9D2ZmNoukb1a0Pi03MuSJwD/8v9eqwEW9dFAzdnWDzTZMN9CfdjVh -2qiO5o5Si/X1Dmjdk2F/EM62YJQBAlkZBetFJ0o2QPGTSD+zrpfITIW8Pu+/5zcC -MZdzyUf0p1GO2Kn7wmqPQjz59zABagmxCNks8HeqPnzmWuADMaggb0nOmrBACE2x -b9XR6/xaXpwTRf0h5N3evivzUHo6XVw8A3gVUNoBm9Of3PlAsjM4I4SWFb6nrwYv -RnI04c+R95Su1fMc2wky0PmW+iWRTaEN/cUdX1SF6jo1nRLELGcbMSGUI6UI8kff -crvCz7uLu7Lr5/CKnEm2bSCZ4eIQpOs4nQ== ------END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem deleted file mode 100644 index eec5c94c0665c..0000000000000 --- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOYqoK+D1FGQFv -TAxj+XAWkiQC0j1et5ujPyKKL6zFifQhrXEH2U1B8A6KyJV4EOh/yrn5rB1tGDkG -woS0kwwdeA9I0jLcr+vom0qBNWRJdkmwdZoi3dD+QEvWJs8xFtvm8dMyTkS2Az8y -dTYd/5EJUbNOCBApItvdVmYlUc19iUaC+5DLTgKCiakH7E9eDovRAMN/VYS3gbnP -zMeibLrav2HV2pKGbUG32ZGtjDHy14DAyyDw6m1PtPau2rtnavEYxs61p8Zzl6P1 -0LLKyG8MySWAEgbIAALZwMO2z8wl+vnC+j2VUcohGaL9Byl8kcr620OYchDLkt3R -6qCrdHyjAgMBAAECggEBAK8J8QvytAxJg/T/+7ZC1PTfp1kZNGGDuaV/o2ytuIul -T//MGQQ+IY8d6Ud9jX9SX84agxalCiP/mkYIbgK0gF7x94yccfTH433ZTxw8yzye -7SqS41JU7K7mmysaqTkKGSFK0gNlbFMud8f0rxxMJ5dOypMQtZwd63lSkLlwIqcn -YhKcAdXM4WGv07MFdwAXvrK2trhBBkCA08ZzyVy4lWE+cVfEkwH6O8EdGchl7g4U -LqIkYzufQWKdH9frt6N3qEV/qs0s1qipVzV07GaPpEdK/G5exJ7NjZaXJkHOuMbt -7ae1zJBexW41/++fjzsWSYljvSEphjqwrGYrr+tMGNECgYEA+Sg1sHcYO4Bg16qG -c8XM9g8DFL396X4MnMh+AFrDV8dbdz39x9fFtpKEfqaAZrG6I8fW3RkMLsJvJ4GI -KDJNz2ezEMbNKlXE9UzvAsrdvWNWDGJvrDu0CbJg2wtBeVEBIUYetSdNHoEwY7ZJ -QHnwbxYYrUFIJ34rI0SQ17/Z4vsCgYEA1A27jYTWr86JmvjQDUiJcJsbhpa/6OzZ -n3KTu0n0oCvl7uvvjUfhlzmcq7kyY0oO1C5TQHLiqBaI5hXw+iYwnESLIn7BwhMX -dcxglvXx95h1NqHYX9GnURl2QtrjmuY5yx+itfmZCRkRPO3c5e9uZyf01CeE4uwl -hDNh0RrSXHkCgYBvlQhmXQ+nJhk4vI+2LXFbCOISWfvqo562YDu9oOg22Xsm7cZH -x2QuHXPk3GBInXOFLqwVHHCOSFlLUgFOLykVp5VUABRFz1+Dk86+a2fetywEI9lr -QtmgNhiWQHY0BIkDA8ogytcIwEaRgUNQ8sswlK68eK39sc1T4BMV7D+CHQKBgG3g -+8lWBwSsKgOCYBQx/P27caTo4mJosE99yG0o4jhI5ulJmiSEFbINqVAWM7TdQBfU -NVFU9nuQybknr2l/dnrSzaG/Otk8mVBx6a7vnETm2/3GGV91PJS6c9wqnfu6xkGp -j99piVH8ikEfI/KFgZi0TJnOLH6FTN9W3J3EnzJJAoGAE4ZLLi+2RP4ZYXI11CJC -BSM1AEpqemgTUSidZyTiJJMGGrC7tyEba+4TIqeGgvD74p57XFbN7LOJWmnAiK8b -BLhQqPgOCXqrna00GnNKKx7AzgYHqlq03tGlX858aH18rkSRVk3UhkTkGTSr3iQn -aJeN/0HbpGr67bimf4bqlCY= ------END PRIVATE KEY----- From b07f2a9b85a0f22b89fed5b51ee8625b00b7262e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 11 May 2023 06:48:59 +0300 Subject: [PATCH 186/280] [fix][test] Fix flaky test ConnectionPoolTest.testSetProxyToTargetBrokerAddress (#20293) --- .../client/impl/ConnectionPoolTest.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index fb564bd5083c1..fe0aa4dd4953b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -154,7 +154,7 @@ public void testEnableConnectionPool() throws Exception { @Test public void testSetProxyToTargetBrokerAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); - conf.setConnectionsPerBroker(5); + conf.setConnectionsPerBroker(1); EventLoopGroup eventLoop = @@ -176,19 +176,24 @@ protected void doResolve(SocketAddress socketAddress, Promise promise) throws Ex @Override protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws Exception { final InetSocketAddress socketAddress1 = (InetSocketAddress) socketAddress; - final boolean isProxy = socketAddress1.getHostName().equals("proxy"); - final boolean isBroker = socketAddress1.getHostName().equals("broker"); + String hostName = socketAddress1.getHostName(); + final boolean isProxy = hostName.equals("proxy"); + final boolean isBroker = hostName.startsWith("broker"); if (!isProxy && !isBroker) { promise.setFailure(new IllegalStateException()); throw new IllegalStateException(); } List result = new ArrayList<>(); + // All 127.0.0.0/8 addresses are valid local loopback addresses if (isProxy) { - result.add(new InetSocketAddress("localhost", brokerPort)); - result.add(InetSocketAddress.createUnresolved("proxy", brokerPort)); - } else { - result.add(new InetSocketAddress("127.0.0.1", brokerPort)); - result.add(InetSocketAddress.createUnresolved("broker", brokerPort)); + result.add(new InetSocketAddress("127.0.0.101", brokerPort)); + result.add(new InetSocketAddress("127.0.0.102", brokerPort)); + } else if (hostName.equals("broker1")){ + result.add(new InetSocketAddress("127.0.0.103", brokerPort)); + result.add(new InetSocketAddress("127.0.0.104", brokerPort)); + } else if (hostName.equals("broker2")){ + result.add(new InetSocketAddress("127.0.0.105", brokerPort)); + result.add(new InetSocketAddress("127.0.0.106", brokerPort)); } promise.setSuccess(result); } @@ -203,23 +208,19 @@ protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws InetSocketAddress.createUnresolved("proxy", 9999)).get(); Assert.assertEquals(cnx.remoteHostName, "proxy"); Assert.assertNull(cnx.proxyToTargetBrokerAddress); - cnx.close(); cnx = pool.getConnection( - InetSocketAddress.createUnresolved("broker", 9999), + InetSocketAddress.createUnresolved("broker1", 9999), InetSocketAddress.createUnresolved("proxy", 9999)).get(); Assert.assertEquals(cnx.remoteHostName, "proxy"); - Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker:9999"); - cnx.close(); + Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker1:9999"); cnx = pool.getConnection( - InetSocketAddress.createUnresolved("broker", 9999), - InetSocketAddress.createUnresolved("broker", 9999)).get(); - Assert.assertEquals(cnx.remoteHostName, "broker"); + InetSocketAddress.createUnresolved("broker2", 9999), + InetSocketAddress.createUnresolved("broker2", 9999)).get(); + Assert.assertEquals(cnx.remoteHostName, "broker2"); Assert.assertNull(cnx.proxyToTargetBrokerAddress); - cnx.close(); - pool.closeAllConnections(); pool.close(); From 4b24c9e85ad534c2087063e7873eded4f9ab7a21 Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Thu, 11 May 2023 22:28:40 +0800 Subject: [PATCH 187/280] [fix][broker] Fix NPE cause by topic publish rate limiter. (#20302) Co-authored-by: druidliu --- .../java/org/apache/pulsar/broker/service/AbstractTopic.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 6245ce19eebc6..8269dc0c3d16f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -115,7 +115,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener Date: Thu, 11 May 2023 17:29:22 -0500 Subject: [PATCH 188/280] [fix][fn] Correct TLS cert config translation from broker to fn worker (#20297) ### Motivation The `initializeWorkerConfigFromBrokerConfig` method converts a `ServiceConfiguration` object to a `WorkerConfig`. This is used when the function worker is running within the broker and when pulsar is running in standalone mode. The TLS certificates that are trusted by the broker should also be trusted by the function worker, and the TLS certificates that are trusted by the broker client should be trusted by the function worker's client. ### Modifications * Improve the `PulsarFunctionTlsTest` test by adding awaitility. Before this change, the test was flaky on my machine. * Fix the mapping of broker config to function worker config. ### Verifying this change A test is added. Note that the old test doesn't technically fail due to the misconfiguration at the moment. The error is in the logs. Here is one of the stack traces that is gone after this change: ``` 2023-05-10T23:19:27,087 - WARN - [pulsar-client-io-253-3:ClientCnx@344] - [localhost/127.0.0.1:59715] Got exception io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:499) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:833) Caused by: javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.handshakeException(ReferenceCountedOpenSslEngine.java:1907) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.wrap(ReferenceCountedOpenSslEngine.java:834) at java.base/javax.net.ssl.SSLEngine.wrap(SSLEngine.java:564) at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:1042) at io.netty.handler.ssl.SslHandler.wrapNonAppData(SslHandler.java:928) at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1418) at io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1256) at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1296) at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at java.base/sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:439) at java.base/sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:306) at java.base/sun.security.validator.Validator.validate(Validator.java:264) at java.base/sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) at java.base/sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) at io.netty.handler.ssl.ReferenceCountedOpenSslClientContext$ExtendedTrustManagerVerifyCallback.verify(ReferenceCountedOpenSslClientContext.java:234) at io.netty.handler.ssl.ReferenceCountedOpenSslContext$AbstractCertificateVerifier.verify(ReferenceCountedOpenSslContext.java:779) at io.netty.internal.tcnative.CertificateVerifierTask.runTask(CertificateVerifierTask.java:36) at io.netty.internal.tcnative.SSLTask.run(SSLTask.java:48) at io.netty.internal.tcnative.SSLTask.run(SSLTask.java:42) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.runAndResetNeedTask(ReferenceCountedOpenSslEngine.java:1496) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.access$700(ReferenceCountedOpenSslEngine.java:94) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine$TaskDecorator.run(ReferenceCountedOpenSslEngine.java:1471) at io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1558) at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1404) ... 21 more Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at java.base/sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) at java.base/sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) at java.base/java.security.cert.CertPathBuilder.build(CertPathBuilder.java:297) at java.base/sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:434) ... 35 more ``` ### Does this pull request potentially affect one of the following parts: This affects the configuration, but I think it should be classified as fixing a bug, so it is not a breaking change. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: the modified test passes locally, so skipping forked test. --- .../apache/pulsar/broker/PulsarService.java | 3 ++- .../worker/PulsarFunctionTlsTest.java | 23 +++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 5fc9920d0f215..f833674ceeb0d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1859,7 +1859,8 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu workerConfig.setTlsAllowInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()); workerConfig.setTlsEnabled(brokerConfig.isTlsEnabled()); workerConfig.setTlsEnableHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); - workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath()); + workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getBrokerClientTrustCertsFilePath()); + workerConfig.setTlsTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath()); // client in worker will use this config to authenticate with broker workerConfig.setBrokerClientAuthenticationPlugin(brokerConfig.getBrokerClientAuthenticationPlugin()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index 00db6a65b16ea..246d980d6178c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.functions.sink.PulsarSink; import org.apache.pulsar.functions.worker.service.WorkerServiceLoader; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -65,6 +66,7 @@ public class PulsarFunctionTlsTest { private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; + private static final String CA_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; LocalBookkeeperEnsemble bkEnsemble; protected PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT]; @@ -111,9 +113,9 @@ void setup() throws Exception { config.setAuthenticationProviders(providers); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - config.setTlsAllowInsecureConnection(true); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); config.setBrokerClientTlsEnabled(true); - config.setBrokerClientTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH); + config.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); config.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + ",tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); @@ -141,7 +143,6 @@ void setup() throws Exception { workerConfig.setUseTls(true); workerConfig.setTlsEnableHostnameVerification(true); workerConfig.setTlsAllowInsecureConnection(false); - workerConfig.setBrokerClientTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH); fnWorkerServices[i] = WorkerServiceLoader.load(workerConfig); configurations[i] = config; @@ -163,8 +164,7 @@ void setup() throws Exception { pulsarAdmins[i] = PulsarAdmin.builder() .serviceHttpUrl(pulsarServices[i].getWebServiceAddressTls()) - .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH) - .allowTlsInsecureConnection(true) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .authentication(authTls) .build(); } @@ -216,6 +216,13 @@ void tearDown() throws Exception { } } + @Test + public void testTLSTrustCertsConfigMapping() throws Exception { + WorkerConfig workerConfig = fnWorkerServices[0].getWorkerConfig(); + assertEquals(workerConfig.getTlsTrustCertsFilePath(), CA_CERT_FILE_PATH); + assertEquals(workerConfig.getBrokerClientTrustCertsFilePath(), CA_CERT_FILE_PATH); + } + @Test public void testFunctionsCreation() throws Exception { @@ -233,6 +240,12 @@ public void testFunctionsCreation() throws Exception { functionConfig, jarFilePathUrl ); + // Function creation is not strongly consistent, so this test can fail with a get that is too eager and + // does not have retries. + final PulsarAdmin admin = pulsarAdmins[i]; + Awaitility.await().ignoreExceptions() + .untilAsserted(() -> admin.functions().getFunction(testTenant, "my-ns", functionName)); + FunctionConfig config = pulsarAdmins[i].functions().getFunction(testTenant, "my-ns", functionName); assertEquals(config.getTenant(), testTenant); assertEquals(config.getNamespace(), "my-ns"); From 94c7bf3e57f6fc4323b36142fba651c4ad21d301 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 12 May 2023 12:27:49 +0800 Subject: [PATCH 189/280] [fix][build] Fix publish image script (#20305) Signed-off-by: Zixuan Liu --- docker/publish.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/publish.sh b/docker/publish.sh index af0d72d4b3437..45b338d85f8ef 100755 --- a/docker/publish.sh +++ b/docker/publish.sh @@ -62,11 +62,11 @@ set -x # Fail if any of the subsequent commands fail set -e -docker tag pulsar:latest ${docker_registry_org}/pulsar:latest -docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:latest +docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:latest +docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:latest -docker tag pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION -docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION +docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION +docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION # Push all images and tags docker push ${docker_registry_org}/pulsar:latest From 5d5ec947249df50bf35a78b6a2a0d3b00d97ca66 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 12 May 2023 19:29:42 +0800 Subject: [PATCH 190/280] [fix][broker] Allow Access to System Topic Metadata for Reader Creation Post-Namespace Deletion (#20304) ## Motivation After initiating the snapshot segment function, deletion of topics necessitates the activation of readers. Furthermore, these readers should be opened and deleted as they are used, which implies that we should not pre-store readers. However, after initiating the deletion of namespaces currently, it is not allowed to obtain the metadata of partition topics or lookup, making it impossible to create readers. This results in the inability to delete namespaces. ## Modification Allow the acquisition of system topic metadata after initiating namespace deletion, thus creating readers to clean up topic data. --- .../admin/impl/PersistentTopicsBase.java | 14 +++++- .../pulsar/broker/lookup/TopicLookupBase.java | 44 +++++++++++-------- .../broker/transaction/TransactionTest.java | 21 +++++++++ 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 9dcbe13d6159b..ed7cd70c6411c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static org.apache.pulsar.common.naming.SystemTopicNames.isSystemTopic; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX; @@ -4408,8 +4409,13 @@ public CompletableFuture getPartitionedTopicMetadata( // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can // serve/redirect request else fail partitioned-metadata-request so, client fails while creating // producer/consumer + // It is necessary for system topic operations because system topics are used to store metadata + // and other vital information. Even after namespace starting deletion,, + // we need to access the metadata of system topics to create readers and clean up topic data. + // If we don't do this, it can prevent namespace deletion due to inaccessible readers. authorizationFuture.thenCompose(__ -> - checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject())) + checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject(), + SystemTopicNames.isSystemTopic(topicName))) .thenCompose(res -> pulsar.getBrokerService().fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName)) .thenAccept(metadata -> { @@ -4436,7 +4442,11 @@ public static CompletableFuture unsafeGetPartitionedTo // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can // serve/redirect request else fail partitioned-metadata-request so, client fails while creating // producer/consumer - checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject()) + // It is necessary for system topic operations because system topics are used to store metadata + // and other vital information. Even after namespace starting deletion,, + // we need to access the metadata of system topics to create readers and clean up topic data. + // If we don't do this, it can prevent namespace deletion due to inaccessible readers. + checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject(), isSystemTopic(topicName)) .thenCompose(res -> pulsar.getBrokerService() .fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName)) .thenAccept(metadata -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index 3b64d2a9f8393..bd70201cba55d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -41,6 +41,7 @@ import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.lookup.data.LookupData; import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.NamespaceOperation; @@ -221,26 +222,31 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe // (2) authorize client checkAuthorizationAsync(pulsarService, topicName, clientAppId, authenticationData).thenRun(() -> { // (3) validate global namespace + // It is necessary for system topic operations because system topics are used to store metadata + // and other vital information. Even after namespace starting deletion, + // we need to access the metadata of system topics to create readers and clean up topic data. + // If we don't do this, it can prevent namespace deletion due to inaccessible readers. checkLocalOrGetPeerReplicationCluster(pulsarService, - topicName.getNamespaceObject()).thenAccept(peerClusterData -> { - if (peerClusterData == null) { - // (4) all validation passed: initiate lookup - validationFuture.complete(null); - return; - } - // if peer-cluster-data is present it means namespace is owned by that peer-cluster and - // request should be redirect to the peer-cluster - if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl()) - && StringUtils.isBlank(peerClusterData.getBrokerServiceUrlTls())) { - validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError, - "Redirected cluster's brokerService url is not configured", - requestId)); - return; - } - validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(), - peerClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, - requestId, - false)); + topicName.getNamespaceObject(), SystemTopicNames.isSystemTopic(topicName)) + .thenAccept(peerClusterData -> { + if (peerClusterData == null) { + // (4) all validation passed: initiate lookup + validationFuture.complete(null); + return; + } + // if peer-cluster-data is present it means namespace is owned by that peer-cluster + // and request should be redirect to the peer-cluster + if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl()) + && StringUtils.isBlank(peerClusterData.getBrokerServiceUrlTls())) { + validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError, + "Redirected cluster's brokerService url is not configured", + requestId)); + return; + } + validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(), + peerClusterData.getBrokerServiceUrlTls(), true, + LookupType.Redirect, requestId, + false)); }).exceptionally(ex -> { Throwable throwable = FutureUtil.unwrapCompletionException(ex); if (throwable instanceof RestException restException){ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index c3533e70cf8be..c4ec2ec766e32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -272,6 +272,27 @@ public void testCreateTransactionSystemTopic() throws Exception { } } + @Test + public void testCanDeleteNamespaceWhenEnableTxnSegmentedSnapshot() throws Exception { + // Enable the segmented snapshot feature + pulsarServiceList.get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + pulsarServiceList.get(0).getConfig().setForceDeleteNamespaceAllowed(true); + + // Create a new namespace + String namespaceName = TENANT + "/testSegmentedSnapshotWithoutCreatingOldSnapshotTopic"; + admin.namespaces().createNamespace(namespaceName); + + // Create a new topic in the namespace + String topicName = "persistent://" + namespaceName + "/newTopic"; + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + + // Destroy the namespace after the test + admin.namespaces().deleteNamespace(namespaceName, true); + pulsarServiceList.get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); + } + @Test public void brokerNotInitTxnManagedLedgerTopic() throws Exception { String subName = "test"; From 7e54e9b0463f5c69846dc44892d88281d5508466 Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Mon, 15 May 2023 16:29:50 +0900 Subject: [PATCH 191/280] [fix][broker] Fix class name typo `PrecisPublishLimiter` to "Precise" (#20310) --- .../pulsar/broker/service/AbstractTopic.java | 2 +- ...imiter.java => PrecisePublishLimiter.java} | 10 +++--- .../PrecisTopicPublishRateThrottleTest.java | 2 +- ...st.java => PrecisePublishLimiterTest.java} | 30 ++++++++-------- .../service/PublishRateLimiterTest.java | 34 ++++++++++--------- 5 files changed, 40 insertions(+), 38 deletions(-) rename pulsar-broker/src/main/java/org/apache/pulsar/broker/service/{PrecisPublishLimiter.java => PrecisePublishLimiter.java} (93%) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/service/{PrecisPublishLimiterTest.java => PrecisePublishLimiterTest.java} (57%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 8269dc0c3d16f..4614b846c8eee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -1274,7 +1274,7 @@ public void updatePublishDispatcher() { || this.topicPublishRateLimiter == PublishRateLimiter.DISABLED_RATE_LIMITER) { // create new rateLimiter if rate-limiter is disabled if (preciseTopicPublishRateLimitingEnable) { - this.topicPublishRateLimiter = new PrecisPublishLimiter(publishRate, + this.topicPublishRateLimiter = new PrecisePublishLimiter(publishRate, () -> this.enableCnxAutoRead(), brokerService.pulsar().getExecutor()); } else { this.topicPublishRateLimiter = new PublishRateLimiterImpl(publishRate); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java similarity index 93% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java index 6e215b3b49515..ce14f6d7dd71e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java @@ -24,7 +24,7 @@ import org.apache.pulsar.common.util.RateLimitFunction; import org.apache.pulsar.common.util.RateLimiter; -public class PrecisPublishLimiter implements PublishRateLimiter { +public class PrecisePublishLimiter implements PublishRateLimiter { protected volatile int publishMaxMessageRate = 0; protected volatile long publishMaxByteRate = 0; // precise mode for publish rate limiter @@ -33,18 +33,18 @@ public class PrecisPublishLimiter implements PublishRateLimiter { private final RateLimitFunction rateLimitFunction; private final ScheduledExecutorService scheduledExecutorService; - public PrecisPublishLimiter(Policies policies, String clusterName, RateLimitFunction rateLimitFunction) { + public PrecisePublishLimiter(Policies policies, String clusterName, RateLimitFunction rateLimitFunction) { this.rateLimitFunction = rateLimitFunction; update(policies, clusterName); this.scheduledExecutorService = null; } - public PrecisPublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction) { + public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction) { this(publishRate, rateLimitFunction, null); } - public PrecisPublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction, - ScheduledExecutorService scheduledExecutorService) { + public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction, + ScheduledExecutorService scheduledExecutorService) { this.rateLimitFunction = rateLimitFunction; update(publishRate); this.scheduledExecutorService = scheduledExecutorService; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java index 07632814378d0..c22ed41fc1533 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java @@ -164,7 +164,7 @@ public void testBrokerLevelPublishRateDynamicUpdate() throws Exception{ "" + rateInMsg)); Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); Assert.assertNotNull(topicRef); - PrecisPublishLimiter limiter = ((PrecisPublishLimiter) ((AbstractTopic) topicRef).topicPublishRateLimiter); + PrecisePublishLimiter limiter = ((PrecisePublishLimiter) ((AbstractTopic) topicRef).topicPublishRateLimiter); Awaitility.await().untilAsserted(() -> Assert.assertEquals(limiter.publishMaxMessageRate, rateInMsg)); Assert.assertEquals(limiter.publishMaxByteRate, 0); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java similarity index 57% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java index a0d0df52d2417..73cb43d52b112 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java @@ -23,35 +23,35 @@ import org.apache.pulsar.common.policies.data.PublishRate; import org.testng.annotations.Test; -public class PrecisPublishLimiterTest { +public class PrecisePublishLimiterTest { @Test void shouldResetMsgLimitAfterUpdate() { - PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(), () -> { + PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> { }); - precisPublishLimiter.update(new PublishRate(1, 1)); - assertFalse(precisPublishLimiter.tryAcquire(99, 99)); - precisPublishLimiter.update(new PublishRate(-1, 100)); - assertTrue(precisPublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(1, 1)); + assertFalse(precisePublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(-1, 100)); + assertTrue(precisePublishLimiter.tryAcquire(99, 99)); } @Test void shouldResetBytesLimitAfterUpdate() { - PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(), () -> { + PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> { }); - precisPublishLimiter.update(new PublishRate(1, 1)); - assertFalse(precisPublishLimiter.tryAcquire(99, 99)); - precisPublishLimiter.update(new PublishRate(100, -1)); - assertTrue(precisPublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(1, 1)); + assertFalse(precisePublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(100, -1)); + assertTrue(precisePublishLimiter.tryAcquire(99, 99)); } @Test void shouldCloseResources() throws Exception { for (int i = 0; i < 20000; i++) { - PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(100, 100), () -> { + PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(100, 100), () -> { }); - precisPublishLimiter.tryAcquire(99, 99); - precisPublishLimiter.close(); + precisePublishLimiter.tryAcquire(99, 99); + precisePublishLimiter.close(); } } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java index 9d2bd49ba04bd..b934ced08c5db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java @@ -39,7 +39,7 @@ public class PublishRateLimiterTest { private final PublishRate publishRate = new PublishRate(10, 100); private final PublishRate newPublishRate = new PublishRate(20, 200); - private PrecisPublishLimiter precisPublishLimiter; + private PrecisePublishLimiter precisePublishLimiter; private PublishRateLimiterImpl publishRateLimiter; @BeforeMethod @@ -47,7 +47,7 @@ public void setup() throws Exception { policies.publishMaxMessageRate = new HashMap<>(); policies.publishMaxMessageRate.put(CLUSTER_NAME, publishRate); - precisPublishLimiter = new PrecisPublishLimiter(policies, CLUSTER_NAME, () -> System.out.print("Refresh permit")); + precisePublishLimiter = new PrecisePublishLimiter(policies, CLUSTER_NAME, () -> System.out.print("Refresh permit")); publishRateLimiter = new PublishRateLimiterImpl(policies, CLUSTER_NAME); } @@ -88,23 +88,25 @@ public void testPublishRateLimiterImplUpdate() { @Test public void testPrecisePublishRateLimiterUpdate() { - assertFalse(precisPublishLimiter.tryAcquire(15, 150)); + assertFalse(precisePublishLimiter.tryAcquire(15, 150)); //update - precisPublishLimiter.update(newPublishRate); - assertTrue(precisPublishLimiter.tryAcquire(15, 150)); + precisePublishLimiter.update(newPublishRate); + assertTrue(precisePublishLimiter.tryAcquire(15, 150)); } @Test public void testPrecisePublishRateLimiterAcquire() throws Exception { - Class precisPublishLimiterClass = Class.forName("org.apache.pulsar.broker.service.PrecisPublishLimiter"); - Field topicPublishRateLimiterOnMessageField = precisPublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnMessage"); - Field topicPublishRateLimiterOnByteField = precisPublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnByte"); + Class precisePublishLimiterClass = Class.forName("org.apache.pulsar.broker.service.PrecisePublishLimiter"); + Field topicPublishRateLimiterOnMessageField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnMessage"); + Field topicPublishRateLimiterOnByteField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnByte"); topicPublishRateLimiterOnMessageField.setAccessible(true); topicPublishRateLimiterOnByteField.setAccessible(true); - RateLimiter topicPublishRateLimiterOnMessage = (RateLimiter)topicPublishRateLimiterOnMessageField.get(precisPublishLimiter); - RateLimiter topicPublishRateLimiterOnByte = (RateLimiter)topicPublishRateLimiterOnByteField.get(precisPublishLimiter); + RateLimiter topicPublishRateLimiterOnMessage = (RateLimiter)topicPublishRateLimiterOnMessageField.get( + precisePublishLimiter); + RateLimiter topicPublishRateLimiterOnByte = (RateLimiter)topicPublishRateLimiterOnByteField.get( + precisePublishLimiter); Method renewTopicPublishRateLimiterOnMessageMethod = topicPublishRateLimiterOnMessage.getClass().getDeclaredMethod("renew", null); Method renewTopicPublishRateLimiterOnByteMethod = topicPublishRateLimiterOnByte.getClass().getDeclaredMethod("renew", null); @@ -112,7 +114,7 @@ public void testPrecisePublishRateLimiterAcquire() throws Exception { renewTopicPublishRateLimiterOnByteMethod.setAccessible(true); // running tryAcquire in order to lazyInit the renewTask - precisPublishLimiter.tryAcquire(1, 10); + precisePublishLimiter.tryAcquire(1, 10); Field onMessageRenewTaskField = topicPublishRateLimiterOnMessage.getClass().getDeclaredField("renewTask"); Field onByteRenewTaskField = topicPublishRateLimiterOnByte.getClass().getDeclaredField("renewTask"); @@ -129,30 +131,30 @@ public void testPrecisePublishRateLimiterAcquire() throws Exception { renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire not exceeded - assertTrue(precisPublishLimiter.tryAcquire(1, 10)); + assertTrue(precisePublishLimiter.tryAcquire(1, 10)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire numOfMessages exceeded - assertFalse(precisPublishLimiter.tryAcquire(11, 100)); + assertFalse(precisePublishLimiter.tryAcquire(11, 100)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire msgSizeInBytes exceeded - assertFalse(precisPublishLimiter.tryAcquire(10, 101)); + assertFalse(precisePublishLimiter.tryAcquire(10, 101)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire exceeded exactly - assertFalse(precisPublishLimiter.tryAcquire(10, 100)); + assertFalse(precisePublishLimiter.tryAcquire(10, 100)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire not exceeded - assertTrue(precisPublishLimiter.tryAcquire(9, 99)); + assertTrue(precisePublishLimiter.tryAcquire(9, 99)); } } From c5ceec72d6d9362c680c08c3ecb5b1f3e9cc682a Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Mon, 15 May 2023 16:09:12 +0800 Subject: [PATCH 192/280] [fix][test] Flaky-test: SimpleProducerConsumerTestStreamingDispatcherTest. rest (#20286) --- .../org/apache/pulsar/broker/service/BrokerServiceTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 24e38438c5329..90b00bacaaa88 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -172,8 +172,6 @@ public void testShutDownWithMaxConcurrentUnload() throws Exception { assertTrue(e instanceof PulsarClientException.TimeoutException); } - pulsar = null; - producer.close(); resetState(); } From 21afdfd6df57ed56438a3434cd081d9b6dd4132d Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 15 May 2023 20:34:51 +0800 Subject: [PATCH 193/280] [improve][misc] Enable rebase and merge to handle one PR of multiple semantic commits (#20325) --- .asf.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 459dcb6e2fe57..5c3b8cb98d964 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -35,12 +35,9 @@ github: # Enable projects for project management boards projects: true enabled_merge_buttons: - # enable squash button: squash: true - # disable merge button: merge: false - # disable rebase button: - rebase: false + rebase: true protected_branches: master: required_status_checks: From 7b54664c364a7360f30cd37f0aa989ab13a14af4 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 15 May 2023 10:47:00 +0800 Subject: [PATCH 194/280] Revert "[fix][client] Seek should be thread-safe (#20242)" This reverts commit bc1764f9ef71dd31e8cd61c7571e493442bc6395. --- .../pulsar/client/impl/ConsumerImpl.java | 109 ++++++++---------- .../pulsar/client/impl/ConsumerImplTest.java | 27 +---- 2 files changed, 50 insertions(+), 86 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 2693f12d3ea8e..8a06ec122b5e6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -208,7 +208,6 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final AtomicReference clientCnxUsedForConsumerRegistration = new AtomicReference<>(); private final List previousExceptions = new CopyOnWriteArrayList(); - static ConsumerImpl newConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, @@ -252,12 +251,10 @@ static ConsumerImpl newConsumerImpl(PulsarClientImpl client, } protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, - ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, - boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, - MessageId startMessageId, - long startMessageRollbackDurationInSec, Schema schema, - ConsumerInterceptors interceptors, - boolean createTopicIfDoesNotExist) { + ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, + boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, MessageId startMessageId, + long startMessageRollbackDurationInSec, Schema schema, ConsumerInterceptors interceptors, + boolean createTopicIfDoesNotExist) { super(client, topic, conf, conf.getReceiverQueueSize(), executorProvider, subscribeFuture, schema, interceptors); this.consumerId = client.newConsumerId(); @@ -329,21 +326,21 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat } this.connectionHandler = new ConnectionHandler(this, - new BackoffBuilder() - .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), - TimeUnit.NANOSECONDS) - .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) - .setMandatoryStop(0, TimeUnit.MILLISECONDS) - .create(), + new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), + TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.MILLISECONDS) + .create(), this); this.topicName = TopicName.get(topic); if (this.topicName.isPersistent()) { this.acknowledgmentsGroupingTracker = - new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); + new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); } else { this.acknowledgmentsGroupingTracker = - NonPersistentAcknowledgmentGroupingTracker.of(); + NonPersistentAcknowledgmentGroupingTracker.of(); } if (conf.getDeadLetterPolicy() != null) { @@ -420,16 +417,16 @@ public CompletableFuture unsubscribeAsync() { log.error("[{}][{}] Failed to unsubscribe: {}", topic, subscription, e.getCause().getMessage()); setState(State.Ready); unsubscribeFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to unsubscribe the subscription %s of topic %s", - topicName.toString(), subscription))); + PulsarClientException.wrap(e.getCause(), + String.format("Failed to unsubscribe the subscription %s of topic %s", + topicName.toString(), subscription))); return null; }); } else { unsubscribeFuture.completeExceptionally( - new PulsarClientException( - String.format("The client is not connected to the broker when unsubscribing the " - + "subscription %s of the topic %s", subscription, topicName.toString()))); + new PulsarClientException( + String.format("The client is not connected to the broker when unsubscribing the " + + "subscription %s of the topic %s", subscription, topicName.toString()))); } return unsubscribeFuture; } @@ -1410,7 +1407,7 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien } private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, - MessageIdData messageId, ClientCnx cnx) { + MessageIdData messageId, ClientCnx cnx) { // Lazy task scheduling to expire incomplete chunk message if (expireTimeOfIncompleteChunkedMessageMillis > 0 && expireChunkMessageTaskScheduled.compareAndSet(false, @@ -1455,7 +1452,7 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m increaseAvailablePermits(cnx); if (expireTimeOfIncompleteChunkedMessageMillis > 0 && System.currentTimeMillis() > (msgMetadata.getPublishTime() - + expireTimeOfIncompleteChunkedMessageMillis)) { + + expireTimeOfIncompleteChunkedMessageMillis)) { doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null); } else { trackMessage(msgId); @@ -1646,7 +1643,7 @@ protected void trackMessage(Message msg) { } protected void trackMessage(MessageId messageId) { - trackMessage(messageId, 0); + trackMessage(messageId, 0); } protected void trackMessage(MessageId messageId, int redeliveryCount) { @@ -1787,7 +1784,7 @@ private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, int redeliveryCo } private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, - ClientCnx currentCnx, boolean checkMaxMessageSize) { + ClientCnx currentCnx, boolean checkMaxMessageSize) { CompressionType compressionType = msgMetadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = msgMetadata.getUncompressedSize(); @@ -1840,7 +1837,7 @@ private void discardCorruptedMessage(MessageIdImpl messageId, ClientCnx currentC } private void discardCorruptedMessage(MessageIdData messageId, ClientCnx currentCnx, - ValidationError validationError) { + ValidationError validationError) { log.error("[{}][{}] Discarding corrupted message at {}:{}", topic, subscription, messageId.getLedgerId(), messageId.getEntryId()); discardMessage(messageId, currentCnx, validationError); @@ -2032,8 +2029,8 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) String originTopicNameStr = getOriginTopicNameStr(message); TypedMessageBuilder typedMessageBuilderNew = producerDLQ.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) - .value(message.getData()) - .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); + .value(message.getData()) + .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); if (message.hasKey()) { typedMessageBuilderNew.key(message.getKey()); } @@ -2062,7 +2059,7 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) } result.complete(false); return null; - }); + }); } }, internalPinnedExecutor).exceptionally(ex -> { log.error("Dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), ex); @@ -2161,15 +2158,9 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, final CompletableFuture seekFuture = new CompletableFuture<>(); ClientCnx cnx = cnx(); - if (!duringSeek.compareAndSet(false, true)) { - log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}", - topic, subscription, seekBy); - seekFuture.cancel(true); - return seekFuture; - } - MessageIdAdv originSeekMessageId = seekMessageId; seekMessageId = (MessageIdAdv) seekId; + duringSeek.set(true); log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); cnx.sendRequestWithId(seek, requestId).thenRun(() -> { @@ -2187,9 +2178,9 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, log.error("[{}][{}] Failed to reset subscription: {}", topic, subscription, e.getCause().getMessage()); seekFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to seek the subscription %s of the topic %s to %s", - subscription, topicName.toString(), seekBy))); + PulsarClientException.wrap(e.getCause(), + String.format("Failed to seek the subscription %s of the topic %s to %s", + subscription, topicName.toString(), seekBy))); return null; }); return seekFuture; @@ -2201,7 +2192,7 @@ public CompletableFuture seekAsync(long timestamp) { return seekAsyncCheckState(seekBy).orElseGet(() -> { long requestId = client.newRequestId(); return seekAsyncInternal(requestId, Commands.newSeek(consumerId, requestId, timestamp), - MessageId.earliest, seekBy); + MessageId.earliest, seekBy); }); } @@ -2367,11 +2358,10 @@ public CompletableFuture> getLastMessageIdsAsync() { public CompletableFuture internalGetLastMessageIdAsync() { if (getState() == State.Closing || getState() == State.Closed) { return FutureUtil - .failedFuture(new PulsarClientException.AlreadyClosedException( - String.format("The consumer %s was already closed when the subscription %s of the topic %s " - + "getting the last message id", consumerName, subscription, - topicName.toString()))); - } + .failedFuture(new PulsarClientException.AlreadyClosedException( + String.format("The consumer %s was already closed when the subscription %s of the topic %s " + + "getting the last message id", consumerName, subscription, topicName.toString()))); + } AtomicLong opTimeoutMs = new AtomicLong(client.getConfiguration().getOperationTimeoutMs()); Backoff backoff = new BackoffBuilder() @@ -2393,12 +2383,11 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, if (isConnected() && cnx != null) { if (!Commands.peerSupportsGetLastMessageId(cnx.getRemoteEndpointProtocolVersion())) { future.completeExceptionally( - new PulsarClientException.NotSupportedException( - String.format( - "The command `GetLastMessageId` is not supported for the protocol version %d. " - + "The consumer is %s, topic %s, subscription %s", - cnx.getRemoteEndpointProtocolVersion(), - consumerName, topicName.toString(), subscription))); + new PulsarClientException.NotSupportedException( + String.format("The command `GetLastMessageId` is not supported for the protocol version %d. " + + "The consumer is %s, topic %s, subscription %s", + cnx.getRemoteEndpointProtocolVersion(), + consumerName, topicName.toString(), subscription))); return; } @@ -2417,31 +2406,31 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, } if (log.isDebugEnabled()) { log.debug("[{}][{}] Successfully getLastMessageId {}:{}", - topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId()); + topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId()); } MessageId lastMsgId = lastMessageId.getBatchIndex() <= 0 ? new MessageIdImpl(lastMessageId.getLedgerId(), - lastMessageId.getEntryId(), lastMessageId.getPartition()) + lastMessageId.getEntryId(), lastMessageId.getPartition()) : new BatchMessageIdImpl(lastMessageId.getLedgerId(), lastMessageId.getEntryId(), - lastMessageId.getPartition(), lastMessageId.getBatchIndex()); + lastMessageId.getPartition(), lastMessageId.getBatchIndex()); future.complete(new GetLastMessageIdResponse(lastMsgId, markDeletePosition)); }).exceptionally(e -> { log.error("[{}][{}] Failed getLastMessageId command", topic, subscription); future.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("The subscription %s of the topic %s gets the last message id was failed", - subscription, topicName.toString()))); + PulsarClientException.wrap(e.getCause(), + String.format("The subscription %s of the topic %s gets the last message id was failed", + subscription, topicName.toString()))); return null; }); } else { long nextDelay = Math.min(backoff.next(), remainingTime.get()); if (nextDelay <= 0) { future.completeExceptionally( - new PulsarClientException.TimeoutException( - String.format("The subscription %s of the topic %s could not get the last message id " - + "withing configured timeout", subscription, topicName.toString()))); + new PulsarClientException.TimeoutException( + String.format("The subscription %s of the topic %s could not get the last message id " + + "withing configured timeout", subscription, topicName.toString()))); return; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index ab11675f5434c..29d180f5f9a16 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.client.impl; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -27,9 +26,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertTrue; -import io.netty.buffer.ByteBuf; + import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -262,26 +259,4 @@ public void testTopicPriorityLevel() { assertThat(consumer.getPriorityLevel()).isEqualTo(1); } - - @Test(invocationTimeOut = 1000) - public void testSeekAsyncInternal() { - // given - ClientCnx cnx = mock(ClientCnx.class); - CompletableFuture clientReq = new CompletableFuture<>(); - when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq); - - consumer.setClientCnx(cnx); - consumer.setState(HandlerState.State.Ready); - - // when - CompletableFuture firstResult = consumer.seekAsync(1L); - CompletableFuture secondResult = consumer.seekAsync(1L); - - clientReq.complete(null); - - // then - assertTrue(firstResult.isDone()); - assertTrue(secondResult.isCancelled()); - verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong()); - } } From 0acf8f89d6057170e990b128936175fc5dd33be3 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 15 May 2023 15:10:11 +0800 Subject: [PATCH 195/280] [fix][client] thread-safe seek Signed-off-by: tison --- .../pulsar/client/impl/ConsumerImpl.java | 11 +++++++- .../pulsar/client/impl/ConsumerImplTest.java | 27 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 8a06ec122b5e6..4a84e765065f2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2158,9 +2158,18 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, final CompletableFuture seekFuture = new CompletableFuture<>(); ClientCnx cnx = cnx(); + if (!duringSeek.compareAndSet(false, true)) { + final String message = String.format( + "[%s][%s] attempting to seek operation that is already in progress (seek by %s)", + topic, subscription, seekBy); + log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}", + topic, subscription, seekBy); + seekFuture.completeExceptionally(new IllegalStateException(message)); + return seekFuture; + } + MessageIdAdv originSeekMessageId = seekMessageId; seekMessageId = (MessageIdAdv) seekId; - duringSeek.set(true); log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); cnx.sendRequestWithId(seek, requestId).thenRun(() -> { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index 29d180f5f9a16..5a223d5da15c0 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -26,7 +27,9 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; - +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -259,4 +262,26 @@ public void testTopicPriorityLevel() { assertThat(consumer.getPriorityLevel()).isEqualTo(1); } + + @Test(invocationTimeOut = 1000) + public void testSeekAsyncInternal() { + // given + ClientCnx cnx = mock(ClientCnx.class); + CompletableFuture clientReq = new CompletableFuture<>(); + when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq); + + consumer.setClientCnx(cnx); + consumer.setState(HandlerState.State.Ready); + + // when + CompletableFuture firstResult = consumer.seekAsync(1L); + CompletableFuture secondResult = consumer.seekAsync(1L); + + clientReq.complete(null); + + // then + assertTrue(firstResult.isDone()); + assertTrue(secondResult.isCompletedExceptionally()); + verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong()); + } } From 8b929e6bf1b1718431abebf04420d7817ad8cdb7 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 16 May 2023 09:41:35 +0800 Subject: [PATCH 196/280] [fix][txn] Implement compatibility for transaction buffer segmented snapshot feature upgrade (#20235) master https://github.com/apache/pulsar/issues/16913 ## Motivation: The transaction buffer segmented snapshot feature aims to improve the transaction buffer's performance by segmenting the snapshot and managing it more efficiently. However, for existing topics that were created before this feature was introduced, we need to ensure a seamless transition and compatibility when enabling the segmented snapshot feature. ## Modifications: 1. Updated the `recoverFromSnapshot()` method to read from another topic if the `persistentSnapshotIndexes` is null. This ensures that the appropriate snapshot data is fetched during the recovery process when upgrading to the segmented snapshot feature. 2. Created a new test `testSnapshotProcessorUpdate()` that verifies the compatibility of the transaction ### Verifying this change - [ ] Make sure that the change passes the CI checks. *(Please pick either of the following options)* This change is a trivial rework / code cleanup without any test coverage. *(or)* This change is already covered by existing tests, such as *(please describe tests)*. *(or)* This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* --- ...napshotSegmentAbortedTxnProcessorImpl.java | 66 ++++++++- .../SegmentAbortedTxnProcessorTest.java | 133 +++++++++++++++++- 2 files changed, 196 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 751c03aff95a9..4f4e58ac3f55b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -47,6 +47,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata; @@ -265,7 +266,7 @@ public CompletableFuture recoverFromSnapshot() { PositionImpl finalStartReadCursorPosition = startReadCursorPosition; TransactionBufferSnapshotIndexes finalPersistentSnapshotIndexes = persistentSnapshotIndexes; if (persistentSnapshotIndexes == null) { - return CompletableFuture.completedFuture(null); + return recoverOldSnapshot(); } else { this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes .getSnapshot().getAborts()); @@ -378,6 +379,69 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob .getExecutor(this)); } + // This method will be deprecated and removed in version 4.x.0 + private CompletableFuture recoverOldSnapshot() { + return topic.getBrokerService().getTopic(TopicName.get(topic.getName()).getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, false) + .thenCompose(topicOption -> { + if (!topicOption.isPresent()) { + return CompletableFuture.completedFuture(null); + } else { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(snapshotReader -> { + PositionImpl startReadCursorPositionInOldSnapshot = null; + try { + while (snapshotReader.hasMoreEvents()) { + Message message = snapshotReader.readNextAsync() + .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshot transactionBufferSnapshot = + message.getValue(); + if (transactionBufferSnapshot != null) { + handleOldSnapshot(transactionBufferSnapshot); + startReadCursorPositionInOldSnapshot = PositionImpl.get( + transactionBufferSnapshot.getMaxReadPositionLedgerId(), + transactionBufferSnapshot.getMaxReadPositionEntryId()); + } + } + } + } catch (TimeoutException ex) { + Throwable t = FutureUtil.unwrapCompletionException(ex); + String errorMessage = String.format("[%s] Transaction buffer recover fail by " + + "read transactionBufferSnapshot timeout!", topic.getName()); + log.error(errorMessage, t); + return FutureUtil.failedFuture(new BrokerServiceException + .ServiceUnitNotReadyException(errorMessage, t)); + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); + return FutureUtil.failedFuture(ex); + } finally { + assert snapshotReader != null; + closeReader(snapshotReader); + } + return CompletableFuture.completedFuture(startReadCursorPositionInOldSnapshot); + }, + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)); + } + }); + } + + // This method will be deprecated and removed in version 4.x.0 + private void handleOldSnapshot(TransactionBufferSnapshot snapshot) { + if (snapshot.getAborts() != null) { + snapshot.getAborts().forEach(abortTxnMetadata -> { + TxnID txnID = new TxnID(abortTxnMetadata.getTxnIdMostBits(), + abortTxnMetadata.getTxnIdLeastBits()); + aborts.put(txnID, txnID); + //The old data will be written into the first segment. + unsealedTxnIds.add(txnID); + }); + } + } + @Override public CompletableFuture clearAbortedTxnSnapshot() { return persistentWorker.appendTask(PersistentWorker.OperationType.Clear, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index c157d7cf8c527..cb15ab003f7b7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.transaction; +import static org.junit.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -44,10 +45,16 @@ import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicStats; import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -247,8 +254,8 @@ public void testClearSnapshotSegments() throws Exception { private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { SystemTopicClient.Reader reader = pulsarService.getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService() - .createReader(TopicName.get(topic)).get(); + .getTxnBufferSnapshotSegmentService() + .createReader(TopicName.get(topic)).get(); int segmentCount = 0; while (reader.hasMoreEvents()) { Message message = reader.readNextAsync() @@ -286,4 +293,126 @@ private void doCompaction(TopicName topic) throws Exception { CompletableFuture compactionFuture = (CompletableFuture) field.get(snapshotTopic); org.awaitility.Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone())); } + + /** + * This test verifies the compatibility of the transaction buffer segmented snapshot feature + * when enabled on an existing topic. + * It performs the following steps: + * 1. Creates a topic with segmented snapshot disabled. + * 2. Sends 10 messages without using transactions. + * 3. Sends 10 messages using transactions and aborts them. + * 4. Sends 10 messages without using transactions. + * 5. Verifies that only the non-transactional messages are received. + * 6. Enables the segmented snapshot feature and sets the snapshot segment size. + * 7. Unloads the topic. + * 8. Sends a new message using a transaction and aborts it. + * 9. Verifies that the topic has exactly one segment. + * 10. Re-subscribes the consumer and re-verifies that only the non-transactional messages are received. + */ + @Test + public void testSnapshotProcessorUpgrade() throws Exception { + this.pulsarService = getPulsarServiceList().get(0); + this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); + + // Create a topic, send 10 messages without using transactions, and send 10 messages using transactions. + // Abort these transactions and verify the data. + final String topicName = "persistent://" + NAMESPACE1 + "/testSnapshotProcessorUpgrade"; + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe(); + + // Send 10 messages without using transactions + for (int i = 0; i < 10; i++) { + producer.send(("test-message-" + i).getBytes()); + } + + // Send 10 messages using transactions and abort them + for (int i = 0; i < 10; i++) { + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + producer.newMessage(txn).value(("test-txn-message-" + i).getBytes()).sendAsync(); + txn.abort().get(); + } + + // Send 10 messages without using transactions + for (int i = 10; i < 20; i++) { + producer.send(("test-message-" + i).getBytes()); + } + + // Verify the data + for (int i = 0; i < 20; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals("test-message-" + i, new String(msg.getData())); + } + + // Enable segmented snapshot + this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + + SEGMENT_SIZE * 3); + + // Unload the topic + admin.topics().unload(topicName); + + // Sends a new message using a transaction and aborts it. + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + producer.newMessage(txn).value("test-message-new".getBytes()).send(); + txn.abort().get(); + + // Verifies that the topic has exactly one segment. + Awaitility.await().untilAsserted(() -> { + String segmentTopic = "persistent://" + NAMESPACE1 + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS; + TopicStats topicStats = admin.topics().getStats(segmentTopic); + assertEquals(1, topicStats.getMsgInCounter()); + }); + + // Re-subscribes the consumer and re-verifies that only the non-transactional messages are received. + consumer.close(); + consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe(); + for (int i = 0; i < 20; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals("test-message-" + i, new String(msg.getData())); + } + } + + /** + * This test verifies that when the segmented snapshot feature is enabled, creating a new topic + * does not create a __transaction_buffer_snapshot topic in the same namespace. + * The test performs the following steps: + * 1. Enable the segmented snapshot feature. + * 2. Create a new namespace. + * 3. Create a new topic in the namespace. + * 4. Check that the __transaction_buffer_snapshot topic is not created in the same namespace. + * 5. Destroy the namespace after the test. + */ + @Test + public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Exception { + // Enable the segmented snapshot feature + pulsarService = getPulsarServiceList().get(0); + pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + + // Create a new namespace + String namespaceName = "tnx/testSegmentedSnapshotWithoutCreatingOldSnapshotTopic"; + admin.namespaces().createNamespace(namespaceName); + + // Create a new topic in the namespace + String topicName = "persistent://" + namespaceName + "/newTopic"; + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + + // Check that the __transaction_buffer_snapshot topic is not created in the same namespace + String transactionBufferSnapshotTopic = "persistent://" + namespaceName + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT; + try { + admin.topics().getStats(transactionBufferSnapshotTopic); + fail("The __transaction_buffer_snapshot topic should not exist"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + // Destroy the namespace after the test + admin.namespaces().deleteNamespace(namespaceName, true); + } } From 426ad3e53ba621c442bce89eb208add8f9d1563e Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Tue, 16 May 2023 17:19:46 +0800 Subject: [PATCH 197/280] [fix][ml] Fix ledger left in OPEN state when enable `inactiveLedgerRollOverTimeMs` (#20276) close `currentLegder` after roll current ledger if full --- .../mledger/impl/ManagedLedgerImpl.java | 31 ++++++++++++++++--- .../mledger/impl/ManagedLedgerTest.java | 15 +++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index c74db884a95a9..15e9d332fa103 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1775,15 +1775,19 @@ public void closeComplete(int rc, LedgerHandle lh, Object o) { + "acked ledgerId %s", currentLedger.getId(), lh.getId()); if (rc == BKException.Code.OK) { - log.debug("Successfully closed ledger {}", lh.getId()); + if (log.isDebugEnabled()) { + log.debug("[{}] Successfully closed ledger {}, trigger by rollover full ledger", + name, lh.getId()); + } } else { - log.warn("Error when closing ledger {}. Status={}", lh.getId(), BKException.getMessage(rc)); + log.warn("[{}] Error when closing ledger {}, trigger by rollover full ledger, Status={}", + name, lh.getId(), BKException.getMessage(rc)); } ledgerClosed(lh); createLedgerAfterClosed(); } - }, System.nanoTime()); + }, null); } } @@ -4353,7 +4357,26 @@ public void checkInactiveLedgerAndRollOver() { long currentTimeMs = System.currentTimeMillis(); if (inactiveLedgerRollOverTimeMs > 0 && currentTimeMs > (lastAddEntryTimeMs + inactiveLedgerRollOverTimeMs)) { log.info("[{}] Closing inactive ledger, last-add entry {}", name, lastAddEntryTimeMs); - ledgerClosed(currentLedger); + if (STATE_UPDATER.compareAndSet(this, State.LedgerOpened, State.ClosingLedger)) { + LedgerHandle currentLedger = this.currentLedger; + currentLedger.asyncClose((rc, lh, o) -> { + checkArgument(currentLedger.getId() == lh.getId(), "ledgerId %s doesn't match with " + + "acked ledgerId %s", currentLedger.getId(), lh.getId()); + + if (rc == BKException.Code.OK) { + if (log.isDebugEnabled()) { + log.debug("[{}] Successfully closed ledger {}, trigger by inactive ledger check", + name, lh.getId()); + } + } else { + log.warn("[{}] Error when closing ledger {}, trigger by inactive ledger check, Status={}", + name, lh.getId(), BKException.getMessage(rc)); + } + + ledgerClosed(lh); + // we do not create ledger here, since topic is inactive for a long time. + }, null); + } } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index dd30cde72e769..70ddbb9998fd8 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -88,6 +88,7 @@ import org.apache.bookkeeper.client.PulsarMockBookKeeper; import org.apache.bookkeeper.client.PulsarMockLedgerHandle; import org.apache.bookkeeper.client.api.LedgerEntries; +import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -3859,12 +3860,26 @@ public void testInactiveLedgerRollOver() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("rollover_inactive", config); ManagedCursor cursor = ledger.openCursor("c1"); + List ledgerIds = new ArrayList<>(); + int totalAddEntries = 5; for (int i = 0; i < totalAddEntries; i++) { String content = "entry"; // 5 bytes ledger.checkInactiveLedgerAndRollOver(); ledger.addEntry(content.getBytes()); Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + + ledgerIds.add(ledger.currentLedger.getId()); + } + + Map ledgerMap = bkc.getLedgerMap(); + // skip check last ledger, it should be open + for (int i = 0; i < ledgerIds.size() - 1; i++) { + long ledgerId = ledgerIds.get(i); + LedgerMetadata ledgerMetadata = ledgerMap.get(ledgerId).getLedgerMetadata(); + if (ledgerMetadata != null) { + assertTrue(ledgerMetadata.isClosed()); + } } List ledgers = ledger.getLedgersInfoAsList(); From ff58c6203ba6367f8ba293bc1a4e994725b5758a Mon Sep 17 00:00:00 2001 From: zhangwd3 Date: Tue, 16 May 2023 11:23:11 +0800 Subject: [PATCH 198/280] correcting spelling mistakes Signed-off-by: zhangwd3 --- .../org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java | 2 +- .../broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java | 2 +- .../apache/pulsar/client/api/SimpleProducerConsumerTest.java | 2 +- .../apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java | 2 +- .../src/main/java/org/apache/pulsar/client/api/Reader.java | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 1405979ae3b12..17aa7170fc63c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -161,7 +161,7 @@ public static long getCpuUsageForCGroup() { * *

* Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum substracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. */ public static ResourceUsage getCpuUsageForEntireHost() { try (Stream stream = Files.lines(Paths.get(PROC_STAT_PATH))) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 5d2b7bdd09adf..2f7ca614943b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -155,7 +155,7 @@ private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) { * * * Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum substracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. */ private double getTotalCpuUsageForEntireHost() { LinuxInfoUtils.ResourceUsage cpuUsageForEntireHost = getCpuUsageForEntireHost(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 1bc437195d96d..f3a00531eba56 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -3136,7 +3136,7 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe pulsarClient.newProducer().topic("persistent://my-property/my-ns/myenc-topic1") .addEncryptionKey("client-non-existant-rsa.pem").cryptoKeyReader(new EncKeyReader()) .create(); - Assert.fail("Producer creation should not suceed if failing to read key"); + Assert.fail("Producer creation should not succeed if failing to read key"); } catch (Exception e) { // ok } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java index 9f94d894632b8..e126d963a88f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java @@ -2296,7 +2296,7 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe .addEncryptionKey("client-non-existant-rsa.pem") .cryptoKeyReader(new EncKeyReader()) .create(); - Assert.fail("Producer creation should not suceed if failing to read key"); + Assert.fail("Producer creation should not succeed if failing to read key"); } catch (Exception e) { // ok } diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java index 451b9fa638203..98fcdb453bb76 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java @@ -36,14 +36,14 @@ public interface Reader extends Closeable { /** * Read the next message in the topic. * - * @return the next messasge + * @return the next message * @throws PulsarClientException */ Message readNext() throws PulsarClientException; /** * Read the next message in the topic waiting for a maximum of timeout - * time units. Returns null if no message is recieved in that time. + * time units. Returns null if no message is received in that time. * * @return the next message(Could be null if none received in time) * @throws PulsarClientException From 9d7609f0de1468b68209875fe44527514c1f92de Mon Sep 17 00:00:00 2001 From: Kengo Seki Date: Tue, 16 May 2023 23:35:04 +0900 Subject: [PATCH 199/280] [improve][io] Upgrade Hadoop and HBase versions (#20008) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 55f0224991759..83aba7682eee0 100644 --- a/pom.xml +++ b/pom.xml @@ -200,8 +200,8 @@ flexible messaging model and an intuitive client API. 0.11.1 0.28.0 2.10.2 - 3.3.4 - 2.4.15 + 3.3.5 + 2.4.16 31.0.1-jre 1.0 0.16.1 From 1a7a876634f32d5d9218977dc83f8ae08a08bf57 Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Wed, 17 May 2023 16:05:01 +0900 Subject: [PATCH 200/280] [improve][client] Document Java Client's Seek logic thread-safety improved in #20242 (#20284) --- .../org/apache/pulsar/client/api/Consumer.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java index 88ad24fe1f484..c67ad08c83631 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java @@ -465,6 +465,9 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message, /** * Reset the subscription associated with this consumer to a specific message id. + *

+ * If there is already a seek operation in progress, the method will log a warning and + * return a future completed exceptionally. * *

The message id can either be a specific message or represent the first or last messages in the topic. *

    @@ -483,6 +486,9 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message, /** * Reset the subscription associated with this consumer to a specific message publish time. + *

    + * If there is already a seek operation in progress, the method will log a warning and + * return a future completed exceptionally. * * @param timestamp * the message publish time where to reposition the subscription @@ -493,6 +499,10 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message, /** * Reset the subscription associated with this consumer to a specific message ID or message publish time. *

    + * If there is already a seek operation in progress, the method will log a warning and + * return a future completed exceptionally. + * + *

    * The Function input is topic+partition. It returns only timestamp or MessageId. *

    * The return value is the seek position/timestamp of the current partition. @@ -523,11 +533,17 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message, /** * The asynchronous version of {@link Consumer#seek(MessageId)}. + *

    + * If there is already a seek operation in progress, the method will log a warning and + * return a future completed exceptionally. */ CompletableFuture seekAsync(MessageId messageId); /** * Reset the subscription associated with this consumer to a specific message publish time. + *

    + * If there is already a seek operation in progress, the method will log a warning and + * return a future completed exceptionally. * * @param timestamp * the message publish time where to reposition the subscription From c7a4060763b905925ffe8e9690ea983d863aa52f Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 04:14:44 -0500 Subject: [PATCH 201/280] [feat][meta] Upgrade to jetcd to 0.7.5 (#20339) --- .../server/src/assemble/LICENSE.bin.txt | 8 ++-- pom.xml | 2 +- .../metadata/BaseMetadataStoreTest.java | 7 +-- .../impl/LeaderElectionImplTest.java | 2 +- .../metadata/impl/EtcdMetadataStoreTest.java | 17 ++++--- pulsar-sql/presto-distribution/LICENSE | 15 +++---- src/owasp-dependency-check-suppressions.xml | 45 ------------------- 7 files changed, 28 insertions(+), 68 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 993e0997e53ee..220f0ac0758b8 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -433,7 +433,6 @@ The Apache Software License, Version 2.0 - io.grpc-grpc-services-1.45.1.jar - io.grpc-grpc-xds-1.45.1.jar - io.grpc-grpc-rls-1.45.1.jar - - com.google.auto.service-auto-service-annotations-1.0.jar * Perfmark - io.perfmark-perfmark-api-0.19.0.jar * OpenCensus @@ -475,6 +474,7 @@ The Apache Software License, Version 2.0 - io.vertx-vertx-core-4.3.8.jar - io.vertx-vertx-web-4.3.8.jar - io.vertx-vertx-web-common-4.3.8.jar + - io.vertx-vertx-grpc-4.3.5.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-3.8.1.jar - org.apache.zookeeper-zookeeper-jute-3.8.1.jar @@ -487,8 +487,10 @@ The Apache Software License, Version 2.0 - com.google.auto.value-auto-value-annotations-1.9.jar - com.google.re2j-re2j-1.5.jar * Jetcd - - io.etcd-jetcd-common-0.5.11.jar - - io.etcd-jetcd-core-0.5.11.jar + - io.etcd-jetcd-api-0.7.5.jar + - io.etcd-jetcd-common-0.7.5.jar + - io.etcd-jetcd-core-0.7.5.jar + - io.etcd-jetcd-grpc-0.7.5.jar * IPAddress - com.github.seancfoley-ipaddress-5.3.3.jar * RxJava diff --git a/pom.xml b/pom.xml index 83aba7682eee0..75c25e5478893 100644 --- a/pom.xml +++ b/pom.xml @@ -242,7 +242,7 @@ flexible messaging model and an intuitive client API. 5.3.27 4.5.13 4.4.15 - 0.5.11 + 0.7.5 2.0 1.10.12 5.3.3 diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java index 57a4e388572fa..ec6e6e03eae71 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java @@ -21,7 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import io.etcd.jetcd.launcher.EtcdCluster; -import io.etcd.jetcd.launcher.EtcdClusterFactory; +import io.etcd.jetcd.test.EtcdClusterExtension; import java.io.File; import java.net.URI; import java.util.UUID; @@ -84,10 +84,11 @@ public Object[][] implementations() { private synchronized String getEtcdClusterConnectString() { if (etcdCluster == null) { - etcdCluster = EtcdClusterFactory.buildCluster("test", 1, false); + etcdCluster = EtcdClusterExtension.builder().withClusterName("test").withNodes(1).withSsl(false).build() + .cluster(); etcdCluster.start(); } - return etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + return etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); } public static Supplier stringSupplier(Supplier supplier) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java index 027521d2ffc17..09c9d71c41a30 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java @@ -29,7 +29,7 @@ public class LeaderElectionImplTest extends BaseMetadataStoreTest { - @Test(dataProvider = "impl", timeOut = 10000) + @Test(dataProvider = "impl", timeOut = 20000) public void validateDeadLock(String provider, Supplier urlSupplier) throws Exception { if (provider.equals("Memory") || provider.equals("RocksDB")) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java index f1a1b5626acce..bdcd0614d0375 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.io.Resources; import io.etcd.jetcd.launcher.EtcdCluster; -import io.etcd.jetcd.launcher.EtcdClusterFactory; +import io.etcd.jetcd.test.EtcdClusterExtension; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -44,7 +44,8 @@ public class EtcdMetadataStoreTest { @Test public void testCluster() throws Exception { @Cleanup - EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-cluster", 3, false); + EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-cluster").withNodes(3) + .withSsl(false).build().cluster(); etcdCluster.start(); EtcdConfig etcdConfig = EtcdConfig.builder().useTls(false) @@ -56,7 +57,7 @@ public void testCluster() throws Exception { new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig); String metadataURL = - "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); @Cleanup MetadataStore store = MetadataStoreFactory.create(metadataURL, @@ -71,7 +72,8 @@ public void testCluster() throws Exception { @Test public void testClusterWithTls() throws Exception { @Cleanup - EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-cluster", 3, true); + EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-cluster").withNodes(3) + .withSsl(true).build().cluster(); etcdCluster.start(); EtcdConfig etcdConfig = EtcdConfig.builder().useTls(true) @@ -86,7 +88,7 @@ public void testClusterWithTls() throws Exception { new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig); String metadataURL = - "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); @Cleanup MetadataStore store = MetadataStoreFactory.create(metadataURL, @@ -101,7 +103,8 @@ public void testClusterWithTls() throws Exception { @Test public void testTlsInstance() throws Exception { @Cleanup - EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-tls", 1, true); + EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-tls").withNodes(1) + .withSsl(true).build().cluster(); etcdCluster.start(); EtcdConfig etcdConfig = EtcdConfig.builder().useTls(true) @@ -115,7 +118,7 @@ public void testTlsInstance() throws Exception { new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig); String metadataURL = - "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); @Cleanup MetadataStore store = MetadataStoreFactory.create(metadataURL, diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 09d1396b70419..a85d1bc363c1b 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -272,9 +272,13 @@ The Apache Software License, Version 2.0 - grpc-protobuf-lite-1.45.1.jar - grpc-stub-1.45.1.jar * JEtcd - - jetcd-common-0.5.11.jar - - jetcd-core-0.5.11.jar - + - jetcd-api-0.7.5.jar + - jetcd-common-0.7.5.jar + - jetcd-core-0.7.5.jar + - jetcd-grpc-0.7.5.jar + * Vertx + - vertx-core-4.3.8.jar + - vertx-grpc-4.3.5.jar * Joda Time - joda-time-2.10.10.jar - failsafe-2.4.4.jar @@ -476,8 +480,6 @@ The Apache Software License, Version 2.0 - swagger-annotations-1.6.2.jar * Perfmark - perfmark-api-0.19.0.jar - * Annotations - - auto-service-annotations-1.0.jar * RabbitMQ Java Client - amqp-client-5.5.3.jar * Stream Lib @@ -520,9 +522,6 @@ MIT License - jcl-over-slf4j-1.7.32.jar * Checker Qual - checker-qual-3.12.0.jar - * Annotations - - animal-sniffer-annotations-1.19.jar - - annotations-4.1.1.4.jar * ScribeJava - scribejava-apis-6.9.0.jar - scribejava-core-6.9.0.jar diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index d599f2de3e85a..a59b3b999e2bf 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -229,51 +229,6 @@ CVE-2021-42550 - - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15106 - - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15112 - - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15113 - - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15106 - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15112 - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15113 - - Date: Wed, 17 May 2023 19:43:42 +0800 Subject: [PATCH 202/280] [fix][doc] Correcting spelling mistakes (#20340) Signed-off-by: zhangwd3 Co-authored-by: tison --- .../java/org/apache/pulsar/client/impl/schema/SchemaUtils.java | 2 +- .../main/java/org/apache/pulsar/common/api/raw/RawMessage.java | 2 +- .../pulsar/functions/instance/state/BKStateStoreImpl.java | 2 +- .../functions/auth/KubernetesSecretsTokenAuthProvider.java | 2 +- .../apache/pulsar/functions/worker/rest/api/ComponentImpl.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java index 526b0c96be05b..8acbf26559b7b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java @@ -210,7 +210,7 @@ public static String jsonifySchemaInfo(SchemaInfo schemaInfo) { } /** - * Jsonify the schema info with verison. + * Jsonify the schema info with version. * * @param schemaInfoWithVersion the schema info * @return the jsonified schema info with version diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java index 8e2f51c4edbf0..a02208396fc91 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java @@ -102,7 +102,7 @@ public interface RawMessage { Optional getKey(); /** - * Get the schema verison of the message. + * Get the schema version of the message. * * @return the schema version of the message */ diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java index df1eae9b6a7ab..bf43f18b175e7 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java @@ -166,7 +166,7 @@ public CompletableFuture getAsync(String key) { data.readBytes(result); // Set position to off the buffer to the beginning, since the position after the // read is going to be end of the buffer - // If we do not rewind to the begining here, users will have to explicitly do + // If we do not rewind to the beginning here, users will have to explicitly do // this in their function code // in order to use any of the ByteBuffer operations result.position(0); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java index e0543f28ac160..913171aca9bc8 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java @@ -161,7 +161,7 @@ public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional Date: Wed, 17 May 2023 10:29:45 -0500 Subject: [PATCH 203/280] [cleanup] Consolidate certs used in tests (#20336) Builds on: https://github.com/apache/pulsar/pull/20289 ### Motivation There are many certificates in our test code base. It would be much simpler to have one place were we create and manage certificates so that when we need to make changes, they are consolidated. There is likely one or two more PRs to finish consolidating certs. ### Modifications * Remove certs that are no longer used * Replace references to old certs with references to the `certificate-authority` certs * Create new server certs with valid hostnames on them so that tests will pass. Document the process used to create these certs. * Fix an issue in the `PulsarTestContext` where the configuration was not correctly updated. * Remove configurations that allow for insecure connections in tests that are doing some kind of TLS verification. The only places where we leave insecure validation in place is tests that are specifically verifying the functionality. * Copy `certificate-authority` to the relevant `bouncy-castle` directory ### Verifying this change When tests pass, this change will be correctly verified. ### Documentation - [x] `doc` This PR includes doc changes ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/48 --- bouncy-castle/bcfips-include-test/pom.xml | 22 +++ .../client/TlsProducerConsumerBase.java | 23 ++- .../authentication/tls/broker-cert.pem | 71 ---------- .../authentication/tls/broker-key.pem | 28 ---- .../resources/authentication/tls/cacert.pem | 78 ---------- .../authentication/tls/client-cert.pem | 71 ---------- .../authentication/tls/client-key.pem | 28 ---- build/regenerate_certs_for_tests.sh | 7 - .../admin/BrokerAdminClientTlsAuthTest.java | 2 +- .../broker/testcontext/PulsarTestContext.java | 3 + .../AuthenticatedProducerConsumerTest.java | 53 ++++--- ...enticationTlsHostnameVerificationTest.java | 26 ++-- .../api/ClientAuthenticationTlsTest.java | 27 ++-- .../client/api/ProducerConsumerBase.java | 5 - .../pulsar/client/api/ProxyProtocolTest.java | 12 +- .../client/api/TlsHostVerificationTest.java | 36 +++-- .../client/api/TlsProducerConsumerBase.java | 23 ++- .../client/api/TlsProducerConsumerTest.java | 20 +-- .../apache/pulsar/client/api/TlsSniTest.java | 6 +- .../TokenExpirationProduceConsumerTest.java | 10 +- .../worker/PulsarFunctionLocalRunTest.java | 16 ++- .../worker/PulsarFunctionPublishTest.java | 16 ++- .../pulsar/io/AbstractPulsarE2ETest.java | 16 ++- .../pulsar/io/PulsarFunctionAdminTest.java | 20 +-- .../pulsar/io/PulsarFunctionTlsTest.java | 21 ++- .../proxy/ProxyPublishConsumeTlsTest.java | 13 +- tests/certificate-authority/.gitignore | 3 + tests/certificate-authority/README.md | 24 +++- tests/certificate-authority/index.txt | 2 + tests/certificate-authority/newcerts/1007.pem | 111 +++++++++++++++ tests/certificate-authority/newcerts/1008.pem | 110 ++++++++++++++ tests/certificate-authority/openssl.cnf | 17 ++- tests/certificate-authority/serial | 2 +- .../server-keys/broker.cert.pem | 134 ++++++++++++++---- .../server-keys/broker.csr.pem | 26 ++-- .../server-keys/broker.key-pk8.pem | 52 +++---- .../server-keys/broker.key.pem | 50 +++---- .../server-keys/proxy.cert.pem | 133 +++++++++++++---- .../server-keys/proxy.csr.pem | 26 ++-- .../server-keys/proxy.key-pk8.pem | 52 +++---- .../server-keys/proxy.key.pem | 50 +++---- 41 files changed, 812 insertions(+), 633 deletions(-) delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem create mode 100644 tests/certificate-authority/.gitignore create mode 100644 tests/certificate-authority/newcerts/1007.pem create mode 100644 tests/certificate-authority/newcerts/1008.pem diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 3b8c6754c3fa9..e8348be9292cd 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -85,6 +85,28 @@ true + + maven-resources-plugin + + + copy-resources + test-compile + + copy-resources + + + ${project.build.testOutputDirectory}/certificate-authority + true + + + ${project.parent.parent.basedir}/tests/certificate-authority + false + + + + + + diff --git a/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java b/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java index 330d4fbc06897..e8e12838defef 100644 --- a/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java +++ b/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java @@ -37,11 +37,6 @@ import org.testng.annotations.BeforeMethod; public class TlsProducerConsumerBase extends ProducerConsumerBase { - protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; private final String clusterName = "use"; @BeforeMethod(alwaysRun = true) @@ -63,9 +58,9 @@ protected void cleanup() throws Exception { protected void internalSetUpForBroker() throws Exception { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setClusterName(clusterName); conf.setTlsRequireTrustedClientCertOnConnect(true); Set tlsProtocols = Sets.newConcurrentHashSet(); @@ -81,12 +76,12 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) } ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(lookupUrl) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) .operationTimeout(1000, TimeUnit.MILLISECONDS); if (addCertificates) { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); clientBuilder.authentication(AuthenticationTls.class.getName(), authParams); } pulsarClient = clientBuilder.build(); @@ -94,15 +89,15 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) protected void internalSetUpForNamespace() throws Exception { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); if (admin != null) { admin.close(); } admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), authParams).build()); admin.clusters().createCluster(clusterName, ClusterData.builder() .serviceUrl(brokerUrl.toString()) diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem deleted file mode 100644 index e2b44e0bf0c42..0000000000000 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem +++ /dev/null @@ -1,71 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 15537474201172114493 (0xd7a0327703a8fc3d) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=CARoot - Validity - Not Before: Feb 22 06:26:33 2023 GMT - Not After : Feb 19 06:26:33 2033 GMT - Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=localhost - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:af:bf:b7:2d:98:ad:9d:f6:da:a3:13:d4:62:0f: - 98:be:1c:a2:89:22:ba:6f:d5:fd:1f:67:e3:91:03: - 98:80:81:0e:ed:d8:f6:70:7f:2c:36:68:3d:53:ea: - 58:3a:a6:d5:89:66:4b:bd:1e:57:71:13:6d:4b:11: - e5:40:a5:76:84:24:92:40:58:80:96:c9:1f:2c:c4: - 55:eb:a3:79:73:70:5c:37:9a:89:ed:2f:ba:6b:e3: - 82:7c:69:4a:02:54:8b:81:5e:3c:bf:4c:8a:cb:ea: - 2c:5e:83:e7:b7:10:08:5f:82:58:a3:89:d1:da:92: - ba:2a:28:ee:30:28:3f:5b:ae:10:71:96:c7:e1:12: - c5:b0:1a:ad:44:6f:44:3a:11:4a:9a:3c:0f:8d:06: - 80:7b:34:ef:3f:6c:f4:5e:c5:44:54:1e:c8:dd:c7: - 80:85:80:d9:68:e6:c6:53:03:77:e1:fe:18:61:07: - 77:05:4c:ed:59:bc:5d:41:38:6a:ef:5d:a1:b2:60: - 98:d4:48:28:95:02:8a:0e:fd:cf:7b:1b:d2:11:cc: - 10:0c:50:73:d7:cc:38:6c:83:dd:79:26:aa:90:c8: - 9b:84:86:bc:59:e9:62:69:f4:98:1b:c4:80:78:7e: - a0:1a:81:9d:d2:e1:66:dd:c4:cc:fc:63:04:ac:ec: - a7:35 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:localhost, IP Address:127.0.0.1 - Signature Algorithm: sha256WithRSAEncryption - 5f:e0:73:7b:5e:db:c0:8b:5e:4c:43:5f:80:94:ca:0b:f8:e9: - 9b:93:91:3d:b1:3a:99:ce:1c:fb:15:32:68:3e:b9:9c:52:d0: - 4b:7f:17:09:ec:af:6b:05:3e:e2:a3:e6:cc:bb:53:d7:ea:4a: - 82:3c:4e:a5:37:ca:f4:1e:38:e2:d6:a5:98:4d:ee:b9:e2:9a: - 48:d2:9f:0a:bc:61:42:70:22:b9:fb:cd:73:72:fb:94:13:ac: - 6e:c5:b6:4b:24:ef:0f:df:2d:e6:56:da:b2:76:e8:16:be:7f: - 3f:1b:99:6e:32:3e:b9:f4:2b:35:72:c7:e4:c6:a5:92:68:c0: - 1f:a0:f7:17:fd:a3:b6:73:98:d3:ea:1c:af:ea:7d:f8:a0:27: - 40:dc:4e:8b:13:28:ba:65:60:c5:90:57:e8:54:c1:83:b4:9d: - f0:ae:2a:de:27:57:e5:a2:e5:f4:87:1c:df:6b:dc:7b:43:ff: - b6:be:0b:3b:b2:8b:1a:36:dc:e3:57:aa:52:ef:23:d6:50:d7: - e4:72:8f:a0:0a:43:de:3d:f2:42:5b:fa:ed:1f:8d:0e:cf:c5: - 6a:ce:3b:8e:fd:6b:68:01:a9:f9:d2:0e:0d:ac:39:8d:f5:6c: - 80:f8:49:af:bb:b9:d4:81:b9:f3:b2:b6:ce:75:1c:20:e8:6a: - 53:dc:26:86 ------BEGIN CERTIFICATE----- -MIIDCTCCAfGgAwIBAgIJANegMncDqPw9MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV -BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ -BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL -Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv1f0fZ+OR -A5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICWyR8sxFXr -o3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHakroqKO4w -KD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CFgNlo5sZT -A3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPXzDhsg915 -JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAGjHjAcMBoG -A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAX+Bz -e17bwIteTENfgJTKC/jpm5ORPbE6mc4c+xUyaD65nFLQS38XCeyvawU+4qPmzLtT -1+pKgjxOpTfK9B444talmE3uueKaSNKfCrxhQnAiufvNc3L7lBOsbsW2SyTvD98t -5lbasnboFr5/PxuZbjI+ufQrNXLH5MalkmjAH6D3F/2jtnOY0+ocr+p9+KAnQNxO -ixMoumVgxZBX6FTBg7Sd8K4q3idX5aLl9Icc32vce0P/tr4LO7KLGjbc41eqUu8j -1lDX5HKPoApD3j3yQlv67R+NDs/Fas47jv1raAGp+dIODaw5jfVsgPhJr7u51IG5 -87K2znUcIOhqU9wmhg== ------END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem deleted file mode 100644 index 004bf8e21a7a9..0000000000000 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCvv7ctmK2d9tqj -E9RiD5i+HKKJIrpv1f0fZ+ORA5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21L -EeVApXaEJJJAWICWyR8sxFXro3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixe -g+e3EAhfglijidHakroqKO4wKD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/ -bPRexURUHsjdx4CFgNlo5sZTA3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO -/c97G9IRzBAMUHPXzDhsg915JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8 -YwSs7Kc1AgMBAAECggEAAaWEK9MwXTiA1+JJrRmETtOp2isPIBkbI/4vLZ6hASM0 -ZpoPxQIMAf58BJs/dF03xu/EaeMs4oxSC9ABG9fxAk/tZtjta3w65Ip6W5jOfHxj -AMpb3HMEBhq9kDjUTq1IGVAutYQcEMkC3WfS9e4ahfqMpguWgbu6LsbvZFgcL9mv -pGnKv9YVe6Xk6isvqtq6G1af0rd7c//xF0i0e/qEo83Buok3gLEZOELZbcRxjUYc -jnyglnXnwkGjuL4E3wgS3l73ZKsb6+AYoqhMPVz8t4/PN3tTrsBJKOSYo8KzIm0U -ek9T8XmPbP0cuheRxp9Dp8TXJJQZK0N9jz+EL0ogQQKBgQDnavm8GpR4pap9cDOc -+YI5s823b507pNdSU8elO9gLsP0JlFzv+sqghVko29r85D7Vn3MkgYTy0S4ANLCs -0NFDY8N2QH6U1dTkk1QXZydVZDuKJ5SSpC4v+Vafl8yDxhB4Nlxhbm9vJEMfLcXh -2kL6UlAuFDtYD0AdczwnHu5DjQKBgQDCauocm55FpcyDMMBO2CjurxcjBYS3S1xT -Bz+sPtxJLjlKbAt8kSHUQcCcX9zhrQBfsT38LATCmKaOFqUW5/PPh2LcrxiMqlL1 -OJBUJ3Te2LTjlUn8r+DHv/69UIh5tchwRr3YgB0DuIs7jfmr4VfiOWTBtPVhoGFR -1Wt60j30SQKBgHzreS26J2VNAFBALgxRf6OIVMbtgDG/FOCDCyU9vazp+F2gcd61 -QYYPFYcBzx9uUiDctroBFHRCyJMh3jEbc6ruAogl3m6XUxmkEeOkMk5dEerM3N2f -tLL+5Gy385U6aI+LwKhzhcG4EGeXPNdjC362ykNldnddnB2Jo/H2N2XNAoGAdnft -xpbxP+GDGKIZXTIM5zzcLWQMdiC+1n1BSHVZiGJZWMczzKknYw7aDq+/iekApE79 -xW8RS373ZvfXi3i2Mcx+6pjrrbOQL4tTL2SHq8+DknaDCi4mG7IbyUKMlxW1WO1S -e929UGogtZ6S+DCte9WbVwosyFuRUetpvgLk67kCgYBWetihZjgBWrqVYT24TTRH -KxzSzH1JgzzF9qgTdlhXDv9hC+Kc0uTKsgViesDqVuCOjkwzY5OQr9c6duO0fwwP -qNk/qltdgjMC5iiv7duyukfbEuqKEdGGer9HFb7en96dZdVQJpYHaaslAGurtD80 -ejCQZgzR2XaHSuIQb0IUVQ== ------END PRIVATE KEY----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem deleted file mode 100644 index 4ed454ec52a52..0000000000000 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem +++ /dev/null @@ -1,78 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 15358526754272834781 (0xd52472b5c5c3f4dd) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=CARoot - Validity - Not Before: Feb 22 06:26:32 2023 GMT - Not After : Feb 19 06:26:32 2033 GMT - Subject: CN=CARoot - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:d0:87:45:0b:b4:83:11:ab:5a:b4:b6:1c:15:d4: - 92:6a:0c:ac:3b:76:da:ff:8d:61:1b:bd:96:bd:d7: - b0:70:23:87:d4:00:19:b2:e5:63:b7:80:58:4a:a4: - d8:a8:a6:4f:eb:c8:8c:54:07:f5:56:52:23:64:fc: - 66:54:39:f1:33:d0:e5:cc:b6:40:c8:d7:9a:9f:0e: - c4:aa:57:b0:b3:e2:41:61:54:ca:1f:90:3b:18:ef: - 60:d2:dc:ee:34:29:33:08:1b:37:4b:c4:ca:7e:cb: - 94:7f:50:c4:8d:16:2f:90:03:94:07:bf:cf:52:ff: - 24:54:56:ac:74:6c:d3:31:8c:ce:ef:b3:14:5a:5b: - 8a:0c:83:2d:e1:f7:4d:60:2f:a1:4d:85:38:96:7f: - 01:2f:9a:99:c7:2e:3d:09:4d:5e:53:df:fd:29:9f: - ff:6b:e4:c2:a1:e3:67:85:db:e2:02:4d:6f:29:d4: - e1:b3:a2:34:71:e0:90:dd:3f:b3:3f:86:41:8c:97: - 09:e6:c3:de:a0:0e:d3:d4:3e:ce:ea:58:70:e6:9f: - 24:a8:19:ca:df:61:b8:9c:c3:4e:53:d0:69:96:44: - 84:76:2b:99:65:08:06:42:d4:b2:76:a7:2f:69:12: - d5:c2:65:a6:ff:2c:77:73:00:e7:97:a5:77:6b:8a: - 9c:3f - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Basic Constraints: critical - CA:TRUE - X509v3 Subject Key Identifier: - A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69 - X509v3 Authority Key Identifier: - keyid:A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69 - DirName:/CN=CARoot - serial:D5:24:72:B5:C5:C3:F4:DD - - Signature Algorithm: sha256WithRSAEncryption - 21:b1:4d:2b:14:1e:5a:91:5d:28:9e:ba:cb:ed:f1:96:da:c3: - fa:8d:b5:74:e4:c5:fb:2f:3e:39:b4:a6:59:69:dd:84:64:a8: - f0:e0:39:d2:ef:87:cc:8b:09:9f:0a:84:1f:d0:96:9c:4b:64: - ea:08:09:26:1c:84:f4:06:5f:5e:b9:ba:b3:3c:6c:81:e0:93: - 46:89:07:51:95:36:77:96:76:5d:a6:68:71:bb:60:88:a7:83: - 27:7c:66:5d:64:36:cb:8e:bd:02:f7:fb:52:63:83:2f:fe:57: - 4c:d5:0c:1b:ea:ef:88:ad:8c:a9:d4:b3:2c:b8:c4:e2:90:cb: - 0f:24:0e:df:fc:2a:c6:83:08:49:45:b0:41:85:0e:b4:6f:f7: - 18:56:7b:a5:0b:f6:1b:7f:72:88:ee:c8:ef:b3:e3:3e:f0:68: - 1b:c9:55:bb:4d:21:65:6b:9e:5c:dd:60:4b:7f:f1:84:f8:67: - 51:c2:60:88:42:6e:6c:9c:14:b8:96:b0:18:10:97:2c:94:e7: - 79:14:7b:d1:a2:a4:d8:94:84:ac:a9:ca:17:95:c2:27:8b:2b: - d8:19:6a:14:4b:c3:03:a6:30:55:40:bd:ce:0c:c2:d5:af:7d: - 6d:65:89:6b:74:ed:21:12:f1:aa:c9:c9:ba:da:9a:ca:14:6c: - 39:f4:02:32 ------BEGIN CERTIFICATE----- -MIIDGjCCAgKgAwIBAgIJANUkcrXFw/TdMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV -BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN -BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANCH -RQu0gxGrWrS2HBXUkmoMrDt22v+NYRu9lr3XsHAjh9QAGbLlY7eAWEqk2KimT+vI -jFQH9VZSI2T8ZlQ58TPQ5cy2QMjXmp8OxKpXsLPiQWFUyh+QOxjvYNLc7jQpMwgb -N0vEyn7LlH9QxI0WL5ADlAe/z1L/JFRWrHRs0zGMzu+zFFpbigyDLeH3TWAvoU2F -OJZ/AS+amccuPQlNXlPf/Smf/2vkwqHjZ4Xb4gJNbynU4bOiNHHgkN0/sz+GQYyX -CebD3qAO09Q+zupYcOafJKgZyt9huJzDTlPQaZZEhHYrmWUIBkLUsnanL2kS1cJl -pv8sd3MA55eld2uKnD8CAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E -FgQUp1VrURB1zk5bC2T/qW0j+1eIWWkwQQYDVR0jBDowOIAUp1VrURB1zk5bC2T/ -qW0j+1eIWWmhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANUkcrXFw/TdMA0GCSqG -SIb3DQEBCwUAA4IBAQAhsU0rFB5akV0onrrL7fGW2sP6jbV05MX7Lz45tKZZad2E -ZKjw4DnS74fMiwmfCoQf0JacS2TqCAkmHIT0Bl9eubqzPGyB4JNGiQdRlTZ3lnZd -pmhxu2CIp4MnfGZdZDbLjr0C9/tSY4Mv/ldM1Qwb6u+IrYyp1LMsuMTikMsPJA7f -/CrGgwhJRbBBhQ60b/cYVnulC/Ybf3KI7sjvs+M+8GgbyVW7TSFla55c3WBLf/GE -+GdRwmCIQm5snBS4lrAYEJcslOd5FHvRoqTYlISsqcoXlcIniyvYGWoUS8MDpjBV -QL3ODMLVr31tZYlrdO0hEvGqycm62prKFGw59AIy ------END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem deleted file mode 100644 index 3cf236c401255..0000000000000 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem +++ /dev/null @@ -1,71 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 15537474201172114494 (0xd7a0327703a8fc3e) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=CARoot - Validity - Not Before: Feb 22 06:26:33 2023 GMT - Not After : Feb 19 06:26:33 2033 GMT - Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=superUser - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:cd:43:7d:98:40:f9:b0:5b:bc:ae:db:c0:0b:ad: - 26:90:96:e0:62:38:ed:68:b1:70:46:3b:de:44:f9: - 14:51:86:10:eb:ca:90:e7:88:e8:f9:91:85:e0:dd: - b5:b4:14:b9:78:e3:86:d5:54:6d:68:ec:14:92:b4: - f8:22:5b:05:3d:ed:31:25:65:08:05:84:ca:e6:0c: - 21:12:58:32:c7:1a:60:a3:4f:d2:4a:9e:28:19:7c: - 45:84:00:8c:89:dc:de:8a:e5:4f:88:91:cc:a4:f1: - 81:45:4c:7d:c2:ff:e2:c1:89:c6:12:73:95:e2:36: - bd:db:ae:8b:5a:68:6a:90:51:de:2b:88:5f:aa:67: - f4:a8:e3:63:dc:be:19:82:cc:9d:7f:e6:8d:fb:82: - be:22:01:3d:56:13:3b:5b:04:b4:e8:c5:18:e6:2e: - 0d:fa:ba:4a:8d:e8:c6:5a:a1:51:9a:4a:62:d7:af: - dd:b4:fc:e2:d5:cd:ae:99:6c:5c:61:56:0b:d7:0c: - 1a:77:5c:f5:3a:6a:54:b5:9e:33:ac:a9:75:28:9a: - 76:af:d0:7a:57:00:1b:91:13:31:fd:42:88:21:47: - 05:10:01:2f:59:bb:c7:3a:d9:e1:58:4c:1b:6c:71: - b6:98:ef:dd:03:82:58:a3:32:dc:90:a1:b6:a6:1e: - e1:0b - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:localhost, IP Address:127.0.0.1 - Signature Algorithm: sha256WithRSAEncryption - b8:fc:d3:8f:8a:e0:6b:74:57:e2:a3:79:b2:18:60:0b:2c:05: - f9:e3:ae:dd:e9:ad:52:88:52:73:b4:12:b0:39:90:65:12:f5: - 95:0e:5f:4b:f2:06:4a:57:ab:e1:f9:b1:34:68:83:d7:d7:5e: - 69:0a:16:44:ea:1d:97:53:51:10:51:8b:ec:0a:b3:c8:a3:3d: - 85:4d:f4:8f:7d:b3:b5:72:e4:9e:d7:f3:01:bf:66:e1:40:92: - 54:63:16:b6:b5:66:ed:30:38:94:1d:1a:8f:28:34:27:ab:c9: - 5f:d5:16:7e:e4:f5:93:d2:19:35:44:0a:c4:2e:6a:25:38:1d: - ee:5a:c8:29:fa:96:dc:95:82:38:9e:36:3a:68:34:7b:4e:d9: - fa:0d:b2:88:a2:6c:4f:03:18:a7:e3:41:67:38:de:e5:f6:ff: - 2a:1c:f0:ec:1a:02:a7:e8:4e:3a:c3:04:72:f8:6a:4f:28:a6: - cf:0b:a2:db:33:74:d1:10:9e:ec:b4:ac:f8:b1:24:f4:ef:0e: - 05:e4:9d:1b:9a:40:f7:09:66:9c:9d:86:8b:76:96:46:e8:d1: - dc:10:c7:7d:0b:69:41:dc:a7:8e:e3:a3:36:e3:42:63:93:8c: - 91:80:0d:27:11:1c:2d:ae:fb:92:88:6c:6b:09:40:1a:30:dd: - 8f:ac:0f:62 ------BEGIN CERTIFICATE----- -MIIDCTCCAfGgAwIBAgIJANegMncDqPw+MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV -BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ -BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL -Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1osXBGO95E -+RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgFhMrmDCES -WDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5XiNr3brota -aGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36ukqN6MZa -oVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpXABuREzH9 -QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAGjHjAcMBoG -A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAuPzT -j4rga3RX4qN5shhgCywF+eOu3emtUohSc7QSsDmQZRL1lQ5fS/IGSler4fmxNGiD -19deaQoWROodl1NREFGL7AqzyKM9hU30j32ztXLkntfzAb9m4UCSVGMWtrVm7TA4 -lB0ajyg0J6vJX9UWfuT1k9IZNUQKxC5qJTgd7lrIKfqW3JWCOJ42Omg0e07Z+g2y -iKJsTwMYp+NBZzje5fb/Khzw7BoCp+hOOsMEcvhqTyimzwui2zN00RCe7LSs+LEk -9O8OBeSdG5pA9wlmnJ2Gi3aWRujR3BDHfQtpQdynjuOjNuNCY5OMkYANJxEcLa77 -kohsawlAGjDdj6wPYg== ------END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem deleted file mode 100644 index 3835b3eacccc0..0000000000000 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDNQ32YQPmwW7yu -28ALrSaQluBiOO1osXBGO95E+RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSS -tPgiWwU97TElZQgFhMrmDCESWDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFF -TH3C/+LBicYSc5XiNr3brotaaGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1W -EztbBLToxRjmLg36ukqN6MZaoVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1 -njOsqXUomnav0HpXABuREzH9QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQ -obamHuELAgMBAAECggEBALGnokJuqiz7mTj2NSdl+6TVEOuyPbiJKpV/J4cm1XEh -ye9qaTQcCRhH3UmcWrG75jM9KevloLRY8A1x1/lUMhtA+XJWGTU9k6a8BLut3nT4 -3X87jNTMQgSczEXNe9WudmZcxhN7rVVtOOdTpt1pP0cnCWna5HTf0D8cuLvM975j -r1YGTjKsCF1W+tp6ZAIIMfJkUI2qBRKvSxVCSs1vZBraox3yUVnq9oRLHxZZoqOd -d51G5phRtn6ReVPBdT8fGUBEGg3jKxTu2/vLQMUyHy0hyCAM20gzOP4FIc2g+QZU -y42byAuc89m0OrdRWsmzHCOxcq9DwY9npaz1RscR/2ECgYEA9bHJQ0Y1afpS5gn2 -KnXenRIw9oal1utQZnohCEJ4um+K/BCEHtDnI825LPNf34IKM2rSmssvHrYN51o0 -92j9lHHXsf6MVluwsTsIu8MtNaJ1BLt96dub4ScGT6vvzObKTwsajUfIHk+FNsKq -zps8yh1q0qyyfAcvR82+Xr6JIsMCgYEA1d+RHGewi/Ub/GCG99A1KFKsgbiIJnWB -IFmrcyPWignhzDUcw2SV9XqAzeK8EOIHNq3e5U/tkA7aCWxtLb5UsQ8xvmwQY2cy -X2XvSdIhO4K2PgRLgjlzZ8RHSULglqyjB2i6TjwjFl8TsRzYr6JlV6+2cMujw4Bl -g3a8gz071BkCgYBLP7BMkmw5kRliqxph1sffg3rLhmG0eU2elTkYtoMTVqZSnRxZ -89FW/eMBCWkLo2BMbyMhlalQ1qFbgh1GyTkhBdzx/uwsZtiu7021dAmcq6z7ThE6 -VrBfPPyJ2jcPon/DxbrUGnAIGILMSsLVlGYB4RCehZYEto6chz8O9Xw60QKBgCnd -us1BqviqwZC04JbQJie/j09RbS2CIQXRJ9PBNzUMXCwaVYgWP5ivI1mqQcBYTqsw -fAqNi+aAUcQ4emLS+Ec0vzsUclzTDbRJAv+DZ8f7fWtEcfeLAYFVldLMiaRVJRDF -OnsoIII3mGY6TFyNQKNanS8VXfheQQDsFFjoera5AoGBALXYEXkESXpw4LT6qJFz -ktQuTZDfS6LtR14/+NkYL9c5wBC4Otkg4bNbT8xGlUjethRfpkm8xRTB6zfC1/p/ -Cg6YU1cwqlkRurAhE3PEv1dCc1IDbzou8xnwqHrd6sGPDQmQ3aEtU5eJhDZKIZfx -nQqPGK92+Jtne7+W1mFZooxs ------END PRIVATE KEY----- diff --git a/build/regenerate_certs_for_tests.sh b/build/regenerate_certs_for_tests.sh index fff1c057060f3..9582a7496cd1d 100755 --- a/build/regenerate_certs_for_tests.sh +++ b/build/regenerate_certs_for_tests.sh @@ -68,13 +68,6 @@ reissue_certificate_no_subject \ $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem \ $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem -generate_ca -cp ca-cert.pem $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem -reissue_certificate $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem \ - $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem -reissue_certificate $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem \ - $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem - generate_ca cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-key.pem \ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java index 19a550457a4dd..0e4f1bccc814d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java @@ -63,7 +63,7 @@ private void buildConf(ServiceConfiguration conf) { conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setAuthenticationEnabled(true); - conf.setSuperUserRoles(Set.of("superproxy", "broker.pulsar.apache.org")); + conf.setSuperUserRoles(Set.of("superproxy", "broker-localhost-SAN")); conf.setAuthenticationProviders( Set.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls")); conf.setAuthorizationEnabled(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java index e170425ffe443..d3d4b7cf9341c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java @@ -323,6 +323,9 @@ public Builder spyConfigCustomizer(Consumer spyConfigCustomiz */ public Builder configCustomizer(Consumer configCustomerizer) { configCustomerizer.accept(svcConfig); + if (config != null) { + configCustomerizer.accept(config); + } return this; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java index 71e30c21b3c20..75ae91f18a305 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java @@ -64,12 +64,6 @@ public class AuthenticatedProducerConsumerTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(AuthenticatedProducerConsumerTest.class); - private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - private final String BASIC_CONF_FILE_PATH = "./src/test/resources/authentication/basic/.htpasswd"; private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); @@ -88,9 +82,9 @@ protected void setup() throws Exception { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); conf.setTlsAllowInsecureConnection(true); conf.setTopicLevelPoliciesEnabled(false); @@ -104,7 +98,8 @@ protected void setup() throws Exception { conf.setBrokerClientTlsEnabled(true); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); + "tlsCertFile:" + getTlsFileForClient("admin.cert") + + ",tlsKeyFile:" + getTlsFileForClient("admin.key-pk8")); Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); @@ -126,7 +121,7 @@ protected void setup() throws Exception { protected final void internalSetup(Authentication auth) throws Exception { admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).authentication(auth) .build()); String lookupUrl; // For http basic authentication test @@ -136,7 +131,7 @@ protected final void internalSetup(Authentication auth) throws Exception { lookupUrl = pulsar.getBrokerServiceUrlTls(); } replacePulsarClient(PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).authentication(auth) .enableTls(true)); } @@ -188,8 +183,8 @@ public void testTlsSyncProducerAndConsumer(int batchMessageDelayMs) throws Excep log.info("-- Starting {} test --", methodName); Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); internalSetup(authTls); @@ -246,8 +241,8 @@ public void testAnonymousSyncProducerAndConsumer(int batchMessageDelayMs) throws log.info("-- Starting {} test --", methodName); Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); internalSetup(authTls); @@ -291,8 +286,8 @@ public void testAuthenticationFilterNegative() throws Exception { log.info("-- Starting {} test --", methodName); Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); internalSetup(authTls); @@ -324,8 +319,8 @@ public void testInternalServerExceptionOnLookup() throws Exception { log.info("-- Starting {} test --", methodName); Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); internalSetup(authTls); @@ -362,8 +357,8 @@ public void testInternalServerExceptionOnLookup() throws Exception { @Test public void testDeleteAuthenticationPoliciesOfTopic() throws Exception { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); internalSetup(authTls); @@ -424,7 +419,8 @@ public void testDeleteAuthenticationPoliciesOfTopic() throws Exception { admin.clusters().deleteCluster("test"); } - private final Authentication tlsAuth = new AuthenticationTls(TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH); + private final Authentication tlsAuth = + new AuthenticationTls(getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8")); private final Authentication tokenAuth = new AuthenticationToken(ADMIN_TOKEN); @DataProvider @@ -454,10 +450,9 @@ public void testTlsTransportWithAnyAuth(Supplier url, Authentication aut @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(url.get()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) - .tlsKeyFilePath(TLS_CLIENT_KEY_FILE_PATH) - .tlsCertificateFilePath(TLS_CLIENT_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .tlsKeyFilePath(getTlsFileForClient("admin.key-pk8")) + .tlsCertificateFilePath(getTlsFileForClient("admin.cert")) .authentication(auth) .allowTlsInsecureConnection(false) .enableTlsHostnameVerification(false) @@ -470,8 +465,8 @@ public void testTlsTransportWithAnyAuth(Supplier url, Authentication aut @Test public void testCleanupEmptyTopicAuthenticationMap() throws Exception { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); internalSetup(authTls); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java index f2631f591217b..e3bd321d76332 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java @@ -46,17 +46,10 @@ public class AuthenticationTlsHostnameVerificationTest extends ProducerConsumerB private final String TLS_MIM_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/hn-verification/broker-cert.pem"; private final String TLS_MIM_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/hn-verification/broker-key.pem"; - private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - private final String BASIC_CONF_FILE_PATH = "./src/test/resources/authentication/basic/.htpasswd"; private boolean hostnameVerificationEnabled = true; - private String clientTrustCertFilePath = TLS_TRUST_CERT_FILE_PATH; + private String clientTrustCertFilePath = CA_CERT_FILE_PATH; protected void setup() throws Exception { super.internalSetup(); @@ -81,7 +74,8 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SERVER_KEY_FILE_PATH); + "tlsCertFile:" + getTlsFileForClient("admin.cert") + + ",tlsKeyFile:" + getTlsFileForClient("admin.key-pk8")); Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); @@ -100,8 +94,8 @@ protected void setup() throws Exception { protected void setupClient() throws Exception { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); @@ -147,11 +141,11 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname // setup broker cert which has CN = "pulsar" different than broker's hostname="localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_MIM_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_MIM_SERVER_KEY_FILE_PATH); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_MIM_SERVER_KEY_FILE_PATH); + "tlsCertFile:" + getTlsFileForClient("admin.cert") + "," + "tlsKeyFile:" + TLS_MIM_SERVER_KEY_FILE_PATH); setup(); @@ -188,9 +182,9 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception { // setup broker cert which has CN = "localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); setup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java index 186bf9d736e45..c9b243257c4e1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java @@ -37,15 +37,9 @@ @Test(groups = "broker-api") public class ClientAuthenticationTlsTest extends ProducerConsumerBase { - private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; private final Authentication authenticationTls = - new AuthenticationTls(TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH); + new AuthenticationTls(getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8")); @Override protected void doInitConf() throws Exception { @@ -57,17 +51,18 @@ protected void doInitConf() throws Exception { providers.add(AuthenticationProviderTls.class.getName()); conf.setAuthenticationProviders(providers); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setTlsAllowInsecureConnection(false); conf.setBrokerClientTlsEnabled(true); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); - conf.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + "tlsCertFile:" + getTlsFileForClient("admin.cert") + + ",tlsKeyFile:" + getTlsFileForClient("admin.key-pk8")); + conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); } @BeforeClass(alwaysRun = true) @@ -94,7 +89,7 @@ public void testAdminWithTrustCert() throws PulsarClientException, PulsarAdminEx @Cleanup PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(getPulsar().getWebServiceAddressTls()) .sslProvider("JDK") - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .build(); pulsarAdmin.clusters().getClusters(); } @@ -105,7 +100,7 @@ public void testAdminWithFull() throws PulsarClientException, PulsarAdminExcepti PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(getPulsar().getWebServiceAddressTls()) .sslProvider("JDK") .authentication(authenticationTls) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .build(); pulsarAdmin.clusters().getClusters(); } @@ -139,7 +134,7 @@ public void testClientWithTrustCert() throws PulsarClientException, PulsarAdminE PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(getPulsar().getBrokerServiceUrlTls()) .sslProvider("JDK") .operationTimeout(3, TimeUnit.SECONDS) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .build(); @Cleanup Producer ignored = pulsarClient.newProducer().topic(UUID.randomUUID().toString()).create(); @@ -152,7 +147,7 @@ public void testClientWithFull() throws PulsarClientException, PulsarAdminExcept .sslProvider("JDK") .operationTimeout(3, TimeUnit.SECONDS) .authentication(authenticationTls) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .build(); @Cleanup Producer ignored = pulsarClient.newProducer().topic(UUID.randomUUID().toString()).create(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java index ca58bddf13c23..f58c1fa26afc7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java @@ -31,11 +31,6 @@ import org.testng.annotations.BeforeMethod; public abstract class ProducerConsumerBase extends MockedPulsarServiceBaseTest { - protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; protected String methodName; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java index 7f632d5a76485..19009689dc8a1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java @@ -45,11 +45,11 @@ public void testSniProxyProtocol() throws Exception { String topicName = "persistent://my-property/use/my-ns/my-topic1"; ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(brokerServiceUrl) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) .proxyServiceUrl(proxyUrl, ProxyProtocol.SNI).operationTimeout(1000, TimeUnit.MILLISECONDS); Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); clientBuilder.authentication(AuthenticationTls.class.getName(), authParams); @Cleanup @@ -68,11 +68,11 @@ public void testSniProxyProtocolWithInvalidProxyUrl() throws Exception { String topicName = "persistent://my-property/use/my-ns/my-topic1"; ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(brokerServiceUrl) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) .proxyServiceUrl(proxyUrl, ProxyProtocol.SNI).operationTimeout(1000, TimeUnit.MILLISECONDS); Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); clientBuilder.authentication(AuthenticationTls.class.getName(), authParams); @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java index 95a78d7ffcef6..fff61c5c8c940 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.auth.AuthenticationTls; @@ -30,21 +31,38 @@ @Test(groups = "broker-api") public class TlsHostVerificationTest extends TlsProducerConsumerBase { + @Override + @Test(enabled = false) + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + builder.configCustomizer(config -> { + // Advertise a hostname that routes but is not on the certificate + // Note that if you are on a Mac, you'll need to run the following to make loopback work for 127.0.0.2 + // $ sudo ifconfig lo0 alias 127.0.0.2 up + config.setAdvertisedAddress("127.0.0.2"); + }); + } + @Test public void testTlsHostVerificationAdminClient() throws Exception { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); - String websocketTlsAddress = pulsar.getWebServiceAddressTls(); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); + Assert.assertTrue(pulsar.getWebServiceAddressTls().startsWith("https://127.0.0.2:"), + "Test relies on this address"); PulsarAdmin adminClientTls = PulsarAdmin.builder() - .serviceHttpUrl(websocketTlsAddress.replace("localhost", "127.0.0.1")) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false) + .serviceHttpUrl(pulsar.getWebServiceAddressTls()) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), authParams).enableTlsHostnameVerification(true) + .requestTimeout(1, java.util.concurrent.TimeUnit.SECONDS) .build(); try { adminClientTls.tenants().getTenants(); Assert.fail("Admin call should be failed due to hostnameVerification enabled"); + } catch (PulsarAdminException.TimeoutException e) { + // The test was previously able to fail here, but that is not the right way for the test to pass. + // If you hit this error and are running on OSX, you may need to run "sudo ifconfig lo0 alias 127.0.0.2 up" + Assert.fail("Admin call should not timeout, it should fail due to SSL error"); } catch (PulsarAdminException e) { // Ok } @@ -53,11 +71,13 @@ public void testTlsHostVerificationAdminClient() throws Exception { @Test public void testTlsHostVerificationDisabledAdminClient() throws Exception { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); + Assert.assertTrue(pulsar.getWebServiceAddressTls().startsWith("https://127.0.0.2:"), + "Test relies on this address"); PulsarAdmin adminClient = PulsarAdmin.builder() .serviceHttpUrl(pulsar.getWebServiceAddressTls()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), authParams).enableTlsHostnameVerification(false) .build(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java index 6a2109836a2c9..39bab20d97df5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java @@ -38,11 +38,6 @@ @Test(groups = "broker-api") public abstract class TlsProducerConsumerBase extends ProducerConsumerBase { - protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; private final String clusterName = "use"; @BeforeMethod @@ -64,9 +59,9 @@ protected void cleanup() throws Exception { protected void internalSetUpForBroker() { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setClusterName(clusterName); conf.setTlsRequireTrustedClientCertOnConnect(true); Set tlsProtocols = Sets.newConcurrentHashSet(); @@ -81,12 +76,12 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) pulsarClient.close(); } ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(lookupUrl) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false) .operationTimeout(1000, TimeUnit.MILLISECONDS); if (addCertificates) { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); clientBuilder.authentication(AuthenticationTls.class.getName(), authParams); } replacePulsarClient(clientBuilder); @@ -94,15 +89,15 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) protected void internalSetUpForNamespace() throws Exception { Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); if (admin != null) { admin.close(); } admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), authParams).build()); admin.clusters().createCluster(clusterName, ClusterData.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java index 0563fc3b9da37..879289eb65dc8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java @@ -146,9 +146,9 @@ public void testTlsCertsFromDynamicStream() throws Exception { .operationTimeout(1000, TimeUnit.MILLISECONDS); AtomicInteger index = new AtomicInteger(0); - ByteArrayInputStream certStream = createByteInputStream(TLS_CLIENT_CERT_FILE_PATH); - ByteArrayInputStream keyStream = createByteInputStream(TLS_CLIENT_KEY_FILE_PATH); - ByteArrayInputStream trustStoreStream = createByteInputStream(TLS_TRUST_CERT_FILE_PATH); + ByteArrayInputStream certStream = createByteInputStream(getTlsFileForClient("admin.cert")); + ByteArrayInputStream keyStream = createByteInputStream(getTlsFileForClient("admin.key-pk8")); + ByteArrayInputStream trustStoreStream = createByteInputStream(CA_CERT_FILE_PATH); Supplier certProvider = () -> getStream(index, certStream); Supplier keyProvider = () -> getStream(index, keyStream); @@ -203,9 +203,9 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception AtomicInteger certIndex = new AtomicInteger(1); AtomicInteger keyIndex = new AtomicInteger(0); AtomicInteger trustStoreIndex = new AtomicInteger(1); - ByteArrayInputStream certStream = createByteInputStream(TLS_CLIENT_CERT_FILE_PATH); - ByteArrayInputStream keyStream = createByteInputStream(TLS_CLIENT_KEY_FILE_PATH); - ByteArrayInputStream trustStoreStream = createByteInputStream(TLS_TRUST_CERT_FILE_PATH); + ByteArrayInputStream certStream = createByteInputStream(getTlsFileForClient("admin.cert")); + ByteArrayInputStream keyStream = createByteInputStream(getTlsFileForClient("admin.key-pk8")); + ByteArrayInputStream trustStoreStream = createByteInputStream(CA_CERT_FILE_PATH); Supplier certProvider = () -> getStream(certIndex, certStream, keyStream/* invalid cert file */); Supplier keyProvider = () -> getStream(keyIndex, keyStream); @@ -252,7 +252,8 @@ private ByteArrayInputStream getStream(AtomicInteger index, ByteArrayInputStream return streams[index.intValue()]; } - private final Authentication tlsAuth = new AuthenticationTls(TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH); + private final Authentication tlsAuth = + new AuthenticationTls(getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8")); @DataProvider public Object[] tlsTransport() { @@ -276,13 +277,14 @@ public void testTlsTransport(Supplier url, Authentication auth) throws E internalSetUpForNamespace(); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(url.get()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .allowTlsInsecureConnection(false) .enableTlsHostnameVerification(false) .authentication(auth); if (auth == null) { - clientBuilder.tlsKeyFilePath(TLS_CLIENT_KEY_FILE_PATH).tlsCertificateFilePath(TLS_CLIENT_CERT_FILE_PATH); + clientBuilder.tlsKeyFilePath(getTlsFileForClient("admin.key-pk8")) + .tlsCertificateFilePath(getTlsFileForClient("admin.cert")); } @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java index fd722e52e5f1f..173fa8acb0fdd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java @@ -50,12 +50,12 @@ public void testIpAddressInBrokerServiceUrl() throws Exception { brokerServiceUrlTls.getPort()); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(brokerServiceIpAddressUrl) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) .enableTlsHostnameVerification(false) .operationTimeout(1000, TimeUnit.MILLISECONDS); Map authParams = new HashMap<>(); - authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); - authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); clientBuilder.authentication(AuthenticationTls.class.getName(), authParams); @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java index e955a9ae7062f..4fc0d315d2253 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java @@ -101,9 +101,9 @@ public String getExpireToken(String role, Date date) { protected void internalSetUpForBroker() { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setClusterName(configClusterName); conf.setAuthenticationRefreshCheckSeconds(1); conf.setTlsRequireTrustedClientCertOnConnect(false); @@ -121,7 +121,7 @@ protected void internalSetUpForBroker() { private PulsarClient getClient(String token) throws Exception { ClientBuilder clientBuilder = PulsarClient.builder() .serviceUrl(pulsar.getBrokerServiceUrlTls()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .enableTls(true) .allowTlsInsecureConnection(false) .enableTlsHostnameVerification(true) @@ -132,7 +132,7 @@ private PulsarClient getClient(String token) throws Exception { private PulsarAdmin getAdmin(String token) throws Exception { PulsarAdminBuilder clientBuilder = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddressTls()) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .allowTlsInsecureConnection(false) .authentication(AuthenticationToken.class.getName(),"token:" +token) .enableTlsHostnameVerification(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index c832cba163d63..aa190cd2e0a73 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -89,6 +89,7 @@ import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.SinkContext; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,11 +122,16 @@ public class PulsarFunctionLocalRunTest { private static final String CLUSTER = "local"; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; + private final String TLS_SERVER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + private final String TLS_SERVER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private final String TLS_CLIENT_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem"); + private final String TLS_CLIENT_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem"); + private final String TLS_TRUST_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); private static final String SYSTEM_PROPERTY_NAME_NAR_FILE_PATH = "pulsar-io-data-generator.nar.path"; private PulsarFunctionTestTemporaryDirectory tempDirectory; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java index 95923586fe23e..c820f512a68de 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java @@ -71,6 +71,7 @@ import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -99,11 +100,16 @@ public class PulsarFunctionPublishTest { String primaryHost; String workerId; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; + private final String TLS_SERVER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + private final String TLS_SERVER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private final String TLS_CLIENT_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem"); + private final String TLS_CLIENT_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem"); + private final String TLS_TRUST_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); private PulsarFunctionTestTemporaryDirectory tempDirectory; @DataProvider(name = "validRoleName") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java index 9991e9f1b70fd..3a99cc647ed5c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java @@ -62,6 +62,7 @@ import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.awaitility.Awaitility; import org.slf4j.Logger; @@ -75,11 +76,16 @@ public abstract class AbstractPulsarE2ETest { public static final Logger log = LoggerFactory.getLogger(AbstractPulsarE2ETest.class); - protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; + protected final String TLS_SERVER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + protected final String TLS_SERVER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + protected final String TLS_CLIENT_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem"); + protected final String TLS_CLIENT_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem"); + protected final String TLS_TRUST_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); protected final String tenant = "external-repl-prop"; protected LocalBookkeeperEnsemble bkEnsemble; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java index 16218f6ce6448..d31d0c66bdf93 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,10 +78,16 @@ public class PulsarFunctionAdminTest { String pulsarFunctionsNamespace = tenant + "/pulsar-function-admin"; String primaryHost; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; + private final String TLS_SERVER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + private final String TLS_SERVER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private final String TLS_CLIENT_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem"); + private final String TLS_CLIENT_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem"); + private final String TLS_TRUST_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); private static final Logger log = LoggerFactory.getLogger(PulsarFunctionAdminTest.class); @@ -113,8 +120,7 @@ void setup(Method method) throws Exception { config.setAuthenticationProviders(providers); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - config.setTlsAllowInsecureConnection(true); - + config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); functionsWorkerService = createPulsarFunctionWorker(config); Optional functionWorkerService = Optional.of(functionsWorkerService); @@ -132,7 +138,6 @@ void setup(Method method) throws Exception { PulsarAdmin.builder() .serviceHttpUrl(pulsar.getWebServiceAddressTls()) .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH) - .allowTlsInsecureConnection(true) .authentication(authTls) .build()); @@ -203,7 +208,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setBrokerClientAuthenticationParameters( String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)); workerConfig.setUseTls(true); - workerConfig.setTlsAllowInsecureConnection(true); workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH); PulsarWorkerService workerService = new PulsarWorkerService(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java index 5de3d4f7e0868..810ac69ac3eb3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java @@ -66,6 +66,7 @@ import org.apache.pulsar.functions.worker.PulsarWorkerService.PulsarClientCreator; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.rest.WorkerServer; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,10 +91,16 @@ public class PulsarFunctionTlsTest { PulsarAdmin functionAdmin; private final List namespaceList = new LinkedList<>(); - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; + private final String TLS_SERVER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + private final String TLS_SERVER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private final String TLS_CLIENT_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem"); + private final String TLS_CLIENT_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem"); + private final String TLS_TRUST_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); private static final Logger log = LoggerFactory.getLogger(PulsarFunctionTlsTest.class); private PulsarFunctionTestTemporaryDirectory tempDirectory; @@ -121,7 +128,7 @@ void setup(Method method) throws Exception { config.setAuthenticationProviders(providers); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - config.setTlsAllowInsecureConnection(true); + config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); config.setAdvertisedAddress("localhost"); PulsarAdmin admin = mock(PulsarAdmin.class); @@ -163,7 +170,7 @@ void setup(Method method) throws Exception { authTls.configure(authParams); functionAdmin = PulsarAdmin.builder().serviceHttpUrl(functionTlsUrl) - .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH).allowTlsInsecureConnection(true) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .authentication(authTls).build(); Thread.sleep(100); @@ -217,7 +224,7 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)); workerConfig.setUseTls(true); workerConfig.setTlsAllowInsecureConnection(true); - workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH); + workerConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); workerConfig.setWorkerPortTls(0); workerConfig.setTlsEnabled(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java index 3ee9b6127de7b..91cd4fab470d6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java @@ -64,12 +64,13 @@ public void setup() throws Exception { config.setWebServicePort(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setBrokerClientTlsEnabled(true); - config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); - config.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); config.setClusterName("use"); - config.setBrokerClientAuthenticationParameters("tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + ",tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); + config.setBrokerClientAuthenticationParameters("tlsCertFile:" + getTlsFileForClient("admin.cert") + + ",tlsKeyFile:" + getTlsFileForClient("admin.key-pk8")); config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); service = spyWithClassAndConstructorArgs(WebSocketService.class, config); @@ -103,7 +104,7 @@ public void socketTest() throws GeneralSecurityException { SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setSslContext(SecurityUtility - .createSslContext(false, SecurityUtility.loadCertificatesFromPemFile(TLS_TRUST_CERT_FILE_PATH), null)); + .createSslContext(false, SecurityUtility.loadCertificatesFromPemFile(CA_CERT_FILE_PATH), null)); WebSocketClient consumeClient = new WebSocketClient(sslContextFactory); SimpleConsumerSocket consumeSocket = new SimpleConsumerSocket(); diff --git a/tests/certificate-authority/.gitignore b/tests/certificate-authority/.gitignore new file mode 100644 index 0000000000000..de3be7546361f --- /dev/null +++ b/tests/certificate-authority/.gitignore @@ -0,0 +1,3 @@ +# Files generated when running openssl +*.old +*.attr diff --git a/tests/certificate-authority/README.md b/tests/certificate-authority/README.md index 008120a35f4a1..02ebbdf9258d5 100644 --- a/tests/certificate-authority/README.md +++ b/tests/certificate-authority/README.md @@ -3,23 +3,33 @@ Generated based on instructions from https://jamielinux.com/docs/openssl-certificate-authority/introduction.html, though the intermediate CA has been omitted for simplicity. -The environment variable, CA_HOME, must be set to point to the directory -containing this file before running any openssl commands. +The following commands must be run in the same directory as this README due to the configuration for the openssl.cnf file. The password for the CA private key is ```PulsarTesting```. ## Generating server keys -In this example, we're generating a key for the broker. +In this example, we're generating a key for the broker and the proxy. If there is a need to create them again, a new +CN will need to be used because we have the index.txt database in this directory. It's also possible that we could +remove this file and start over. At the time of adding this change, I didn't see a need to change the paradigm. -The common name when generating the CSR should be the domain name of the broker. +The common name when generating the CSR used to be the domain name of the broker. However, now we rely on the Subject +Alternative Name, or the SAN, to be the domain name. This is because the CN is deprecated in the certificate spec. The +[openssl.cnf](openssl.cnf) file has been updated to reflect this change. The proxy and the broker have the following +SAN: ```DNS:localhost, IP:127.0.0.1```. ```bash openssl genrsa -out server-keys/broker.key.pem 2048 -openssl req -config openssl.cnf -key server-keys/broker.key.pem -new -sha256 -out server-keys/broker.csr.pem -openssl ca -config openssl.cnf -extensions server_cert \ - -days 100000 -notext -md sha256 -in server-keys/broker.csr.pem -out server-keys/broker.cert.pem +openssl req -config openssl.cnf -subj "/CN=broker-localhost-SAN" -key server-keys/broker.key.pem -new -sha256 -out server-keys/broker.csr.pem +openssl ca -config openssl.cnf -extensions broker_cert -days 100000 -md sha256 -in server-keys/broker.csr.pem \ + -out server-keys/broker.cert.pem -batch -key PulsarTesting openssl pkcs8 -topk8 -inform PEM -outform PEM -in server-keys/broker.key.pem -out server-keys/broker.key-pk8.pem -nocrypt + +openssl genrsa -out server-keys/proxy.key.pem 2048 +openssl req -config openssl.cnf -subj "/CN=proxy-localhost-SAN" -key server-keys/proxy.key.pem -new -sha256 -out server-keys/proxy.csr.pem +openssl ca -config openssl.cnf -extensions proxy_cert -days 100000 -md sha256 -in server-keys/proxy.csr.pem \ + -out server-keys/proxy.cert.pem -batch -key PulsarTesting +openssl pkcs8 -topk8 -inform PEM -outform PEM -in server-keys/proxy.key.pem -out server-keys/proxy.key-pk8.pem -nocrypt ``` You need to configure the server with broker.key-pk8.pem and broker.cert.pem. diff --git a/tests/certificate-authority/index.txt b/tests/certificate-authority/index.txt index 376f86725c21a..acb5eed051c13 100644 --- a/tests/certificate-authority/index.txt +++ b/tests/certificate-authority/index.txt @@ -5,3 +5,5 @@ V 22920409135604Z 1003 unknown /CN=proxy V 22920410132517Z 1004 unknown /CN=superproxy V 22920411084025Z 1005 unknown /CN=user1 V 22960802101401Z 1006 unknown /CN=proxy.pulsar.apache.org +V 22970222155018Z 1007 unknown /CN=broker-localhost-SAN +V 22970222155019Z 1008 unknown /CN=proxy-localhost-SAN diff --git a/tests/certificate-authority/newcerts/1007.pem b/tests/certificate-authority/newcerts/1007.pem new file mode 100644 index 0000000000000..4237719f20ebd --- /dev/null +++ b/tests/certificate-authority/newcerts/1007.pem @@ -0,0 +1,111 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4103 (0x1007) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=foobar + Validity + Not Before: May 10 15:50:18 2023 GMT + Not After : Feb 22 15:50:18 2297 GMT + Subject: CN=broker-localhost-SAN + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:de:d1:da:bb:91:b3:16:c4:b2:e8:89:30:9e:c1: + 5e:0b:cf:db:c4:c3:d9:b1:af:40:a5:0b:38:36:1b: + 14:fe:0f:22:9c:e6:59:6a:15:5b:db:f6:f7:f3:a5: + 02:29:94:7a:d2:0c:67:ad:aa:63:62:7e:fc:58:11: + 29:48:b8:3c:91:b2:73:7e:12:6b:f2:ea:36:77:0f: + 15:9b:46:95:ce:73:15:8d:c8:d9:97:57:03:90:33: + 2d:7d:f3:ee:e5:01:6d:d8:c6:da:ab:07:b9:dd:1c: + e0:4b:ce:6a:de:a8:d2:e3:c1:52:6d:83:3a:0a:f0: + ed:cf:f7:56:6a:87:0e:73:e3:12:82:2b:65:ab:d8: + a9:44:5b:4a:2f:a5:92:94:32:f1:a1:e4:af:18:0f: + 0f:18:60:cd:f7:d0:9d:03:9f:d7:e9:a8:60:54:bb: + 3b:9a:05:db:fd:38:04:3c:b4:23:41:16:6c:7c:3b: + d9:b6:e0:2f:bd:cb:62:55:1b:e8:d0:8f:43:76:ef: + 55:86:cf:25:c3:bc:ae:e3:46:50:89:f7:71:ad:06: + 5e:28:e6:f6:f0:76:27:ea:7e:1b:67:53:39:26:20: + 19:18:82:b1:11:5f:ea:91:c2:e3:d3:f6:5a:c7:fd: + 61:a2:92:de:7d:7c:da:6d:e8:bf:39:52:10:31:60: + 4b:e1 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Cert Type: + SSL Server + Netscape Comment: + OpenSSL Generated Server Certificate + X509v3 Subject Key Identifier: + 17:07:3B:AA:85:83:B5:04:83:EC:B2:6C:1E:3A:F0:F5:59:AA:61:28 + X509v3 Subject Alternative Name: + DNS:localhost, DNS:unresolvable-broker-address, IP Address:127.0.0.1 + X509v3 Authority Key Identifier: + keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27 + DirName:/CN=foobar + serial:D7:E2:87:4F:A0:79:E2:0C + + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication + Signature Algorithm: sha256WithRSAEncryption + e4:27:61:e2:0f:b6:a0:ca:9f:ce:e3:53:0b:44:ab:86:a1:e2: + 4d:88:e1:7d:2e:b0:aa:32:96:2b:3d:da:60:70:6a:c3:62:c5: + 76:f2:8f:0d:16:31:f2:ad:e5:2f:43:f3:cb:e4:fa:95:6c:20: + 81:33:1a:c7:5a:55:57:c9:ab:ca:66:45:30:58:00:db:e8:51: + c9:2c:a9:72:c1:18:f5:01:87:9f:73:20:85:6c:e5:6c:3f:c9: + 67:b4:f0:20:e5:ed:e2:4a:08:0b:af:68:43:e5:a9:c7:e1:39: + e8:b5:49:cb:47:4a:6d:e5:16:ae:88:92:13:85:8e:42:1e:0a: + eb:59:ed:a7:c1:9b:bc:4b:7b:99:f8:1d:f0:d7:1d:90:c9:cf: + 86:6a:d3:10:d0:36:e4:f5:b9:33:79:c7:a2:68:31:f7:bb:8d: + 1e:d6:33:79:bd:e7:0e:4f:4d:e9:2e:15:04:4f:6b:4b:2e:93: + 28:72:d1:0e:aa:ee:e6:ef:68:be:58:2b:cc:56:01:27:16:f9: + 34:8e:66:86:27:0a:b0:fb:32:56:a9:8a:d9:6f:b1:86:bd:ba: + fd:50:6c:d5:b2:54:e7:4e:c6:2d:19:88:a9:89:2c:ef:be:08: + 0d:2b:49:91:0b:09:42:64:06:a3:9d:d7:94:ed:e8:74:74:48: + 43:57:41:6f:e5:06:98:46:1d:c5:60:9c:69:f8:fb:fe:a6:01: + 4a:35:be:21:36:c2:a3:44:c8:c4:2c:21:09:f4:28:9a:ad:a0: + 97:1e:00:29:cc:0f:26:fa:59:21:25:c0:9e:fa:22:53:67:6d: + ab:a6:56:08:fd:37:1d:69:fe:ef:6f:29:89:1a:66:7b:c7:ff: + b1:34:f1:d6:be:21:81:e3:bc:4f:13:02:a7:4b:9d:13:05:46: + 40:88:4a:aa:db:fb:64:f8:6b:fb:5d:a0:b1:0c:1a:b8:4c:ab: + 6f:69:fe:0b:55:4e:b3:38:1f:91:0b:71:77:1e:11:39:54:9a: + 62:51:ea:6d:a8:5e:0d:4a:91:fb:d8:be:5d:93:e8:43:f3:4a: + 11:fb:31:cf:14:1a:1c:8d:31:1b:99:31:e0:2b:81:01:91:6f: + da:ba:cb:1f:51:21:55:29:3f:4c:71:e3:d0:29:41:de:a0:00: + da:07:ed:5e:c9:af:32:61:6d:55:f8:f5:2d:46:03:34:33:fb: + 2e:1e:aa:7c:fe:d2:30:4d:40:cc:ed:76:ec:f6:bd:ed:35:c8: + d8:b3:46:56:aa:2c:53:84:56:45:b0:a3:f6:35:66:93:da:8c: + 17:39:c1:29:7c:99:c5:0b:73:c1:f9:16:d0:57:fc:57:59:06: + af:39:9f:a9:51:35:0b:c7 +-----BEGIN CERTIFICATE----- +MIIExzCCAq+gAwIBAgICEAcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v +YmFyMCAXDTIzMDUxMDE1NTAxOFoYDzIyOTcwMjIyMTU1MDE4WjAfMR0wGwYDVQQD +DBRicm9rZXItbG9jYWxob3N0LVNBTjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAN7R2ruRsxbEsuiJMJ7BXgvP28TD2bGvQKULODYbFP4PIpzmWWoVW9v2 +9/OlAimUetIMZ62qY2J+/FgRKUi4PJGyc34Sa/LqNncPFZtGlc5zFY3I2ZdXA5Az +LX3z7uUBbdjG2qsHud0c4EvOat6o0uPBUm2DOgrw7c/3VmqHDnPjEoIrZavYqURb +Si+lkpQy8aHkrxgPDxhgzffQnQOf1+moYFS7O5oF2/04BDy0I0EWbHw72bbgL73L +YlUb6NCPQ3bvVYbPJcO8ruNGUIn3ca0GXijm9vB2J+p+G2dTOSYgGRiCsRFf6pHC +49P2Wsf9YaKS3n182m3ovzlSEDFgS+ECAwEAAaOCARcwggETMAkGA1UdEwQCMAAw +EQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVy +YXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBcHO6qFg7UEg+yybB46 +8PVZqmEoMDcGA1UdEQQwMC6CCWxvY2FsaG9zdIIbdW5yZXNvbHZhYmxlLWJyb2tl +ci1hZGRyZXNzhwR/AAABMEEGA1UdIwQ6MDiAFFcL6csj6L9HPlB6P0V+oRhDnRUn +oRWkEzARMQ8wDQYDVQQDDAZmb29iYXKCCQDX4odPoHniDDAOBgNVHQ8BAf8EBAMC +BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAOQnYeIP +tqDKn87jUwtEq4ah4k2I4X0usKoylis92mBwasNixXbyjw0WMfKt5S9D88vk+pVs +IIEzGsdaVVfJq8pmRTBYANvoUcksqXLBGPUBh59zIIVs5Ww/yWe08CDl7eJKCAuv +aEPlqcfhOei1SctHSm3lFq6IkhOFjkIeCutZ7afBm7xLe5n4HfDXHZDJz4Zq0xDQ +NuT1uTN5x6JoMfe7jR7WM3m95w5PTekuFQRPa0sukyhy0Q6q7ubvaL5YK8xWAScW ++TSOZoYnCrD7MlapitlvsYa9uv1QbNWyVOdOxi0ZiKmJLO++CA0rSZELCUJkBqOd +15Tt6HR0SENXQW/lBphGHcVgnGn4+/6mAUo1viE2wqNEyMQsIQn0KJqtoJceACnM +Dyb6WSElwJ76IlNnbaumVgj9Nx1p/u9vKYkaZnvH/7E08da+IYHjvE8TAqdLnRMF +RkCISqrb+2T4a/tdoLEMGrhMq29p/gtVTrM4H5ELcXceETlUmmJR6m2oXg1KkfvY +vl2T6EPzShH7Mc8UGhyNMRuZMeArgQGRb9q6yx9RIVUpP0xx49ApQd6gANoH7V7J +rzJhbVX49S1GAzQz+y4eqnz+0jBNQMztduz2ve01yNizRlaqLFOEVkWwo/Y1ZpPa +jBc5wSl8mcULc8H5FtBX/FdZBq85n6lRNQvH +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/newcerts/1008.pem b/tests/certificate-authority/newcerts/1008.pem new file mode 100644 index 0000000000000..85687bdfd30ba --- /dev/null +++ b/tests/certificate-authority/newcerts/1008.pem @@ -0,0 +1,110 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4104 (0x1008) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=foobar + Validity + Not Before: May 10 15:50:19 2023 GMT + Not After : Feb 22 15:50:19 2297 GMT + Subject: CN=proxy-localhost-SAN + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:cc:15:c9:85:06:43:47:bd:46:9f:4f:03:1a:e0: + 6e:94:13:4e:b0:30:ea:88:ca:3a:e4:39:92:12:c1: + 77:51:8c:0d:3c:b9:26:5c:2f:dc:fc:b1:5a:bf:0e: + 47:ff:09:60:30:79:8e:55:26:fe:d0:a1:ed:9f:6d: + 8a:6a:06:85:f0:d0:dc:94:a6:54:a1:a6:c9:3e:57: + d5:69:7d:e9:25:c1:ef:6b:77:e1:62:76:d8:e4:54: + 91:40:bc:0b:11:74:b8:30:bb:d4:02:77:d6:bd:d2: + d0:e7:ad:df:7d:98:96:74:42:ad:53:b3:88:c8:dc: + 1d:db:51:63:84:ee:7e:85:73:14:5e:d4:c8:f0:01: + 5f:67:52:ed:94:87:f7:d6:aa:28:8b:2c:84:98:8c: + b9:91:b5:38:99:80:5d:b3:d4:db:95:96:09:ef:1d: + a1:6f:86:c8:17:86:f7:0a:1e:72:3b:50:8c:53:e5: + ce:d4:8c:cf:cc:81:3d:46:55:ff:65:25:0b:36:31: + 31:a6:22:27:47:96:59:38:c1:cd:66:a6:9a:83:98: + dc:b8:2e:10:8d:ba:45:ae:aa:20:6e:e3:0b:bd:ec: + e6:63:b5:40:55:d4:fe:97:b1:f1:8d:9a:c0:a2:46: + 8e:a3:ed:a0:1b:ed:40:b0:00:a5:28:f9:da:03:bd: + c1:a9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Cert Type: + SSL Server + Netscape Comment: + OpenSSL Generated Server Certificate + X509v3 Subject Key Identifier: + C5:33:73:67:03:B7:51:08:F4:BD:D3:CD:4F:DC:CF:83:11:53:AD:39 + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + X509v3 Authority Key Identifier: + keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27 + DirName:/CN=foobar + serial:D7:E2:87:4F:A0:79:E2:0C + + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication + Signature Algorithm: sha256WithRSAEncryption + 43:ef:67:29:9a:0c:53:97:7c:fc:72:73:6c:8d:48:78:4e:ec: + e3:14:9d:d9:1e:83:4c:d6:f0:56:e9:c4:d8:de:f5:54:fb:a5: + 3b:ff:59:23:75:26:74:f0:86:90:d0:4d:41:25:03:87:e0:60: + a4:9b:33:3d:bd:1c:79:b8:db:86:1c:38:09:26:0d:80:3e:f9: + 1e:28:11:0d:3d:6b:1e:1a:7a:9a:fa:fc:18:22:7f:fd:46:55: + c2:2f:56:5c:5c:8a:45:f2:74:7a:e4:6c:d0:e0:ea:ec:74:b7: + 0d:a8:f3:ca:18:cf:a4:be:a0:e0:4a:32:ca:15:7e:5d:06:56: + b7:71:7c:e0:dc:19:fa:be:3e:94:84:20:be:96:34:61:0b:f0: + d1:d6:31:49:0b:b0:20:b8:f9:5c:49:08:13:9b:45:c0:6f:58: + 16:81:0b:0c:f8:66:38:58:83:d4:b0:bc:14:35:8d:e2:1d:d5: + 2d:ea:02:ae:42:e1:88:22:5a:b0:cf:e5:31:b1:cb:d3:e9:d2: + 5e:88:55:bd:62:ac:85:aa:4e:fc:18:6b:65:f9:9e:fc:93:27: + 0c:c6:29:aa:f0:64:6e:72:dc:d9:95:ae:38:ae:64:9e:c6:44: + 8a:0b:0f:0e:d4:69:7e:79:e0:46:d0:75:96:2a:1a:60:af:30: + 23:dc:d2:67:0d:08:2a:9d:58:29:09:1e:c8:08:d5:3a:88:2d: + 1a:dc:47:dc:5d:bd:0d:5c:54:f1:5d:5a:6d:0d:de:bc:18:67: + 2d:dd:1b:fe:8b:0e:03:19:b0:0f:f2:59:69:d0:7a:4f:a1:33: + 74:f7:22:ef:ff:90:e1:4b:8e:ac:13:00:6f:00:9b:55:83:d2: + 96:db:a8:81:c9:a9:8d:c6:a6:21:3d:14:d3:43:71:28:c6:ea: + 6d:2d:91:b9:58:bf:ec:18:75:c4:8c:10:43:88:60:08:c0:bb: + 9d:fb:90:80:1e:d5:a3:ea:e7:8a:16:f7:f4:d7:cb:35:93:03: + 55:e4:cc:58:31:1e:df:6e:e4:1b:6e:ad:3a:76:56:e5:8b:4e: + d9:71:af:11:92:a7:7a:e2:66:cc:d2:73:f3:ec:e8:3b:67:f0: + 6a:31:10:82:e8:c4:1e:ae:c3:54:a7:e2:42:86:fe:43:75:ad: + ef:83:d7:1c:2f:91:94:1c:57:9d:1c:43:94:b1:47:b2:6c:96: + fd:83:69:0f:6c:e2:18:9b:65:8e:71:08:01:b3:73:46:aa:3c: + 2e:07:14:cd:03:ae:dc:5a:51:da:c5:41:53:cc:f5:fc:c8:db: + 4e:76:27:99:9a:ec:40:68:07:d6:10:e1:f9:68:6b:5d:52:95: + 3d:01:f4:a7:40:11:61:0a +-----BEGIN CERTIFICATE----- +MIIEpzCCAo+gAwIBAgICEAgwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v +YmFyMCAXDTIzMDUxMDE1NTAxOVoYDzIyOTcwMjIyMTU1MDE5WjAeMRwwGgYDVQQD +DBNwcm94eS1sb2NhbGhvc3QtU0FOMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAzBXJhQZDR71Gn08DGuBulBNOsDDqiMo65DmSEsF3UYwNPLkmXC/c/LFa +vw5H/wlgMHmOVSb+0KHtn22KagaF8NDclKZUoabJPlfVaX3pJcHva3fhYnbY5FSR +QLwLEXS4MLvUAnfWvdLQ563ffZiWdEKtU7OIyNwd21FjhO5+hXMUXtTI8AFfZ1Lt +lIf31qooiyyEmIy5kbU4mYBds9TblZYJ7x2hb4bIF4b3Ch5yO1CMU+XO1IzPzIE9 +RlX/ZSULNjExpiInR5ZZOMHNZqaag5jcuC4QjbpFrqogbuMLvezmY7VAVdT+l7Hx +jZrAokaOo+2gG+1AsAClKPnaA73BqQIDAQABo4H5MIH2MAkGA1UdEwQCMAAwEQYJ +YIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRl +ZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFMUzc2cDt1EI9L3TzU/cz4MR +U605MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATBBBgNVHSMEOjA4gBRXC+nL +I+i/Rz5Qej9FfqEYQ50VJ6EVpBMwETEPMA0GA1UEAwwGZm9vYmFyggkA1+KHT6B5 +4gwwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3 +DQEBCwUAA4ICAQBD72cpmgxTl3z8cnNsjUh4TuzjFJ3ZHoNM1vBW6cTY3vVU+6U7 +/1kjdSZ08IaQ0E1BJQOH4GCkmzM9vRx5uNuGHDgJJg2APvkeKBENPWseGnqa+vwY +In/9RlXCL1ZcXIpF8nR65GzQ4OrsdLcNqPPKGM+kvqDgSjLKFX5dBla3cXzg3Bn6 +vj6UhCC+ljRhC/DR1jFJC7AguPlcSQgTm0XAb1gWgQsM+GY4WIPUsLwUNY3iHdUt +6gKuQuGIIlqwz+UxscvT6dJeiFW9YqyFqk78GGtl+Z78kycMximq8GRuctzZla44 +rmSexkSKCw8O1Gl+eeBG0HWWKhpgrzAj3NJnDQgqnVgpCR7ICNU6iC0a3EfcXb0N +XFTxXVptDd68GGct3Rv+iw4DGbAP8llp0HpPoTN09yLv/5DhS46sEwBvAJtVg9KW +26iByamNxqYhPRTTQ3EoxuptLZG5WL/sGHXEjBBDiGAIwLud+5CAHtWj6ueKFvf0 +18s1kwNV5MxYMR7fbuQbbq06dlbli07Zca8Rkqd64mbM0nPz7Og7Z/BqMRCC6MQe +rsNUp+JChv5Dda3vg9ccL5GUHFedHEOUsUeybJb9g2kPbOIYm2WOcQgBs3NGqjwu +BxTNA67cWlHaxUFTzPX8yNtOdieZmuxAaAfWEOH5aGtdUpU9AfSnQBFhCg== +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/openssl.cnf b/tests/certificate-authority/openssl.cnf index 9c8585edc9a1f..f7a23b3b33f6d 100644 --- a/tests/certificate-authority/openssl.cnf +++ b/tests/certificate-authority/openssl.cnf @@ -27,7 +27,7 @@ default_ca = CA_default [ CA_default ] # Directory and file locations. -dir = $ENV::CA_HOME +dir = . certs = $dir/certs crl_dir = $dir/crl new_certs_dir = $dir/newcerts @@ -92,12 +92,25 @@ authorityKeyIdentifier = keyid,issuer keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, emailProtection -[ server_cert ] +[ broker_cert ] # Extensions for server certificates (`man x509v3_config`). basicConstraints = CA:FALSE nsCertType = server nsComment = "OpenSSL Generated Server Certificate" subjectKeyIdentifier = hash +# The unresolvable address is used for SNI testing +subjectAltName = DNS:localhost, DNS:unresolvable-broker-address, IP:127.0.0.1 +authorityKeyIdentifier = keyid,issuer:always +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth + +[ proxy_cert ] +# Extensions for server certificates (`man x509v3_config`). +basicConstraints = CA:FALSE +nsCertType = server +nsComment = "OpenSSL Generated Server Certificate" +subjectKeyIdentifier = hash +subjectAltName = DNS:localhost, IP:127.0.0.1 authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = serverAuth diff --git a/tests/certificate-authority/serial b/tests/certificate-authority/serial index fb35a14c02716..6cb3869343bf8 100644 --- a/tests/certificate-authority/serial +++ b/tests/certificate-authority/serial @@ -1 +1 @@ -1007 +1009 diff --git a/tests/certificate-authority/server-keys/broker.cert.pem b/tests/certificate-authority/server-keys/broker.cert.pem index b5c7a5dc709a1..4237719f20ebd 100644 --- a/tests/certificate-authority/server-keys/broker.cert.pem +++ b/tests/certificate-authority/server-keys/broker.cert.pem @@ -1,27 +1,111 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4103 (0x1007) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=foobar + Validity + Not Before: May 10 15:50:18 2023 GMT + Not After : Feb 22 15:50:18 2297 GMT + Subject: CN=broker-localhost-SAN + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:de:d1:da:bb:91:b3:16:c4:b2:e8:89:30:9e:c1: + 5e:0b:cf:db:c4:c3:d9:b1:af:40:a5:0b:38:36:1b: + 14:fe:0f:22:9c:e6:59:6a:15:5b:db:f6:f7:f3:a5: + 02:29:94:7a:d2:0c:67:ad:aa:63:62:7e:fc:58:11: + 29:48:b8:3c:91:b2:73:7e:12:6b:f2:ea:36:77:0f: + 15:9b:46:95:ce:73:15:8d:c8:d9:97:57:03:90:33: + 2d:7d:f3:ee:e5:01:6d:d8:c6:da:ab:07:b9:dd:1c: + e0:4b:ce:6a:de:a8:d2:e3:c1:52:6d:83:3a:0a:f0: + ed:cf:f7:56:6a:87:0e:73:e3:12:82:2b:65:ab:d8: + a9:44:5b:4a:2f:a5:92:94:32:f1:a1:e4:af:18:0f: + 0f:18:60:cd:f7:d0:9d:03:9f:d7:e9:a8:60:54:bb: + 3b:9a:05:db:fd:38:04:3c:b4:23:41:16:6c:7c:3b: + d9:b6:e0:2f:bd:cb:62:55:1b:e8:d0:8f:43:76:ef: + 55:86:cf:25:c3:bc:ae:e3:46:50:89:f7:71:ad:06: + 5e:28:e6:f6:f0:76:27:ea:7e:1b:67:53:39:26:20: + 19:18:82:b1:11:5f:ea:91:c2:e3:d3:f6:5a:c7:fd: + 61:a2:92:de:7d:7c:da:6d:e8:bf:39:52:10:31:60: + 4b:e1 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Cert Type: + SSL Server + Netscape Comment: + OpenSSL Generated Server Certificate + X509v3 Subject Key Identifier: + 17:07:3B:AA:85:83:B5:04:83:EC:B2:6C:1E:3A:F0:F5:59:AA:61:28 + X509v3 Subject Alternative Name: + DNS:localhost, DNS:unresolvable-broker-address, IP Address:127.0.0.1 + X509v3 Authority Key Identifier: + keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27 + DirName:/CN=foobar + serial:D7:E2:87:4F:A0:79:E2:0C + + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication + Signature Algorithm: sha256WithRSAEncryption + e4:27:61:e2:0f:b6:a0:ca:9f:ce:e3:53:0b:44:ab:86:a1:e2: + 4d:88:e1:7d:2e:b0:aa:32:96:2b:3d:da:60:70:6a:c3:62:c5: + 76:f2:8f:0d:16:31:f2:ad:e5:2f:43:f3:cb:e4:fa:95:6c:20: + 81:33:1a:c7:5a:55:57:c9:ab:ca:66:45:30:58:00:db:e8:51: + c9:2c:a9:72:c1:18:f5:01:87:9f:73:20:85:6c:e5:6c:3f:c9: + 67:b4:f0:20:e5:ed:e2:4a:08:0b:af:68:43:e5:a9:c7:e1:39: + e8:b5:49:cb:47:4a:6d:e5:16:ae:88:92:13:85:8e:42:1e:0a: + eb:59:ed:a7:c1:9b:bc:4b:7b:99:f8:1d:f0:d7:1d:90:c9:cf: + 86:6a:d3:10:d0:36:e4:f5:b9:33:79:c7:a2:68:31:f7:bb:8d: + 1e:d6:33:79:bd:e7:0e:4f:4d:e9:2e:15:04:4f:6b:4b:2e:93: + 28:72:d1:0e:aa:ee:e6:ef:68:be:58:2b:cc:56:01:27:16:f9: + 34:8e:66:86:27:0a:b0:fb:32:56:a9:8a:d9:6f:b1:86:bd:ba: + fd:50:6c:d5:b2:54:e7:4e:c6:2d:19:88:a9:89:2c:ef:be:08: + 0d:2b:49:91:0b:09:42:64:06:a3:9d:d7:94:ed:e8:74:74:48: + 43:57:41:6f:e5:06:98:46:1d:c5:60:9c:69:f8:fb:fe:a6:01: + 4a:35:be:21:36:c2:a3:44:c8:c4:2c:21:09:f4:28:9a:ad:a0: + 97:1e:00:29:cc:0f:26:fa:59:21:25:c0:9e:fa:22:53:67:6d: + ab:a6:56:08:fd:37:1d:69:fe:ef:6f:29:89:1a:66:7b:c7:ff: + b1:34:f1:d6:be:21:81:e3:bc:4f:13:02:a7:4b:9d:13:05:46: + 40:88:4a:aa:db:fb:64:f8:6b:fb:5d:a0:b1:0c:1a:b8:4c:ab: + 6f:69:fe:0b:55:4e:b3:38:1f:91:0b:71:77:1e:11:39:54:9a: + 62:51:ea:6d:a8:5e:0d:4a:91:fb:d8:be:5d:93:e8:43:f3:4a: + 11:fb:31:cf:14:1a:1c:8d:31:1b:99:31:e0:2b:81:01:91:6f: + da:ba:cb:1f:51:21:55:29:3f:4c:71:e3:d0:29:41:de:a0:00: + da:07:ed:5e:c9:af:32:61:6d:55:f8:f5:2d:46:03:34:33:fb: + 2e:1e:aa:7c:fe:d2:30:4d:40:cc:ed:76:ec:f6:bd:ed:35:c8: + d8:b3:46:56:aa:2c:53:84:56:45:b0:a3:f6:35:66:93:da:8c: + 17:39:c1:29:7c:99:c5:0b:73:c1:f9:16:d0:57:fc:57:59:06: + af:39:9f:a9:51:35:0b:c7 -----BEGIN CERTIFICATE----- -MIIEkDCCAnigAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTE4MDYyMjA4NTUzMloYDzIyOTIwNDA2MDg1NTMyWjAjMSEwHwYDVQQD -DBhicm9rZXIucHVsc2FyLmFwYWNoZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB -DwAwggEKAoIBAQDQouKhZah4hMCqmg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63 -EvNjEK4L/ZWSEV45L/wc6YV14RmM6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0O -OQXVicqZLOc6igZQhRg8ANDYdTJUTF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01 -+ARO9OotMJtBY+vIU5bV6JydfgkhQH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+X -aqTN3/tF8+MBcFB3G04s1qc2CJPJM3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00s -bxa4FGbKgfDobbkJ+GgblWCkAcLN95sKTqtHAgMBAAGjgd0wgdowCQYDVR0TBAIw -ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu -ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUaxFvJrkEGqk8azTA -DyVyTyTbJAIwQQYDVR0jBDowOIAUVwvpyyPov0c+UHo/RX6hGEOdFSehFaQTMBEx -DzANBgNVBAMMBmZvb2JhcoIJANfih0+geeIMMA4GA1UdDwEB/wQEAwIFoDATBgNV -HSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA35QDGclHzQtHs3yQ -ZzNOSKisg5srTiIoQgRzfHrXfkthNFCnBzhKjBxqk3EIasVtvyGuk0ThneC1ai3y -ZK3BivnMZfm1SfyvieFoqWetsxohWfcpOSVkpvO37P6v/NmmaTIGkBN3gxKCx0QN -zqApLQyNTM++X3wxetYH/afAGUrRmBGWZuJheQpB9yZ+FB6BRp8YuYIYBzANJyW9 -spvXW03TpqX2AIoRBoGMLzK72vbhAbLWiCIfEYREhbZVRkP+yvD338cWrILlOEur -x/n8L/FTmbf7mXzHg4xaQ3zg/5+0OCPMDPUBE4xWDBAbZ82hgOcTqfVjwoPgo2V0 -fbbx6redq44J3Vn5d9Xhi59fkpqEjHpX4xebr5iMikZsNTJMeLh0h3uf7DstuO9d -mfnF5j+yDXCKb9XzCsTSvGCN+spmUh6RfSrbkw8/LrRvBUpKVEM0GfKSnaFpOaSS -efM4UEi72FRjszzHEkdvpiLhYvihINLJmDXszhc3fCi42be/DGmUhuhTZWynOPmp -0N0V/8/sGT5gh4fGEtGzS/8xEvZwO9uDlccJiG8Pi+aO0/K9urB9nppd/xKWXv3C -cib/QrW0Qow4TADWC1fnGYCpFzzaZ2esPL2MvzOYXnW4/AbEqmb6Weatluai64ZK -3N2cGJWRyvpvvmbP2hKCa4eLgEc= +MIIExzCCAq+gAwIBAgICEAcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v +YmFyMCAXDTIzMDUxMDE1NTAxOFoYDzIyOTcwMjIyMTU1MDE4WjAfMR0wGwYDVQQD +DBRicm9rZXItbG9jYWxob3N0LVNBTjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAN7R2ruRsxbEsuiJMJ7BXgvP28TD2bGvQKULODYbFP4PIpzmWWoVW9v2 +9/OlAimUetIMZ62qY2J+/FgRKUi4PJGyc34Sa/LqNncPFZtGlc5zFY3I2ZdXA5Az +LX3z7uUBbdjG2qsHud0c4EvOat6o0uPBUm2DOgrw7c/3VmqHDnPjEoIrZavYqURb +Si+lkpQy8aHkrxgPDxhgzffQnQOf1+moYFS7O5oF2/04BDy0I0EWbHw72bbgL73L +YlUb6NCPQ3bvVYbPJcO8ruNGUIn3ca0GXijm9vB2J+p+G2dTOSYgGRiCsRFf6pHC +49P2Wsf9YaKS3n182m3ovzlSEDFgS+ECAwEAAaOCARcwggETMAkGA1UdEwQCMAAw +EQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVy +YXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBcHO6qFg7UEg+yybB46 +8PVZqmEoMDcGA1UdEQQwMC6CCWxvY2FsaG9zdIIbdW5yZXNvbHZhYmxlLWJyb2tl +ci1hZGRyZXNzhwR/AAABMEEGA1UdIwQ6MDiAFFcL6csj6L9HPlB6P0V+oRhDnRUn +oRWkEzARMQ8wDQYDVQQDDAZmb29iYXKCCQDX4odPoHniDDAOBgNVHQ8BAf8EBAMC +BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAOQnYeIP +tqDKn87jUwtEq4ah4k2I4X0usKoylis92mBwasNixXbyjw0WMfKt5S9D88vk+pVs +IIEzGsdaVVfJq8pmRTBYANvoUcksqXLBGPUBh59zIIVs5Ww/yWe08CDl7eJKCAuv +aEPlqcfhOei1SctHSm3lFq6IkhOFjkIeCutZ7afBm7xLe5n4HfDXHZDJz4Zq0xDQ +NuT1uTN5x6JoMfe7jR7WM3m95w5PTekuFQRPa0sukyhy0Q6q7ubvaL5YK8xWAScW ++TSOZoYnCrD7MlapitlvsYa9uv1QbNWyVOdOxi0ZiKmJLO++CA0rSZELCUJkBqOd +15Tt6HR0SENXQW/lBphGHcVgnGn4+/6mAUo1viE2wqNEyMQsIQn0KJqtoJceACnM +Dyb6WSElwJ76IlNnbaumVgj9Nx1p/u9vKYkaZnvH/7E08da+IYHjvE8TAqdLnRMF +RkCISqrb+2T4a/tdoLEMGrhMq29p/gtVTrM4H5ELcXceETlUmmJR6m2oXg1KkfvY +vl2T6EPzShH7Mc8UGhyNMRuZMeArgQGRb9q6yx9RIVUpP0xx49ApQd6gANoH7V7J +rzJhbVX49S1GAzQz+y4eqnz+0jBNQMztduz2ve01yNizRlaqLFOEVkWwo/Y1ZpPa +jBc5wSl8mcULc8H5FtBX/FdZBq85n6lRNQvH -----END CERTIFICATE----- diff --git a/tests/certificate-authority/server-keys/broker.csr.pem b/tests/certificate-authority/server-keys/broker.csr.pem index d2342595eb254..9d28c52be7909 100644 --- a/tests/certificate-authority/server-keys/broker.csr.pem +++ b/tests/certificate-authority/server-keys/broker.csr.pem @@ -1,15 +1,15 @@ -----BEGIN CERTIFICATE REQUEST----- -MIICaDCCAVACAQAwIzEhMB8GA1UEAwwYYnJva2VyLnB1bHNhci5hcGFjaGUub3Jn -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0KLioWWoeITAqpoOGkuU -YUBv2NYAPsj/QxhfZpcN+bX4X1rOtxLzYxCuC/2VkhFeOS/8HOmFdeEZjOmydFf9 -L16OMZiEm6qbTQ/9pESLKZC2UORdDjkF1YnKmSznOooGUIUYPADQ2HUyVExeuQ6m -rl+Dibd2DIRdhUre40LZY9OiEAWtNfgETvTqLTCbQWPryFOW1eicnX4JIUB/ayAy -OwDHmOY9NoBpCSa3pX5vlqDrFHs/l2qkzd/7RfPjAXBQdxtOLNanNgiTyTN2Bsb8 -RLR6kxiNeLfSfKeTu1/SR6XGPF9NLG8WuBRmyoHw6G25CfhoG5VgpAHCzfebCk6r -RwIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAHVVGKnfqBDmu+e5MWK9i0ja/JFv -dhST705gdKDOPc7MXDVr+zJZKgvnDtzDrWTe7Zk0p7xQf3kc773eYCdlznX+J1Fw -EfIHXQTBZRZxmHnYqc012i5tshvEOS0o61ZEgxz8hxGLwGlRaIcy+qt927fscpQ5 -7VEnlxzD4YeHwryIXH5hOr/J1OmlL58Fxwh2NJfso7ErRuHW44XK4qdwWCQs/nVN -EQyV6RCbaiRq9Ks4j3FwtqmfgzMB1+T3L+CiuhPol2/rZwD3o5j7SP8ZGxC15Tzi -wHG71H0wp1CY+tkAcvm2zmoHR9z1SD84raZLYJVRgUio7myW/DVBqPxCSvU= +MIICZDCCAUwCAQAwHzEdMBsGA1UEAwwUYnJva2VyLWxvY2FsaG9zdC1TQU4wggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDe0dq7kbMWxLLoiTCewV4Lz9vE +w9mxr0ClCzg2GxT+DyKc5llqFVvb9vfzpQIplHrSDGetqmNifvxYESlIuDyRsnN+ +Emvy6jZ3DxWbRpXOcxWNyNmXVwOQMy198+7lAW3YxtqrB7ndHOBLzmreqNLjwVJt +gzoK8O3P91Zqhw5z4xKCK2Wr2KlEW0ovpZKUMvGh5K8YDw8YYM330J0Dn9fpqGBU +uzuaBdv9OAQ8tCNBFmx8O9m24C+9y2JVG+jQj0N271WGzyXDvK7jRlCJ93GtBl4o +5vbwdifqfhtnUzkmIBkYgrERX+qRwuPT9lrH/WGikt59fNpt6L85UhAxYEvhAgMB +AAGgADANBgkqhkiG9w0BAQsFAAOCAQEAcLkzWe6zgkVpk4OUlCv1HUqmntiBHdmh +24v3OKhQjyMs+m/srBe6r+lBdZLnbSH5fq7eToEwRMt/vNirsXCaxGGVOUnThStt +Z/rR45bpJl1TomXwEG9xHq7yOaHVfxkNZgaHCu6BZ1ZjYgfqWffFf9hcqAJ3xZm0 +XMPfJs9i/TRHCsdUge1BHZrxD/fzriWM7k89XktY+0zSqGfdhOXE0FO30bnC4mJG +vZXY5reyIuRlFBpnDUtuYBaSfbUYguaSUYIoHOUsQrmMSqPLJUyY1zNm1t5f1jIx +ZaK8NEIq2AHHqEx7I/7+lVRRWa9IjdqWIT9KD5TraOML9oS74sto2w== -----END CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/server-keys/broker.key-pk8.pem b/tests/certificate-authority/server-keys/broker.key-pk8.pem index 2b51d015b8ace..dd9fa523e8ede 100644 --- a/tests/certificate-authority/server-keys/broker.key-pk8.pem +++ b/tests/certificate-authority/server-keys/broker.key-pk8.pem @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDQouKhZah4hMCq -mg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63EvNjEK4L/ZWSEV45L/wc6YV14RmM -6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0OOQXVicqZLOc6igZQhRg8ANDYdTJU -TF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01+ARO9OotMJtBY+vIU5bV6Jydfgkh -QH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+XaqTN3/tF8+MBcFB3G04s1qc2CJPJ -M3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00sbxa4FGbKgfDobbkJ+GgblWCkAcLN -95sKTqtHAgMBAAECggEBALE1eMtfnk3nbAI74bih84D7C0Ug14p8jJv/qqBnsx4j -WrgbWDMVrJa7Rym2FQHBMMfgIwKnso0iSeJvaPz683j1lk833YKe0VQOPgD1m0IN -wV1J6mQ3OOZcKDIcerY1IBHqSmBEzR7dxIbnaxlCAX9gb0hdBK6zCwA5TMG5OQ5Y -3cGOmevK5i2PiejhpruA8h7E48P1ATaGHUZif9YD724oi6AcilQ8H/DlOjZTvlmK -r4aJ30f72NwGM8Ecet5CE2wyflAGtY0k+nChYkPRfy54u64Z/T9B53AvneFaj8jv -yFepZgRTs2cWhEl0KQGuBHQ4+IeOfMt2LebhvjWW8YkCgYEA7BXVsnqPHKRDd8wP -eNkolY4Fjdq4wu9ad+DaFiZcJuv7ugr+Kplltq6e4aU36zEdBYdPp/6KM/HGE/Xj -bo0CELNUKs/Ny9H/UJc8DDbVEmoF3XGiIbKKq1T8NTXTETFnwrGkBFD8nl7YTsOF -M4FZmSok0MhhkpEULAqxBS6YpQsCgYEA4jxM1egTVSWjTreg2UdYo2507jKa7maP -PRtoPsNJzWNbOpfj26l3/8pd6oYKWck6se6RxIUxUrk3ywhNJIIOvWEC7TaOH1c9 -T4NQNcweqBW9+A1x5gyzT14gDaBfl45gs82vI+kcpVv/w2N3HZOQZX3yAUqWpfw2 -yw1uQDXtgDUCgYEAiYPWbBXTkp1j5z3nrT7g0uxc89n5USLWkYlZvxktCEbg4+dP -UUT06EoipdD1F3wOKZA9p98uZT9pX2sUxOpBz7SFTEKq3xQ9IZZWFc9CoW08aVat -V++FsnLYTa5CeXtLsy6CGTmLTDx2xrpAtlWb+QmBVFPD8fmrxFOd9STFKS0CgYAt -6ztVN3OlFqyc75yQPXD6SxMkvdTAisSMDKIOCylRrNb5f5baIP2gR3zkeyxiqPtm -3htsHfSy67EtXpP50wQW4Dft2eLi7ZweJXMEWFfomfEjBeeWYAGNHHe5DFIauuVZ -2WexDEGqNpAlIm0s7aSjVPrn1DHbouOkNyenlMqN+QKBgQDVYVhk9widShSnCmUA -G30moXDgj3eRqCf5T7NEr9GXD1QBD/rQSPh5agnDV7IYLpV7/wkYLI7l9x7mDwu+ -I9mRXkyAmTVEctLTdXQHt0jdJa5SfUaVEDUzQbr0fUjkmythTvqZ809+d3ELPeLI -5qJ7jxgksHWji4lYfL4r4J6Zaw== +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe0dq7kbMWxLLo +iTCewV4Lz9vEw9mxr0ClCzg2GxT+DyKc5llqFVvb9vfzpQIplHrSDGetqmNifvxY +ESlIuDyRsnN+Emvy6jZ3DxWbRpXOcxWNyNmXVwOQMy198+7lAW3YxtqrB7ndHOBL +zmreqNLjwVJtgzoK8O3P91Zqhw5z4xKCK2Wr2KlEW0ovpZKUMvGh5K8YDw8YYM33 +0J0Dn9fpqGBUuzuaBdv9OAQ8tCNBFmx8O9m24C+9y2JVG+jQj0N271WGzyXDvK7j +RlCJ93GtBl4o5vbwdifqfhtnUzkmIBkYgrERX+qRwuPT9lrH/WGikt59fNpt6L85 +UhAxYEvhAgMBAAECggEAbQ4xDFTHXoFvPzjGPy1NJmLZoXhp9/lanmzbWj/vClnG +Cx0C7lT93K8HtIwyfr9ZTa0coXcfpXmZcFEV762cl4LL3AyQIRhZB/SuEo19jMnu +5rJDLTs9Vzp1LYxShGsqpErPg54IbhxP+0pQLCJc9XQNL+RmaCx7eKoJ9aGchULY +BdbCRctYhHaq/AC2qNYZF41Ys9zdNN0/2NnRqfgaaAj9hUzu3LlaJX1TWHbgikLo +QdmRNmTMMxfLl2kyoweDEC6MdSKCbcdDyM46Va3yY3KTuAdjP6x1ALzAarBDj6zb +a0Vp7g80OcKjmLYt8rFImjwb7D9EanOy2GkK2CwhAQKBgQD8v4gOTeNYC4Xo6Rti +psv+QCuH8hiLdee4KFdzqthlELDfhncDKXfwcZI3PME/aBWvvAn+rokl/UfCm5nQ +fwXW3MYyNpmk0HJjWAQcexsdHCM9I0CgEp8uInn+8EFqlYVh2ltoQLhGuIwOqqPk +3cQV8ImW+CmqmWYzbSZw97OqqQKBgQDhr7+oOf+sly0MVf5WixHTURJAU6p2VyTt +aNsNiuLN/W3Vax9Ql9HEm3RPx2SxFEIxllPcwb1Vms/ONmvT1t3xWGp7TIOX/0/m +uhNIG2/Bcr47NgWjhiV4zE/TfawkcP7/MujUxl2/zm7RHLrU2bmi8rV+PGqlVE0w +v7iKj4bSeQKBgHkI34a6Fdzb58yZlNuxNI8U+8OmU8q1M7ok13w0nFwJminwop2J +Bj7GpFZ/aauLlJcLXV3xBwyCNhMjoI0PxyQVpXP2Ya1jhOO+Cnn5GgrepqFoeFIv +mLrnF7TWKP15jN5HSu6pz5VOWwPLA6Fd8cDv53O8c3eW7jJCWt5OQGPBAoGALwbI +EO3E8NmvcVqZ3L6twDKscur8IhyWfUHUI0ZFbFbahBYGOGzqMOWTnuwVdzCZemuw +nddg9G2Fz5pXbZTgOmIKDhcrdIimxZUQX34YE18tdHkVQ7W4KSuplpAhRpalC9g3 +295ZupXxUXGDHMchf2rDlsJQFpMyYm4Qrg6qMUECgYBhBG/iLvE6/Z1oh6eky6WU +Dx7nL+FDDu3JbNAWs6RMDs9fE+7iDEj8MxZL0PUnVMsQvf43Ew1boyIGwlNtdRZ5 +dLaKFAgJ8YbqIptnFfGQ/8vKhK+x4FWzEd3kuRzQZPVRVV5Op6bRp1Kb8NMPCQXb +72qifYUaI7uirULNSit9wg== -----END PRIVATE KEY----- diff --git a/tests/certificate-authority/server-keys/broker.key.pem b/tests/certificate-authority/server-keys/broker.key.pem index dc22667ab47a5..5c20238c7b9c9 100644 --- a/tests/certificate-authority/server-keys/broker.key.pem +++ b/tests/certificate-authority/server-keys/broker.key.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEA0KLioWWoeITAqpoOGkuUYUBv2NYAPsj/QxhfZpcN+bX4X1rO -txLzYxCuC/2VkhFeOS/8HOmFdeEZjOmydFf9L16OMZiEm6qbTQ/9pESLKZC2UORd -DjkF1YnKmSznOooGUIUYPADQ2HUyVExeuQ6mrl+Dibd2DIRdhUre40LZY9OiEAWt -NfgETvTqLTCbQWPryFOW1eicnX4JIUB/ayAyOwDHmOY9NoBpCSa3pX5vlqDrFHs/ -l2qkzd/7RfPjAXBQdxtOLNanNgiTyTN2Bsb8RLR6kxiNeLfSfKeTu1/SR6XGPF9N -LG8WuBRmyoHw6G25CfhoG5VgpAHCzfebCk6rRwIDAQABAoIBAQCxNXjLX55N52wC -O+G4ofOA+wtFINeKfIyb/6qgZ7MeI1q4G1gzFayWu0cpthUBwTDH4CMCp7KNIkni -b2j8+vN49ZZPN92CntFUDj4A9ZtCDcFdSepkNzjmXCgyHHq2NSAR6kpgRM0e3cSG -52sZQgF/YG9IXQSuswsAOUzBuTkOWN3BjpnryuYtj4no4aa7gPIexOPD9QE2hh1G -Yn/WA+9uKIugHIpUPB/w5To2U75Ziq+Gid9H+9jcBjPBHHreQhNsMn5QBrWNJPpw -oWJD0X8ueLuuGf0/QedwL53hWo/I78hXqWYEU7NnFoRJdCkBrgR0OPiHjnzLdi3m -4b41lvGJAoGBAOwV1bJ6jxykQ3fMD3jZKJWOBY3auMLvWnfg2hYmXCbr+7oK/iqZ -ZbaunuGlN+sxHQWHT6f+ijPxxhP1426NAhCzVCrPzcvR/1CXPAw21RJqBd1xoiGy -iqtU/DU10xExZ8KxpARQ/J5e2E7DhTOBWZkqJNDIYZKRFCwKsQUumKULAoGBAOI8 -TNXoE1Ulo063oNlHWKNudO4ymu5mjz0baD7DSc1jWzqX49upd//KXeqGClnJOrHu -kcSFMVK5N8sITSSCDr1hAu02jh9XPU+DUDXMHqgVvfgNceYMs09eIA2gX5eOYLPN -ryPpHKVb/8Njdx2TkGV98gFKlqX8NssNbkA17YA1AoGBAImD1mwV05KdY+c9560+ -4NLsXPPZ+VEi1pGJWb8ZLQhG4OPnT1FE9OhKIqXQ9Rd8DimQPaffLmU/aV9rFMTq -Qc+0hUxCqt8UPSGWVhXPQqFtPGlWrVfvhbJy2E2uQnl7S7Mughk5i0w8dsa6QLZV -m/kJgVRTw/H5q8RTnfUkxSktAoGALes7VTdzpRasnO+ckD1w+ksTJL3UwIrEjAyi -DgspUazW+X+W2iD9oEd85HssYqj7Zt4bbB30suuxLV6T+dMEFuA37dni4u2cHiVz -BFhX6JnxIwXnlmABjRx3uQxSGrrlWdlnsQxBqjaQJSJtLO2ko1T659Qx26LjpDcn -p5TKjfkCgYEA1WFYZPcInUoUpwplABt9JqFw4I93kagn+U+zRK/Rlw9UAQ/60Ej4 -eWoJw1eyGC6Ve/8JGCyO5fce5g8LviPZkV5MgJk1RHLS03V0B7dI3SWuUn1GlRA1 -M0G69H1I5JsrYU76mfNPfndxCz3iyOaie48YJLB1o4uJWHy+K+CemWs= +MIIEogIBAAKCAQEA3tHau5GzFsSy6IkwnsFeC8/bxMPZsa9ApQs4NhsU/g8inOZZ +ahVb2/b386UCKZR60gxnrapjYn78WBEpSLg8kbJzfhJr8uo2dw8Vm0aVznMVjcjZ +l1cDkDMtffPu5QFt2Mbaqwe53RzgS85q3qjS48FSbYM6CvDtz/dWaocOc+MSgitl +q9ipRFtKL6WSlDLxoeSvGA8PGGDN99CdA5/X6ahgVLs7mgXb/TgEPLQjQRZsfDvZ +tuAvvctiVRvo0I9Ddu9Vhs8lw7yu40ZQifdxrQZeKOb28HYn6n4bZ1M5JiAZGIKx +EV/qkcLj0/Zax/1hopLefXzabei/OVIQMWBL4QIDAQABAoIBAG0OMQxUx16Bbz84 +xj8tTSZi2aF4aff5Wp5s21o/7wpZxgsdAu5U/dyvB7SMMn6/WU2tHKF3H6V5mXBR +Fe+tnJeCy9wMkCEYWQf0rhKNfYzJ7uayQy07PVc6dS2MUoRrKqRKz4OeCG4cT/tK +UCwiXPV0DS/kZmgse3iqCfWhnIVC2AXWwkXLWIR2qvwAtqjWGReNWLPc3TTdP9jZ +0an4GmgI/YVM7ty5WiV9U1h24IpC6EHZkTZkzDMXy5dpMqMHgxAujHUigm3HQ8jO +OlWt8mNyk7gHYz+sdQC8wGqwQ4+s22tFae4PNDnCo5i2LfKxSJo8G+w/RGpzsthp +CtgsIQECgYEA/L+IDk3jWAuF6OkbYqbL/kArh/IYi3XnuChXc6rYZRCw34Z3Ayl3 +8HGSNzzBP2gVr7wJ/q6JJf1HwpuZ0H8F1tzGMjaZpNByY1gEHHsbHRwjPSNAoBKf +LiJ5/vBBapWFYdpbaEC4RriMDqqj5N3EFfCJlvgpqplmM20mcPezqqkCgYEA4a+/ +qDn/rJctDFX+VosR01ESQFOqdlck7WjbDYrizf1t1WsfUJfRxJt0T8dksRRCMZZT +3MG9VZrPzjZr09bd8Vhqe0yDl/9P5roTSBtvwXK+OzYFo4YleMxP032sJHD+/zLo +1MZdv85u0Ry61Nm5ovK1fjxqpVRNML+4io+G0nkCgYB5CN+GuhXc2+fMmZTbsTSP +FPvDplPKtTO6JNd8NJxcCZop8KKdiQY+xqRWf2mri5SXC11d8QcMgjYTI6CND8ck +FaVz9mGtY4Tjvgp5+RoK3qahaHhSL5i65xe01ij9eYzeR0ruqc+VTlsDywOhXfHA +7+dzvHN3lu4yQlreTkBjwQKBgC8GyBDtxPDZr3Famdy+rcAyrHLq/CIcln1B1CNG +RWxW2oQWBjhs6jDlk57sFXcwmXprsJ3XYPRthc+aV22U4DpiCg4XK3SIpsWVEF9+ +GBNfLXR5FUO1uCkrqZaQIUaWpQvYN9veWbqV8VFxgxzHIX9qw5bCUBaTMmJuEK4O +qjFBAoGAYQRv4i7xOv2daIenpMullA8e5y/hQw7tyWzQFrOkTA7PXxPu4gxI/DMW +S9D1J1TLEL3+NxMNW6MiBsJTbXUWeXS2ihQICfGG6iKbZxXxkP/LyoSvseBVsxHd +5Lkc0GT1UVVeTqem0adSm/DTDwkF2+9qon2FGiO7oq1CzUorfcI= -----END RSA PRIVATE KEY----- diff --git a/tests/certificate-authority/server-keys/proxy.cert.pem b/tests/certificate-authority/server-keys/proxy.cert.pem index 02caee5826346..85687bdfd30ba 100644 --- a/tests/certificate-authority/server-keys/proxy.cert.pem +++ b/tests/certificate-authority/server-keys/proxy.cert.pem @@ -1,27 +1,110 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4104 (0x1008) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=foobar + Validity + Not Before: May 10 15:50:19 2023 GMT + Not After : Feb 22 15:50:19 2297 GMT + Subject: CN=proxy-localhost-SAN + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:cc:15:c9:85:06:43:47:bd:46:9f:4f:03:1a:e0: + 6e:94:13:4e:b0:30:ea:88:ca:3a:e4:39:92:12:c1: + 77:51:8c:0d:3c:b9:26:5c:2f:dc:fc:b1:5a:bf:0e: + 47:ff:09:60:30:79:8e:55:26:fe:d0:a1:ed:9f:6d: + 8a:6a:06:85:f0:d0:dc:94:a6:54:a1:a6:c9:3e:57: + d5:69:7d:e9:25:c1:ef:6b:77:e1:62:76:d8:e4:54: + 91:40:bc:0b:11:74:b8:30:bb:d4:02:77:d6:bd:d2: + d0:e7:ad:df:7d:98:96:74:42:ad:53:b3:88:c8:dc: + 1d:db:51:63:84:ee:7e:85:73:14:5e:d4:c8:f0:01: + 5f:67:52:ed:94:87:f7:d6:aa:28:8b:2c:84:98:8c: + b9:91:b5:38:99:80:5d:b3:d4:db:95:96:09:ef:1d: + a1:6f:86:c8:17:86:f7:0a:1e:72:3b:50:8c:53:e5: + ce:d4:8c:cf:cc:81:3d:46:55:ff:65:25:0b:36:31: + 31:a6:22:27:47:96:59:38:c1:cd:66:a6:9a:83:98: + dc:b8:2e:10:8d:ba:45:ae:aa:20:6e:e3:0b:bd:ec: + e6:63:b5:40:55:d4:fe:97:b1:f1:8d:9a:c0:a2:46: + 8e:a3:ed:a0:1b:ed:40:b0:00:a5:28:f9:da:03:bd: + c1:a9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Cert Type: + SSL Server + Netscape Comment: + OpenSSL Generated Server Certificate + X509v3 Subject Key Identifier: + C5:33:73:67:03:B7:51:08:F4:BD:D3:CD:4F:DC:CF:83:11:53:AD:39 + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + X509v3 Authority Key Identifier: + keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27 + DirName:/CN=foobar + serial:D7:E2:87:4F:A0:79:E2:0C + + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication + Signature Algorithm: sha256WithRSAEncryption + 43:ef:67:29:9a:0c:53:97:7c:fc:72:73:6c:8d:48:78:4e:ec: + e3:14:9d:d9:1e:83:4c:d6:f0:56:e9:c4:d8:de:f5:54:fb:a5: + 3b:ff:59:23:75:26:74:f0:86:90:d0:4d:41:25:03:87:e0:60: + a4:9b:33:3d:bd:1c:79:b8:db:86:1c:38:09:26:0d:80:3e:f9: + 1e:28:11:0d:3d:6b:1e:1a:7a:9a:fa:fc:18:22:7f:fd:46:55: + c2:2f:56:5c:5c:8a:45:f2:74:7a:e4:6c:d0:e0:ea:ec:74:b7: + 0d:a8:f3:ca:18:cf:a4:be:a0:e0:4a:32:ca:15:7e:5d:06:56: + b7:71:7c:e0:dc:19:fa:be:3e:94:84:20:be:96:34:61:0b:f0: + d1:d6:31:49:0b:b0:20:b8:f9:5c:49:08:13:9b:45:c0:6f:58: + 16:81:0b:0c:f8:66:38:58:83:d4:b0:bc:14:35:8d:e2:1d:d5: + 2d:ea:02:ae:42:e1:88:22:5a:b0:cf:e5:31:b1:cb:d3:e9:d2: + 5e:88:55:bd:62:ac:85:aa:4e:fc:18:6b:65:f9:9e:fc:93:27: + 0c:c6:29:aa:f0:64:6e:72:dc:d9:95:ae:38:ae:64:9e:c6:44: + 8a:0b:0f:0e:d4:69:7e:79:e0:46:d0:75:96:2a:1a:60:af:30: + 23:dc:d2:67:0d:08:2a:9d:58:29:09:1e:c8:08:d5:3a:88:2d: + 1a:dc:47:dc:5d:bd:0d:5c:54:f1:5d:5a:6d:0d:de:bc:18:67: + 2d:dd:1b:fe:8b:0e:03:19:b0:0f:f2:59:69:d0:7a:4f:a1:33: + 74:f7:22:ef:ff:90:e1:4b:8e:ac:13:00:6f:00:9b:55:83:d2: + 96:db:a8:81:c9:a9:8d:c6:a6:21:3d:14:d3:43:71:28:c6:ea: + 6d:2d:91:b9:58:bf:ec:18:75:c4:8c:10:43:88:60:08:c0:bb: + 9d:fb:90:80:1e:d5:a3:ea:e7:8a:16:f7:f4:d7:cb:35:93:03: + 55:e4:cc:58:31:1e:df:6e:e4:1b:6e:ad:3a:76:56:e5:8b:4e: + d9:71:af:11:92:a7:7a:e2:66:cc:d2:73:f3:ec:e8:3b:67:f0: + 6a:31:10:82:e8:c4:1e:ae:c3:54:a7:e2:42:86:fe:43:75:ad: + ef:83:d7:1c:2f:91:94:1c:57:9d:1c:43:94:b1:47:b2:6c:96: + fd:83:69:0f:6c:e2:18:9b:65:8e:71:08:01:b3:73:46:aa:3c: + 2e:07:14:cd:03:ae:dc:5a:51:da:c5:41:53:cc:f5:fc:c8:db: + 4e:76:27:99:9a:ec:40:68:07:d6:10:e1:f9:68:6b:5d:52:95: + 3d:01:f4:a7:40:11:61:0a -----BEGIN CERTIFICATE----- -MIIEjzCCAnegAwIBAgICEAYwDQYJKoZIhvcNAQENBQAwETEPMA0GA1UEAwwGZm9v -YmFyMCAXDTIyMTAxODEwMTQwMVoYDzIyOTYwODAyMTAxNDAxWjAiMSAwHgYDVQQD -DBdwcm94eS5wdWxzYXIuYXBhY2hlLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAPPnBnkHqKvXuv7BKOoQ8nAa7gEVAjzRANhOx2Yk3/JpN1/Ash48 -UltPjHtop1kXLrnjM3DahQuolz1A/N5sN2RGoe+/Y/aI/FRDF25yGzEoM/kwZDjm -ejQj2Hb6YsupI+YYtPr5ZDSeIBvvlVurXfXJkZf5CXYeEjqr1pEpLpNCZoWoOiiC -73/0KBoOToR5+akw+Db2Qr5FSz7AuTQ9KUZ1HZNl4xZBuEha6avESdRykH2XQzDs -qMBVruByHbzO1pg/op4iOhqQ6DFu67veKjWzMLxKR7x/A8UOd9f9D3+pabBoU72b -NqgwbKCnERoo3Y0ge1B1x7GORR7GHrWSKlUCAwEAAaOB3TCB2jAJBgNVHRMEAjAA -MBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5l -cmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1UdDgQWBBQqVR7lwaEgKHsI8+D8 -nNxPmgWZ7TBBBgNVHSMEOjA4gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJ6EVpBMwETEP -MA0GA1UEAwwGZm9vYmFyggkA1+KHT6B54gwwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud -JQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBDQUAA4ICAQBtoQTZ5u6NpDIKHo6V -yZqkRrMcg9J61zRm0tbf4D/iIsfWNiJrAWSudK4OgkUrXj4LFWKvzzcZtPltuUr5 -yODXZgz8lnyLbw6GyrKFU4Gpbr8Be30Y1yF7dfTV0yp5ZoIXNILfKhU3not1yL41 -0owaO7N0PyDAzQ7erPbbB9UG7xhYM5qFfAnevwX1rde12JHJULfeE9Ushuv+DcK5 -JmNvkRE+nB/dljsST9pW+zjBDuhwTiDZMPtUPyM0tPn6+x5zwF0pWFKhCkO8lVhr -TxCG/bMF3j/0MxjQvDvcijJFHaZqLHsw/FqgEM5SNgAsTuuY7wBohSNRddfvahV1 -xPdXUrALuDH/NmIzaYZW6hh6mOhl+R7lP2XXZbFTpTGVdoosdBTGkjbPGKMrT/L8 -hwLvFezXaHZzqj4hLnmqFbhu+dDH55EE1HT5RP7kxGCq1AMuwlsjOVxURS0FZi87 -Oaq19NKsyWfdf8igONsk0GBt5HeG+93fJkW/SxssTJdz1xc91KgGDlP3nAW3xBAz -TRvgiKIeMzOh+SWkTyz/cJugyxD+wXaAEL7VYsgOwilV+rbWKTDPvnNORqrLO/md -MHZqYWkFlld2kw8i4LYc6zXOsOWlOv0ZM7VcEs7ufBADQEiZPkDNvWlzM97oDabE -n/htdqxnoZ3NHJ1HJnz03jKSfg== +MIIEpzCCAo+gAwIBAgICEAgwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v +YmFyMCAXDTIzMDUxMDE1NTAxOVoYDzIyOTcwMjIyMTU1MDE5WjAeMRwwGgYDVQQD +DBNwcm94eS1sb2NhbGhvc3QtU0FOMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAzBXJhQZDR71Gn08DGuBulBNOsDDqiMo65DmSEsF3UYwNPLkmXC/c/LFa +vw5H/wlgMHmOVSb+0KHtn22KagaF8NDclKZUoabJPlfVaX3pJcHva3fhYnbY5FSR +QLwLEXS4MLvUAnfWvdLQ563ffZiWdEKtU7OIyNwd21FjhO5+hXMUXtTI8AFfZ1Lt +lIf31qooiyyEmIy5kbU4mYBds9TblZYJ7x2hb4bIF4b3Ch5yO1CMU+XO1IzPzIE9 +RlX/ZSULNjExpiInR5ZZOMHNZqaag5jcuC4QjbpFrqogbuMLvezmY7VAVdT+l7Hx +jZrAokaOo+2gG+1AsAClKPnaA73BqQIDAQABo4H5MIH2MAkGA1UdEwQCMAAwEQYJ +YIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRl +ZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFMUzc2cDt1EI9L3TzU/cz4MR +U605MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATBBBgNVHSMEOjA4gBRXC+nL +I+i/Rz5Qej9FfqEYQ50VJ6EVpBMwETEPMA0GA1UEAwwGZm9vYmFyggkA1+KHT6B5 +4gwwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3 +DQEBCwUAA4ICAQBD72cpmgxTl3z8cnNsjUh4TuzjFJ3ZHoNM1vBW6cTY3vVU+6U7 +/1kjdSZ08IaQ0E1BJQOH4GCkmzM9vRx5uNuGHDgJJg2APvkeKBENPWseGnqa+vwY +In/9RlXCL1ZcXIpF8nR65GzQ4OrsdLcNqPPKGM+kvqDgSjLKFX5dBla3cXzg3Bn6 +vj6UhCC+ljRhC/DR1jFJC7AguPlcSQgTm0XAb1gWgQsM+GY4WIPUsLwUNY3iHdUt +6gKuQuGIIlqwz+UxscvT6dJeiFW9YqyFqk78GGtl+Z78kycMximq8GRuctzZla44 +rmSexkSKCw8O1Gl+eeBG0HWWKhpgrzAj3NJnDQgqnVgpCR7ICNU6iC0a3EfcXb0N +XFTxXVptDd68GGct3Rv+iw4DGbAP8llp0HpPoTN09yLv/5DhS46sEwBvAJtVg9KW +26iByamNxqYhPRTTQ3EoxuptLZG5WL/sGHXEjBBDiGAIwLud+5CAHtWj6ueKFvf0 +18s1kwNV5MxYMR7fbuQbbq06dlbli07Zca8Rkqd64mbM0nPz7Og7Z/BqMRCC6MQe +rsNUp+JChv5Dda3vg9ccL5GUHFedHEOUsUeybJb9g2kPbOIYm2WOcQgBs3NGqjwu +BxTNA67cWlHaxUFTzPX8yNtOdieZmuxAaAfWEOH5aGtdUpU9AfSnQBFhCg== -----END CERTIFICATE----- diff --git a/tests/certificate-authority/server-keys/proxy.csr.pem b/tests/certificate-authority/server-keys/proxy.csr.pem index 8dbf74bb819ad..6cebd3548a14b 100644 --- a/tests/certificate-authority/server-keys/proxy.csr.pem +++ b/tests/certificate-authority/server-keys/proxy.csr.pem @@ -1,15 +1,15 @@ -----BEGIN CERTIFICATE REQUEST----- -MIICZzCCAU8CAQAwIjEgMB4GA1UEAwwXcHJveHkucHVsc2FyLmFwYWNoZS5vcmcw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDz5wZ5B6ir17r+wSjqEPJw -Gu4BFQI80QDYTsdmJN/yaTdfwLIePFJbT4x7aKdZFy654zNw2oULqJc9QPzebDdk -RqHvv2P2iPxUQxduchsxKDP5MGQ45no0I9h2+mLLqSPmGLT6+WQ0niAb75Vbq131 -yZGX+Ql2HhI6q9aRKS6TQmaFqDoogu9/9CgaDk6EefmpMPg29kK+RUs+wLk0PSlG -dR2TZeMWQbhIWumrxEnUcpB9l0Mw7KjAVa7gch28ztaYP6KeIjoakOgxbuu73io1 -szC8Ske8fwPFDnfX/Q9/qWmwaFO9mzaoMGygpxEaKN2NIHtQdcexjkUexh61kipV -AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAMBYwlvpcPsZQQMwUbts7GsX35Hcn -FAl8iWcKr9uw/9sSrZkstI9Aa8As+KYPeY3Z2p5TYY1TXokZa936NB00CWnY+gxY -lfKXy31yPqEHSwir1pQDU+WTILwZfbptFpAFEBy0SCDWrBZJUbM1ngqcVDg9jlQi -iZMDYbsnZ828Hn4e97P83bOubSBWIf1Rp6LcbIzJtwGCGVp+XPJYPMFXmpzAtwrT -tSgzCnHXseYKwIbjr+ReW58jE8Z59UqBm3/VeidLg94VfITuN5et42yypWd9Z7DU -C/qE8gjrqlvl49Xi6ye/RxKTMN+8TiQigU5ngEnYvNKbpKhU4veXHKjfrg== +MIICYzCCAUsCAQAwHjEcMBoGA1UEAwwTcHJveHktbG9jYWxob3N0LVNBTjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMwVyYUGQ0e9Rp9PAxrgbpQTTrAw +6ojKOuQ5khLBd1GMDTy5Jlwv3PyxWr8OR/8JYDB5jlUm/tCh7Z9timoGhfDQ3JSm +VKGmyT5X1Wl96SXB72t34WJ22ORUkUC8CxF0uDC71AJ31r3S0Oet332YlnRCrVOz +iMjcHdtRY4TufoVzFF7UyPABX2dS7ZSH99aqKIsshJiMuZG1OJmAXbPU25WWCe8d +oW+GyBeG9woecjtQjFPlztSMz8yBPUZV/2UlCzYxMaYiJ0eWWTjBzWammoOY3Lgu +EI26Ra6qIG7jC73s5mO1QFXU/pex8Y2awKJGjqPtoBvtQLAApSj52gO9wakCAwEA +AaAAMA0GCSqGSIb3DQEBCwUAA4IBAQCqq+WUWY5w8RHsMi/zeR0JjXhbgdYSlsfP +J0fUTB88Jr/litM2u96/HFPC4K8MDlei7cKccyrRsLd+iEbG3ydNRB66WBCjq93o +/5LSMYGCejMDAdeE8qWBDf53Xlg+hGhMlFbbmL4JGTKX0OjAKnF5xT5i7n9rh/dM +FwoIU6ac+5btszD62IRGhyOmOHXSqL+KYArKhXKzGICFKvrOwfGrSC7Rt9zwV+lL +UB6NRWR5viSgIkwYw/W4R9M1iQPQLjGm/ibjW/FXWzr98LNgs8kjqMoAIDs7z8YU +FCT6YDb8zJlqiOBKnU7ReetKsHwPM2mAyWMS6z8R3LOSNdgrP70Y -----END CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/server-keys/proxy.key-pk8.pem b/tests/certificate-authority/server-keys/proxy.key-pk8.pem index 114fe2fb04d09..0dc72cde403f8 100644 --- a/tests/certificate-authority/server-keys/proxy.key-pk8.pem +++ b/tests/certificate-authority/server-keys/proxy.key-pk8.pem @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDz5wZ5B6ir17r+ -wSjqEPJwGu4BFQI80QDYTsdmJN/yaTdfwLIePFJbT4x7aKdZFy654zNw2oULqJc9 -QPzebDdkRqHvv2P2iPxUQxduchsxKDP5MGQ45no0I9h2+mLLqSPmGLT6+WQ0niAb -75Vbq131yZGX+Ql2HhI6q9aRKS6TQmaFqDoogu9/9CgaDk6EefmpMPg29kK+RUs+ -wLk0PSlGdR2TZeMWQbhIWumrxEnUcpB9l0Mw7KjAVa7gch28ztaYP6KeIjoakOgx -buu73io1szC8Ske8fwPFDnfX/Q9/qWmwaFO9mzaoMGygpxEaKN2NIHtQdcexjkUe -xh61kipVAgMBAAECggEBAJ/DuDC1fJ477OiNPLC+MyCN81NQIKwXt/b4+5KEGxHe -LACT59j4aHYZkIsSDXTFQ71N/1cwPLBbWd4s4LcNqecMgWzbMK7AIpFLdWDKa9dy -X0EemrfO+UOIK3YcI3UGsVY63un7TNFOtve1o19tzFmBFNa4saLmpcg64Y0qrbCV -KcHslT1T07szp5s+weiMxgsD17foNSBEXLxP7+1F9NPlWuiHh+Rl2/t+K2tjrXeI -EN9dtv29q4v9jCRU4yhIunAjLEvrMYCSGhXEGa+MRkgXkTPhhVN5nWX6M0uDyKgK -aJJBv+/H6QVj4XetubYdLjII0L2q/vckoD5JsaYfz40CgYEA/7ID5OWbp/OOCjK1 -wbMByKwLUL5tHapZIoYdNg/w6zjjYl1TM9e18p1llOb+oPTEk+p8LigkkkDvPrEZ -zAhAU3Z3nRWGkVOLNYycuSed283Up0Kml08vsRNGDa78bma4GaWnJpOuPx5fB1HN -njjq9XhYzIEAHO4dT2dAQB003JMCgYEA9DFp0FnfsZsuAMLwBJJ08yHn4CjoYpMq -TAg3JScEjnm1ELJBvqLYRHzqHVeSKUHTtVDwaAqMe43qEnQ3IuFS+dhJGfOX41Cf -Yw7WDZvIeuPZER7WXUY27wmjGbjx6SdIuDYnYYA0P3RSGm3VcGZqaLoW7MvfDB0y -pYpVSV6pFncCgYEAz5/dSaCoJFjAncdPj1mruSb6iTYXpF8OwdnlHmETX+1xtg3R -4ebm93qXYbGwUUJv3SwqadBu4dOYcW+dYu/QS/WGaydvfdI41+K14CMrK7CXXLni -TDsgnsjnuXS9xWfjVfANKmYAt4AR6f+i1zegknKGqIiXbuZrJm7Q3T7aDcECgYEA -7tXBm6G7kzemt+Hx5VblgcOgyfLYz0kG7pR+cx0FbOCHAsyGVxFpGxtd09MJxsZ2 -bXm7mNbwbgvwa5o1Ly1Y/brYTMSewxrguX8SRv8eB2wAq6kQmuwI4KT5XDgyiwr8 -Kgf1XnyJHaMEhor0XlodK08PCw2fm3aXSafSIM+v66MCgYEAiGDfCy25tcI4UpAb -v8WjI2Y7EXE2vJQ/mqMhKzmfME8HMzBvuzhwAERJgPHh1lNOIwH1LnF4lZS7jr75 -A78lgfTj6ZKNHpr5s5+5zdllvFwQ51SczCUnZv0flb/S5Qeciqh0a//pe9FQvL3+ -3cqpvX158ljL8FYfcPQOBuIUdjw= +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMFcmFBkNHvUaf +TwMa4G6UE06wMOqIyjrkOZISwXdRjA08uSZcL9z8sVq/Dkf/CWAweY5VJv7Qoe2f +bYpqBoXw0NyUplShpsk+V9Vpfeklwe9rd+FidtjkVJFAvAsRdLgwu9QCd9a90tDn +rd99mJZ0Qq1Ts4jI3B3bUWOE7n6FcxRe1MjwAV9nUu2Uh/fWqiiLLISYjLmRtTiZ +gF2z1NuVlgnvHaFvhsgXhvcKHnI7UIxT5c7UjM/MgT1GVf9lJQs2MTGmIidHllk4 +wc1mppqDmNy4LhCNukWuqiBu4wu97OZjtUBV1P6XsfGNmsCiRo6j7aAb7UCwAKUo ++doDvcGpAgMBAAECggEAPTAXEFQVXe/ouaDV3HwHi0vSns67srF3QK/mFMt+e6uS +2G7mimMrTXPbMkcU3OkxtrbrLqqXYXP7K36LLkiwZcgpKkRIQYMg+RkaehtvCIwB +vWXe5Eefta2JMzBt3RjylGHsKaVGc/k9+whNZnmWOls3Xk4Ip7gfF39qaBOdSWLy +gVJJ9qbM6nHejl3vdSew5nRGlqGu0qonB+OhbM2GGSLEjH4BBbDlvI1Gl/mdKMLK +XMJf6WvO/d43+y+/EWmln2NF5lXHRwD+Gk84316mf+ANFy8V3yvp7BC9kDWqLeDH +A6wK9fmwPzwEWZKgzurN/Euz8gv1hEVcxH54HUBgVQKBgQD/iprN8phLUeA1VwJg +zm/AX6jM4GIEig0rXkYOOz1KI7wx6DunfnFINxSpDXf93cK7k6wtmLkxu0ONO9LE +1q4bmaBPkObfGOzBFixSOEE9ecyH9+EiFQi4O1zBqZLPObIY7tJ5xVsAmvQmwrf9 +Agy6fKNGM1SFBWyHgmTUvtLrlwKBgQDMc4slLWyTZYZn7bpbuXslRBWSp3KROP8M +R5wCGe0xmOz2A/UEfGMruAS3/2+GJQlMpW25ACi3GX+Wq0OAlVxHZfArw9vaJSXm +Nj1rC0bkICKYHojS0kFckTHBGUEKOAaSdP1Ehm+eUaH0kVapfw9bSQ0+Yd5Z0XHS +65QqcY9kvwKBgFD8emc+tSlZv3boJmbLxfrv1i1oB2hs4BOYgxdLivcOMDyY3x8M +IZbDbhbNn/Oi7m5INM8WkcrDEHuYNAoSB4fTvky5HZIi8hWXk2BTV8nF6h5FXuJQ +TD0nAxSVS2PFYz4noijZdSfR9AK8v1a96Y7IpW5AIk8uEuE3YAFUoL/tAoGAMGXh +uIlKPJI6APw7s17zEd1OJgtRiaMubR++hJjSl30WCx7gr5EqgLztEQl8wwqdavF2 +SecJvF5i363nKtcwow40joes0bUdhaOtYlunCnW4+r2vsghnxJvyZT2vMdYVaDId +ik0wuw+kARsuoq0bW4athejxE94Kzd1Kk8mSIk0CgYEAiBYxMU8c821HwRzHGUi4 +MEqrOV7D0WrQIeeunT9yBZvP1/OpH9VcHXM53JOQLX/1bF3gGzLevOxvzhSid9Kg +7avc497uam9LfmIueIsc90yH6Qf83O2pwLc6x/Jx9cyVX4elMC9b+CpXPzC4RDla +NGH4KgEBAySS0x7x6a31/yg= -----END PRIVATE KEY----- diff --git a/tests/certificate-authority/server-keys/proxy.key.pem b/tests/certificate-authority/server-keys/proxy.key.pem index ec79e9ddf234c..17c431ba9f5ca 100644 --- a/tests/certificate-authority/server-keys/proxy.key.pem +++ b/tests/certificate-authority/server-keys/proxy.key.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpgIBAAKCAQEA8+cGeQeoq9e6/sEo6hDycBruARUCPNEA2E7HZiTf8mk3X8Cy -HjxSW0+Me2inWRcuueMzcNqFC6iXPUD83mw3ZEah779j9oj8VEMXbnIbMSgz+TBk -OOZ6NCPYdvpiy6kj5hi0+vlkNJ4gG++VW6td9cmRl/kJdh4SOqvWkSkuk0Jmhag6 -KILvf/QoGg5OhHn5qTD4NvZCvkVLPsC5ND0pRnUdk2XjFkG4SFrpq8RJ1HKQfZdD -MOyowFWu4HIdvM7WmD+iniI6GpDoMW7ru94qNbMwvEpHvH8DxQ531/0Pf6lpsGhT -vZs2qDBsoKcRGijdjSB7UHXHsY5FHsYetZIqVQIDAQABAoIBAQCfw7gwtXyeO+zo -jTywvjMgjfNTUCCsF7f2+PuShBsR3iwAk+fY+Gh2GZCLEg10xUO9Tf9XMDywW1ne -LOC3DannDIFs2zCuwCKRS3VgymvXcl9BHpq3zvlDiCt2HCN1BrFWOt7p+0zRTrb3 -taNfbcxZgRTWuLGi5qXIOuGNKq2wlSnB7JU9U9O7M6ebPsHojMYLA9e36DUgRFy8 -T+/tRfTT5Vroh4fkZdv7fitrY613iBDfXbb9vauL/YwkVOMoSLpwIyxL6zGAkhoV -xBmvjEZIF5Ez4YVTeZ1l+jNLg8ioCmiSQb/vx+kFY+F3rbm2HS4yCNC9qv73JKA+ -SbGmH8+NAoGBAP+yA+Tlm6fzjgoytcGzAcisC1C+bR2qWSKGHTYP8Os442JdUzPX -tfKdZZTm/qD0xJPqfC4oJJJA7z6xGcwIQFN2d50VhpFTizWMnLknndvN1KdCppdP -L7ETRg2u/G5muBmlpyaTrj8eXwdRzZ446vV4WMyBABzuHU9nQEAdNNyTAoGBAPQx -adBZ37GbLgDC8ASSdPMh5+Ao6GKTKkwINyUnBI55tRCyQb6i2ER86h1XkilB07VQ -8GgKjHuN6hJ0NyLhUvnYSRnzl+NQn2MO1g2byHrj2REe1l1GNu8Joxm48eknSLg2 -J2GAND90Uhpt1XBmami6FuzL3wwdMqWKVUleqRZ3AoGBAM+f3UmgqCRYwJ3HT49Z -q7km+ok2F6RfDsHZ5R5hE1/tcbYN0eHm5vd6l2GxsFFCb90sKmnQbuHTmHFvnWLv -0Ev1hmsnb33SONfiteAjKyuwl1y54kw7IJ7I57l0vcVn41XwDSpmALeAEen/otc3 -oJJyhqiIl27mayZu0N0+2g3BAoGBAO7VwZuhu5M3prfh8eVW5YHDoMny2M9JBu6U -fnMdBWzghwLMhlcRaRsbXdPTCcbGdm15u5jW8G4L8GuaNS8tWP262EzEnsMa4Ll/ -Ekb/HgdsAKupEJrsCOCk+Vw4MosK/CoH9V58iR2jBIaK9F5aHStPDwsNn5t2l0mn -0iDPr+ujAoGBAIhg3wstubXCOFKQG7/FoyNmOxFxNryUP5qjISs5nzBPBzMwb7s4 -cABESYDx4dZTTiMB9S5xeJWUu46++QO/JYH04+mSjR6a+bOfuc3ZZbxcEOdUnMwl -J2b9H5W/0uUHnIqodGv/6XvRULy9/t3Kqb19efJYy/BWH3D0DgbiFHY8 +MIIEowIBAAKCAQEAzBXJhQZDR71Gn08DGuBulBNOsDDqiMo65DmSEsF3UYwNPLkm +XC/c/LFavw5H/wlgMHmOVSb+0KHtn22KagaF8NDclKZUoabJPlfVaX3pJcHva3fh +YnbY5FSRQLwLEXS4MLvUAnfWvdLQ563ffZiWdEKtU7OIyNwd21FjhO5+hXMUXtTI +8AFfZ1LtlIf31qooiyyEmIy5kbU4mYBds9TblZYJ7x2hb4bIF4b3Ch5yO1CMU+XO +1IzPzIE9RlX/ZSULNjExpiInR5ZZOMHNZqaag5jcuC4QjbpFrqogbuMLvezmY7VA +VdT+l7HxjZrAokaOo+2gG+1AsAClKPnaA73BqQIDAQABAoIBAD0wFxBUFV3v6Lmg +1dx8B4tL0p7Ou7Kxd0Cv5hTLfnurkthu5opjK01z2zJHFNzpMba26y6ql2Fz+yt+ +iy5IsGXIKSpESEGDIPkZGnobbwiMAb1l3uRHn7WtiTMwbd0Y8pRh7CmlRnP5PfsI +TWZ5ljpbN15OCKe4Hxd/amgTnUli8oFSSfamzOpx3o5d73UnsOZ0RpahrtKqJwfj +oWzNhhkixIx+AQWw5byNRpf5nSjCylzCX+lrzv3eN/svvxFppZ9jReZVx0cA/hpP +ON9epn/gDRcvFd8r6ewQvZA1qi3gxwOsCvX5sD88BFmSoM7qzfxLs/IL9YRFXMR+ +eB1AYFUCgYEA/4qazfKYS1HgNVcCYM5vwF+ozOBiBIoNK15GDjs9SiO8Meg7p35x +SDcUqQ13/d3Cu5OsLZi5MbtDjTvSxNauG5mgT5Dm3xjswRYsUjhBPXnMh/fhIhUI +uDtcwamSzzmyGO7SecVbAJr0JsK3/QIMunyjRjNUhQVsh4Jk1L7S65cCgYEAzHOL +JS1sk2WGZ+26W7l7JUQVkqdykTj/DEecAhntMZjs9gP1BHxjK7gEt/9vhiUJTKVt +uQAotxl/lqtDgJVcR2XwK8Pb2iUl5jY9awtG5CAimB6I0tJBXJExwRlBCjgGknT9 +RIZvnlGh9JFWqX8PW0kNPmHeWdFx0uuUKnGPZL8CgYBQ/HpnPrUpWb926CZmy8X6 +79YtaAdobOATmIMXS4r3DjA8mN8fDCGWw24WzZ/zou5uSDTPFpHKwxB7mDQKEgeH +075MuR2SIvIVl5NgU1fJxeoeRV7iUEw9JwMUlUtjxWM+J6Io2XUn0fQCvL9WvemO +yKVuQCJPLhLhN2ABVKC/7QKBgDBl4biJSjySOgD8O7Ne8xHdTiYLUYmjLm0fvoSY +0pd9Fgse4K+RKoC87REJfMMKnWrxdknnCbxeYt+t5yrXMKMONI6HrNG1HYWjrWJb +pwp1uPq9r7IIZ8Sb8mU9rzHWFWgyHYpNMLsPpAEbLqKtG1uGrYXo8RPeCs3dSpPJ +kiJNAoGBAIgWMTFPHPNtR8EcxxlIuDBKqzlew9Fq0CHnrp0/cgWbz9fzqR/VXB1z +OdyTkC1/9Wxd4Bsy3rzsb84UonfSoO2r3OPe7mpvS35iLniLHPdMh+kH/NztqcC3 +OsfycfXMlV+HpTAvW/gqVz8wuEQ5WjRh+CoBAQMkktMe8emt9f8o -----END RSA PRIVATE KEY----- From d92010c00e30199c5cd53021e20ba5d7bb36701c Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 11:28:32 -0500 Subject: [PATCH 204/280] [feat] OIDC: support JWKS refresh for missing Key ID (#20338) ### Motivation When the `AuthenticationProviderOpenID` encounters an unknown Key ID (known as the `kid` in the JWT) for a trusted token issuer, the token is rejected with the following error `java.lang.IllegalArgumentException: No JWK found for Key ID `. This behavior is technically valid, but it isn't ideal because it is possible that the Identity Provider has issued new signing keys and started using them before the Pulsar authentication provider has refreshed its cache. This PR introduces a behavior to retrieve This PR adds a configuration called `openIDKeyIdCacheMissRefreshSeconds` that represents the length of time that must pass before the provider will reload the JWKS from the Identity Provider. The `openIDKeyIdCacheMissRefreshSeconds` setting limits the impact of an attacker invalidating the JWKS cache. When `openIDKeyIdCacheMissRefreshSeconds <= 0`, the JWKS will be refreshed for any missing key id when the issuer is trusted. This is only meant for testing. ### Modifications * Add `openIDKeyIdCacheMissRefreshSeconds` setting and default it to 5 minutes. * Add functionality to invalidate and refresh the cache when a token has an unknown `kid` for a trusted issuer. ### Verifying this change New tests are added. ### Does this pull request potentially affect one of the following parts: This adds a new configuration, but it is very minor. ### Documentation - [x] `doc-required` ### Matching PR in forked repository PR in forked repository: Skipping since tests passed already on my local machine --- .../oidc/AuthenticationProviderOpenID.java | 2 + .../broker/authentication/oidc/JwksCache.java | 48 +++++++-- ...ticationProviderOpenIDIntegrationTest.java | 100 ++++++++++++++++++ 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java index 00ec09bd1817f..2078666a08dd9 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java @@ -133,6 +133,8 @@ public class AuthenticationProviderOpenID implements AuthenticationProvider { static final int CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT = 18 * 60 * 60; static final String CACHE_EXPIRATION_SECONDS = "openIDCacheExpirationSeconds"; static final int CACHE_EXPIRATION_SECONDS_DEFAULT = 24 * 60 * 60; + static final String KEY_ID_CACHE_MISS_REFRESH_SECONDS = "openIDKeyIdCacheMissRefreshSeconds"; + static final int KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT = 5 * 60; static final String HTTP_CONNECTION_TIMEOUT_MILLIS = "openIDHttpConnectionTimeoutMillis"; static final int HTTP_CONNECTION_TIMEOUT_MILLIS_DEFAULT = 10_000; static final String HTTP_READ_TIMEOUT_MILLIS = "openIDHttpReadTimeoutMillis"; diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java index b5e038342c2e9..73934e9c1e05e 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java @@ -24,6 +24,8 @@ import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric; import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt; import com.auth0.jwk.Jwk; @@ -43,6 +45,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import javax.naming.AuthenticationException; import org.apache.pulsar.broker.ServiceConfiguration; @@ -52,7 +55,8 @@ public class JwksCache { // Map from an issuer's JWKS URI to its JWKS. When the Issuer is not empty, use the fallback client. private final AsyncLoadingCache, List> cache; - + private final ConcurrentHashMap, Long> jwksLastRefreshTime = new ConcurrentHashMap<>(); + private final long keyIdCacheMissRefreshNanos; private final ObjectReader reader = new ObjectMapper().readerFor(HashMap.class); private final AsyncHttpClient httpClient; private final OpenidApi openidApi; @@ -61,7 +65,8 @@ public class JwksCache { // Store the clients this.httpClient = httpClient; this.openidApi = apiClient != null ? new OpenidApi(apiClient) : null; - + keyIdCacheMissRefreshNanos = TimeUnit.SECONDS.toNanos(getConfigValueAsInt(config, + KEY_ID_CACHE_MISS_REFRESH_SECONDS, KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT)); // Configure the cache int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT); int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS, @@ -69,6 +74,8 @@ public class JwksCache { int expireAfterSeconds = getConfigValueAsInt(config, CACHE_EXPIRATION_SECONDS, CACHE_EXPIRATION_SECONDS_DEFAULT); AsyncCacheLoader, List> loader = (jwksUri, executor) -> { + // Store the time of the retrieval, even though it might be a little early or the call might fail. + jwksLastRefreshTime.put(jwksUri, System.nanoTime()); if (jwksUri.isPresent()) { return getJwksFromJwksUri(jwksUri.get()); } else { @@ -87,7 +94,37 @@ CompletableFuture getJwk(String jwksUri, String keyId) { incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); return CompletableFuture.failedFuture(new IllegalArgumentException("jwksUri must not be null.")); } - return cache.get(Optional.of(jwksUri)).thenApply(jwks -> getJwkForKID(jwks, keyId)); + return getJwkAndMaybeReload(Optional.of(jwksUri), keyId, false); + } + + /** + * Retrieve the JWK for the given key ID from the given JWKS URI. If the key ID is not found, and failOnMissingKeyId + * is false, then the JWK will be reloaded from the JWKS URI and the key ID will be searched for again. + */ + private CompletableFuture getJwkAndMaybeReload(Optional maybeJwksUri, + String keyId, + boolean failOnMissingKeyId) { + return cache + .get(maybeJwksUri) + .thenCompose(jwks -> { + try { + return CompletableFuture.completedFuture(getJwkForKID(maybeJwksUri, jwks, keyId)); + } catch (IllegalArgumentException e) { + if (failOnMissingKeyId) { + throw e; + } else { + Long lastRefresh = jwksLastRefreshTime.get(maybeJwksUri); + if (lastRefresh == null || System.nanoTime() - lastRefresh > keyIdCacheMissRefreshNanos) { + // In this case, the key ID was not found, but we haven't refreshed the JWKS in a while, + // so it is possible the key ID was added. Refresh the JWKS and try again. + cache.synchronous().invalidate(maybeJwksUri); + } + // There is a small race condition where the JWKS could be refreshed by another thread, + // so we retry getting the JWK, even though we might not have invalidated the cache. + return getJwkAndMaybeReload(maybeJwksUri, keyId, true); + } + } + }); } private CompletableFuture> getJwksFromJwksUri(String jwksUri) { @@ -119,8 +156,7 @@ CompletableFuture getJwkFromKubernetesApiServer(String keyId) { return CompletableFuture.failedFuture(new AuthenticationException( "Failed to retrieve public key from Kubernetes API server: Kubernetes fallback is not enabled.")); } - return cache.get(Optional.empty(), (__, executor) -> getJwksFromKubernetesApiServer()) - .thenApply(jwks -> getJwkForKID(jwks, keyId)); + return getJwkAndMaybeReload(Optional.empty(), keyId, false); } private CompletableFuture> getJwksFromKubernetesApiServer() { @@ -170,7 +206,7 @@ public void onDownloadProgress(long bytesRead, long contentLength, boolean done) return future; } - private Jwk getJwkForKID(List jwks, String keyId) { + private Jwk getJwkForKID(Optional maybeJwksUri, List jwks, String keyId) { for (Jwk jwk : jwks) { if (jwk.getId().equals(keyId)) { return jwk; diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java index 0075d70f599d3..d2d2de1a1149d 100644 --- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java @@ -29,6 +29,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.impl.DefaultJwtBuilder; import io.jsonwebtoken.io.Decoders; @@ -58,6 +59,7 @@ import org.apache.pulsar.common.api.AuthData; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; /** @@ -75,6 +77,7 @@ public class AuthenticationProviderOpenIDIntegrationTest { // The valid issuer String issuer; String issuerWithTrailingSlash; + String issuerWithMissingKid; // This issuer is configured to return an issuer in the openid-configuration // that does not match the issuer on the token String issuerThatFails; @@ -90,6 +93,7 @@ void beforeClass() throws IOException { server.start(); issuer = server.baseUrl(); issuerWithTrailingSlash = issuer + "/trailing-slash/"; + issuerWithMissingKid = issuer + "/missing-kid"; issuerThatFails = issuer + "/fail"; issuerK8s = issuer + "/k8s"; @@ -183,6 +187,50 @@ void beforeClass() throws IOException { } """.formatted(validJwk, n, e, invalidJwk)))); + server.stubFor( + get(urlEqualTo("/missing-kid/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "jwks_uri": "%s/keys" + } + """.formatted(issuerWithMissingKid, issuerWithMissingKid)))); + + // Set up JWKS endpoint where it first responds without the KID, then with the KID. This is a stateful stub. + // Note that the state machine is circular to make it easier to verify the two code paths that rely on + // this logic. + server.stubFor( + get(urlMatching( "/missing-kid/keys")) + .inScenario("Changing KIDs") + .whenScenarioStateIs(Scenario.STARTED) + .willSetStateTo("serve-kid") + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody("{\"keys\":[]}"))); + server.stubFor( + get(urlMatching( "/missing-kid/keys")) + .inScenario("Changing KIDs") + .whenScenarioStateIs("serve-kid") + .willSetStateTo(Scenario.STARTED) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody( + """ + { + "keys" : [ + { + "kid":"%s", + "kty":"RSA", + "alg":"RS256", + "n":"%s", + "e":"%s" + } + ] + } + """.formatted(validJwk, n, e)))); + ServiceConfiguration conf = new ServiceConfiguration(); conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); @@ -207,6 +255,12 @@ void afterClass() { server.stop(); } + @BeforeMethod + public void beforeMethod() { + // Scenarios are stateful. Start each test with the correct state. + server.resetScenarios(); + } + @Test public void testTokenWithValidJWK() throws Exception { String role = "superuser"; @@ -268,6 +322,52 @@ public void testTokenWithInvalidIssuer() throws Exception { assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); } } + @Test + public void testKidCacheMissWhenRefreshConfigZero() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + // Allows us to retrieve the JWK immediately after the cache miss of the KID + props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "0"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); + } + + @Test + public void testKidCacheMissWhenRefreshConfigLongerThanDelta() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + // This value is high enough that the provider will not refresh the JWK + props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "100"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof IllegalArgumentException, "Found exception: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("No JWK found for Key ID valid"), + "Found exception: " + e.getCause()); + } + } @Test public void testKubernetesApiServerAsDiscoverTrustedIssuerSuccess() throws Exception { From 09e646f72eb640987ca9162988b6b146c485cd31 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 16:57:16 -0500 Subject: [PATCH 205/280] [cleanup] Remove Streaming Dispatcher Code (#20279) Discussed on mailing list here: https://lists.apache.org/thread/ky2bkzlz93njx3ntnvkpd0l77qzzgcmv Fixes: https://github.com/apache/pulsar/issues/19088 Fixes: https://github.com/apache/pulsar/issues/16450 Fixes: https://github.com/apache/pulsar/issues/15422 Fixes: https://github.com/apache/pulsar/issues/11428 ### Motivation I am putting this PR together to move the mailing list conversation along. Copied from @eolivelli on the mailing list thread: > There are many flaky tests about that feature and I believe that it has never been used in Production by anyone, because it happened a few times that we did some changes in the regular Dispatcher and introduced bugs on the StreamingDispacther (usually manifested as flaky tests) ### Modifications * Remove all `StreamingDispatcher` code, tests, and configuration. ### Verifying this change It should be sufficient to see the tests pass. ### Does this pull request potentially affect one of the following parts: This affects configuration. ### Documentation - [x] `doc-not-needed` There are no references to "streaming dispatcher" or "streamingdispatcher" in our github `pulsar-site` repo, which indicates to me that no docs need to be updated. ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/46 --- .../pulsar/broker/ServiceConfiguration.java | 7 - ...tStreamingDispatcherMultipleConsumers.java | 240 ---------- ...reamingDispatcherSingleActiveConsumer.java | 233 --------- .../persistent/PersistentSubscription.java | 17 +- .../PendingReadEntryRequest.java | 76 --- .../StreamingDispatcher.java | 53 --- .../StreamingEntryReader.java | 342 -------------- .../streamingdispatch/package-info.java | 19 - ...iloverConsumerStreamingDispatcherTest.java | 37 -- ...entFailoverStreamingDispatcherE2ETest.java | 38 -- ...tStreamingDispatcherBlockConsumerTest.java | 38 -- ...atchStreamingDispatcherThrottlingTest.java | 39 -- ...istentTopicStreamingDispatcherE2ETest.java | 42 -- ...ersistentTopicStreamingDispatcherTest.java | 41 -- ...erConsumerTestStreamingDispatcherTest.java | 36 -- .../StreamingEntryReaderTests.java | 441 ------------------ .../api/MessageDispatchThrottlingTest.java | 1 - 17 files changed, 3 insertions(+), 1697 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 8668c17c3d88d..9966912bc8eae 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1174,13 +1174,6 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private boolean allowOverrideEntryFilters = false; - @FieldContext( - category = CATEGORY_SERVER, - doc = "Whether to use streaming read dispatcher. Currently is in preview and can be changed " - + "in subsequent release." - ) - private boolean streamingDispatch = false; - @FieldContext( dynamic = true, category = CATEGORY_SERVER, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java deleted file mode 100644 index 44e8a423344cb..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import com.google.common.collect.Lists; -import java.util.Set; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.service.Subscription; -import org.apache.pulsar.broker.service.streamingdispatch.PendingReadEntryRequest; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingEntryReader; - -/** - * A {@link PersistentDispatcherMultipleConsumers} implemented {@link StreamingDispatcher}. - * It'll use {@link StreamingEntryReader} to read new entries instead read as micro batch from managed ledger. - */ -@Slf4j -public class PersistentStreamingDispatcherMultipleConsumers extends PersistentDispatcherMultipleConsumers - implements StreamingDispatcher { - - private int sendingTaskCounter = 0; - private final StreamingEntryReader streamingEntryReader = new StreamingEntryReader((ManagedCursorImpl) cursor, - this, topic); - private final Executor topicExecutor; - - public PersistentStreamingDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor, - Subscription subscription) { - super(topic, cursor, subscription); - this.topicExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topic.getName()); - } - - /** - * {@inheritDoc} - */ - @Override - public synchronized void readEntryComplete(Entry entry, PendingReadEntryRequest ctx) { - - ReadType readType = (ReadType) ctx.ctx; - if (ctx.isLast()) { - readFailureBackoff.reduceToHalf(); - if (readType == ReadType.Normal) { - havePendingRead = false; - } else { - havePendingReplayRead = false; - } - } - - if (readBatchSize < serviceConfig.getDispatcherMaxReadBatchSize()) { - int newReadBatchSize = Math.min(readBatchSize * 2, serviceConfig.getDispatcherMaxReadBatchSize()); - if (log.isDebugEnabled()) { - log.debug("[{}] Increasing read batch size from {} to {}", name, readBatchSize, newReadBatchSize); - } - readBatchSize = newReadBatchSize; - } - - if (shouldRewindBeforeReadingOrReplaying && readType == ReadType.Normal) { - // All consumers got disconnected before the completion of the read operation - entry.release(); - cursor.rewind(); - shouldRewindBeforeReadingOrReplaying = false; - readMoreEntries(); - return; - } - - if (log.isDebugEnabled()) { - log.debug("[{}] Distributing a messages to {} consumers", name, consumerList.size()); - } - - cursor.seek(((ManagedLedgerImpl) cursor.getManagedLedger()) - .getNextValidPosition((PositionImpl) entry.getPosition())); - - long size = entry.getLength(); - updatePendingBytesToDispatch(size); - // dispatch messages to a separate thread, but still in order for this subscription - // sendMessagesToConsumers is responsible for running broker-side filters - // that may be quite expensive - if (serviceConfig.isDispatcherDispatchMessagesInSubscriptionThread()) { - // setting sendInProgress here, because sendMessagesToConsumers will be executed - // in a separate thread, and we want to prevent more reads - acquireSendInProgress(); - dispatchMessagesThread.execute(() -> { - if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), false)) { - readMoreEntries(); - } else { - updatePendingBytesToDispatch(-size); - } - }); - } else { - if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), true)) { - readMoreEntriesAsync(); - } else { - updatePendingBytesToDispatch(-size); - } - } - ctx.recycle(); - } - - /** - * {@inheritDoc} - */ - @Override - public void canReadMoreEntries(boolean withBackoff) { - havePendingRead = false; - topic.getBrokerService().executor().schedule(() -> { - topicExecutor.execute(() -> { - synchronized (PersistentStreamingDispatcherMultipleConsumers.this) { - if (!havePendingRead) { - log.info("[{}] Scheduling read operation", name); - readMoreEntries(); - } else { - log.info("[{}] Skipping read since we have pendingRead", name); - } - } - }); - }, withBackoff - ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS); - } - - /** - * {@inheritDoc} - */ - @Override - public void notifyConsumersEndOfTopic() { - if (cursor.getNumberOfEntriesInBacklog(false) == 0) { - // Topic has been terminated and there are no more entries to read - // Notify the consumer only if all the messages were already acknowledged - checkAndApplyReachedEndOfTopicOrTopicMigration(consumerList); - } - } - - @Override - protected void cancelPendingRead() { - if (havePendingRead && streamingEntryReader.cancelReadRequests()) { - havePendingRead = false; - } - } - - @Override - protected synchronized void acquireSendInProgress() { - sendingTaskCounter++; - } - - @Override - protected synchronized void releaseSendInProgress() { - sendingTaskCounter--; - } - - @Override - protected synchronized boolean isSendInProgress() { - return sendingTaskCounter > 0; - } - - @Override - public synchronized void readMoreEntries() { - if (isSendInProgress()) { - // we cannot read more entries while sending the previous batch - // otherwise we could re-read the same entries and send duplicates - return; - } - // totalAvailablePermits may be updated by other threads - int currentTotalAvailablePermits = totalAvailablePermits; - if (currentTotalAvailablePermits > 0 && isAtleastOneConsumerAvailable()) { - Pair calculateResult = calculateToRead(currentTotalAvailablePermits); - int messagesToRead = calculateResult.getLeft(); - long bytesToRead = calculateResult.getRight(); - if (-1 == messagesToRead || bytesToRead == -1) { - // Skip read as topic/dispatcher has exceed the dispatch rate or previous pending read hasn't complete. - return; - } - - Set messagesToReplayNow = getMessagesToReplayNow(messagesToRead); - - if (!messagesToReplayNow.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Schedule replay of {} messages for {} consumers", name, messagesToReplayNow.size(), - consumerList.size()); - } - - havePendingReplayRead = true; - Set deletedMessages = topic.isDelayedDeliveryEnabled() - ? asyncReplayEntriesInOrder(messagesToReplayNow) : asyncReplayEntries(messagesToReplayNow); - // clear already acked positions from replay bucket - - deletedMessages.forEach(position -> redeliveryMessages.remove(((PositionImpl) position).getLedgerId(), - ((PositionImpl) position).getEntryId())); - // if all the entries are acked-entries and cleared up from redeliveryMessages, try to read - // next entries as readCompletedEntries-callback was never called - if ((messagesToReplayNow.size() - deletedMessages.size()) == 0) { - havePendingReplayRead = false; - // We should not call readMoreEntries() recursively in the same thread - // as there is a risk of StackOverflowError - topic.getBrokerService().executor().execute(this::readMoreEntries); - } - } else if (BLOCKED_DISPATCHER_ON_UNACKMSG_UPDATER.get(this) == TRUE) { - log.debug("[{}] Dispatcher read is blocked due to unackMessages {} reached to max {}", name, - totalUnackedMessages, topic.getMaxUnackedMessagesOnSubscription()); - } else if (!havePendingRead) { - if (log.isDebugEnabled()) { - log.debug("[{}] Schedule read of {} messages for {} consumers", name, messagesToRead, - consumerList.size()); - } - havePendingRead = true; - streamingEntryReader.asyncReadEntries(messagesToRead, bytesToRead, - ReadType.Normal); - } else { - log.debug("[{}] Cannot schedule next read until previous one is done", name); - } - } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Consumer buffer is full, pause reading", name); - } - } - } - -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java deleted file mode 100644 index efe9de778a3e7..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; -import com.google.common.collect.Lists; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.service.Consumer; -import org.apache.pulsar.broker.service.EntryBatchIndexesAcks; -import org.apache.pulsar.broker.service.EntryBatchSizes; -import org.apache.pulsar.broker.service.SendMessageInfo; -import org.apache.pulsar.broker.service.Subscription; -import org.apache.pulsar.broker.service.streamingdispatch.PendingReadEntryRequest; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingEntryReader; -import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; - -/** - * A {@link PersistentDispatcherSingleActiveConsumer} implemented {@link StreamingDispatcher}. - * It'll use {@link StreamingEntryReader} to read new entries instead read as micro batch from managed ledger. - */ -@Slf4j -public class PersistentStreamingDispatcherSingleActiveConsumer extends PersistentDispatcherSingleActiveConsumer - implements StreamingDispatcher { - - private final StreamingEntryReader streamingEntryReader = new StreamingEntryReader((ManagedCursorImpl) cursor, - this, topic); - - private final Executor dispatcherExecutor; - - public PersistentStreamingDispatcherSingleActiveConsumer(ManagedCursor cursor, SubType subscriptionType, - int partitionIndex, PersistentTopic topic, - Subscription subscription) { - super(cursor, subscriptionType, partitionIndex, topic, subscription); - this.dispatcherExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(name); - } - - /** - * {@inheritDoc} - */ - @Override - public void canReadMoreEntries(boolean withBackoff) { - havePendingRead = false; - topic.getBrokerService().executor().schedule(() -> { - topicExecutor.execute(() -> { - synchronized (PersistentStreamingDispatcherSingleActiveConsumer.this) { - Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); - if (currentConsumer != null && !havePendingRead) { - if (log.isDebugEnabled()) { - log.debug("[{}-{}] Scheduling read ", name, currentConsumer); - } - readMoreEntries(currentConsumer); - } else { - log.info("[{}-{}] Skipping read as we still havePendingRead {}", name, - currentConsumer, havePendingRead); - } - } - }); - }, withBackoff - ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS); - } - - @Override - protected void cancelPendingRead() { - if (havePendingRead && streamingEntryReader.cancelReadRequests()) { - havePendingRead = false; - } - } - - /** - * {@inheritDoc} - */ - @Override - public synchronized void notifyConsumersEndOfTopic() { - if (cursor.getNumberOfEntriesInBacklog(false) == 0) { - // Topic has been terminated and there are no more entries to read - // Notify the consumer only if all the messages were already acknowledged - checkAndApplyReachedEndOfTopicOrTopicMigration(consumers); - } - } - - @Override - public String getName() { - return name; - } - - /** - * {@inheritDoc} - */ - @Override - public void readEntryComplete(Entry entry, PendingReadEntryRequest ctx) { - dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx)); - } - - public synchronized void internalReadEntryComplete(Entry entry, PendingReadEntryRequest ctx) { - if (ctx.isLast()) { - readFailureBackoff.reduceToHalf(); - havePendingRead = false; - } - - isFirstRead = false; - - if (readBatchSize < serviceConfig.getDispatcherMaxReadBatchSize()) { - int newReadBatchSize = Math.min(readBatchSize * 2, serviceConfig.getDispatcherMaxReadBatchSize()); - if (log.isDebugEnabled()) { - log.debug("[{}-{}] Increasing read batch size from {} to {}", name, - ((Consumer) ctx.ctx).consumerName(), readBatchSize, newReadBatchSize); - } - readBatchSize = newReadBatchSize; - } - - Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); - - if (isKeyHashRangeFiltered) { - byte[] key = peekStickyKey(entry.getDataBuffer()); - Consumer consumer = stickyKeyConsumerSelector.select(key); - // Skip the entry if it's not for current active consumer. - if (consumer == null || currentConsumer != consumer) { - entry.release(); - return; - } - } - Consumer consumer = (Consumer) ctx.ctx; - ctx.recycle(); - if (currentConsumer == null || consumer != currentConsumer) { - // Active consumer has changed since the read request has been issued. We need to rewind the cursor and - // re-issue the read request for the new consumer - if (log.isDebugEnabled()) { - log.debug("[{}] Rewind because no available consumer found to dispatch message to.", name); - } - - entry.release(); - streamingEntryReader.cancelReadRequests(); - havePendingRead = false; - if (currentConsumer != null) { - notifyActiveConsumerChanged(currentConsumer); - readMoreEntries(currentConsumer); - } - } else { - EntryBatchSizes batchSizes = EntryBatchSizes.get(1); - SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); - EntryBatchIndexesAcks batchIndexesAcks = EntryBatchIndexesAcks.get(1); - filterEntriesForConsumer(Lists.newArrayList(entry), batchSizes, sendMessageInfo, batchIndexesAcks, - cursor, false, consumer); - // Update cursor's read position. - cursor.seek(((ManagedLedgerImpl) cursor.getManagedLedger()) - .getNextValidPosition((PositionImpl) entry.getPosition())); - dispatchEntriesToConsumer(currentConsumer, Lists.newArrayList(entry), batchSizes, - batchIndexesAcks, sendMessageInfo, DEFAULT_CONSUMER_EPOCH); - } - } - - @Override - protected void readMoreEntries(Consumer consumer) { - // consumer can be null when all consumers are disconnected from broker. - // so skip reading more entries if currently there is no active consumer. - if (null == consumer) { - if (log.isDebugEnabled()) { - log.debug("[{}] Skipping read for the topic, Due to the current consumer is null", topic.getName()); - } - return; - } - if (havePendingRead) { - if (log.isDebugEnabled()) { - log.debug("[{}] Skipping read for the topic, Due to we have pending read.", topic.getName()); - } - return; - } - - if (consumer.getAvailablePermits() > 0) { - synchronized (this) { - if (havePendingRead) { - if (log.isDebugEnabled()) { - log.debug("[{}] Skipping read for the topic, Due to we have pending read.", topic.getName()); - } - return; - } - - Pair calculateResult = calculateToRead(consumer); - int messagesToRead = calculateResult.getLeft(); - long bytesToRead = calculateResult.getRight(); - - - if (-1 == messagesToRead || bytesToRead == -1) { - // Skip read as topic/dispatcher has exceed the dispatch rate. - return; - } - - // Schedule read - if (log.isDebugEnabled()) { - log.debug("[{}-{}] Schedule read of {} messages", name, consumer, messagesToRead); - } - havePendingRead = true; - - if (consumer.readCompacted()) { - topic.getCompactedTopic().asyncReadEntriesOrWait(cursor, messagesToRead, isFirstRead, - this, consumer); - } else { - streamingEntryReader.asyncReadEntries(messagesToRead, bytesToRead, consumer); - } - } - } else { - if (log.isDebugEnabled()) { - log.debug("[{}-{}] Consumer buffer is full, pause reading", name, consumer); - } - } - } - -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index d283cac77c7b6..09dabcd4bfc59 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -223,26 +223,18 @@ public CompletableFuture addConsumer(Consumer consumer) { if (dispatcher == null || !dispatcher.isConsumerConnected()) { Dispatcher previousDispatcher = null; - boolean useStreamingDispatcher = topic.getBrokerService().getPulsar() - .getConfiguration().isStreamingDispatch(); switch (consumer.subType()) { case Exclusive: if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) { previousDispatcher = dispatcher; - dispatcher = useStreamingDispatcher - ? new PersistentStreamingDispatcherSingleActiveConsumer( - cursor, SubType.Exclusive, 0, topic, this) - : new PersistentDispatcherSingleActiveConsumer( + dispatcher = new PersistentDispatcherSingleActiveConsumer( cursor, SubType.Exclusive, 0, topic, this); } break; case Shared: if (dispatcher == null || dispatcher.getType() != SubType.Shared) { previousDispatcher = dispatcher; - dispatcher = useStreamingDispatcher - ? new PersistentStreamingDispatcherMultipleConsumers( - topic, cursor, this) - : new PersistentDispatcherMultipleConsumers(topic, cursor, this); + dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursor, this); } break; case Failover: @@ -256,10 +248,7 @@ public CompletableFuture addConsumer(Consumer consumer) { if (dispatcher == null || dispatcher.getType() != SubType.Failover) { previousDispatcher = dispatcher; - dispatcher = useStreamingDispatcher - ? new PersistentStreamingDispatcherSingleActiveConsumer( - cursor, SubType.Failover, partitionIndex, topic, this) : - new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover, + dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover, partitionIndex, topic, this); } break; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java deleted file mode 100644 index e98e5de07c165..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.streamingdispatch; - -import io.netty.util.Recycler; -import lombok.Data; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedger; -import org.apache.bookkeeper.mledger.impl.PositionImpl; - -/** - * Representing a pending read request to read an entry from {@link ManagedLedger} carrying necessary context. - */ -@Data -public class PendingReadEntryRequest { - - private static final Recycler RECYCLER = new Recycler() { - protected PendingReadEntryRequest newObject(Recycler.Handle handle) { - return new PendingReadEntryRequest(handle); - } - }; - - public static PendingReadEntryRequest create(Object ctx, PositionImpl position) { - PendingReadEntryRequest pendingReadEntryRequest = RECYCLER.get(); - pendingReadEntryRequest.ctx = ctx; - pendingReadEntryRequest.position = position; - pendingReadEntryRequest.retry = 0; - pendingReadEntryRequest.isLast = false; - return pendingReadEntryRequest; - } - - public void recycle() { - entry = null; - ctx = null; - position = null; - retry = -1; - recyclerHandle.recycle(this); - } - - public boolean isLastRequest() { - return isLast; - } - - private final Recycler.Handle recyclerHandle; - - // Entry read from ledger - public Entry entry; - - // Passed in context that'll be pass to callback - public Object ctx; - - // Position of entry to be read - public PositionImpl position; - - // Number of time request has been retried. - int retry; - - // If request is the last one of a set of requests. - boolean isLast; -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java deleted file mode 100644 index 4e9e8befe0f9e..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.streamingdispatch; - -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedger; -import org.apache.pulsar.broker.service.Dispatcher; -import org.apache.pulsar.common.classification.InterfaceStability; - -/** - * A {@link Dispatcher} that'll use {@link StreamingEntryReader} to read entries from {@link ManagedLedger}. - */ -@InterfaceStability.Unstable -public interface StreamingDispatcher extends Dispatcher { - - /** - * Notify dispatcher issued read entry request has complete. - * @param entry Entry read. - * @param ctx Context passed in when issuing read entries request. - */ - void readEntryComplete(Entry entry, PendingReadEntryRequest ctx); - - /** - * Notify dispatcher can issue next read request. - */ - void canReadMoreEntries(boolean withBackoff); - - /** - * Notify dispatcher to inform consumers reached end of topic. - */ - void notifyConsumersEndOfTopic(); - - /** - * @return Name of the dispatcher. - */ - String getName(); -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java deleted file mode 100644 index 6ffc5ba0f623a..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.streamingdispatch; - -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.WaitingEntryCallBack; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; -import org.apache.pulsar.client.impl.Backoff; - -/** - * Entry reader that fulfill read request by streamline the read instead of reading with micro batch. - */ -@Slf4j -public class StreamingEntryReader implements AsyncCallbacks.ReadEntryCallback, WaitingEntryCallBack { - - private final int maxRetry = 3; - - // Queue for read request issued yet waiting for complete from managed ledger. - private ConcurrentLinkedQueue issuedReads = new ConcurrentLinkedQueue<>(); - - // Queue for read request that's wait for new entries from managed ledger. - private ConcurrentLinkedQueue pendingReads = new ConcurrentLinkedQueue<>(); - - private final ManagedCursorImpl cursor; - - private final StreamingDispatcher dispatcher; - - private final PersistentTopic topic; - - private final Executor topicExecutor; - - private final Executor dispatcherExecutor; - - private AtomicInteger currentReadSizeByte = new AtomicInteger(0); - - private volatile State state; - - private static final AtomicReferenceFieldUpdater STATE_UPDATER = - AtomicReferenceFieldUpdater.newUpdater(StreamingEntryReader.class, State.class, "state"); - - private volatile long maxReadSizeByte; - - private final Backoff readFailureBackoff = new Backoff(10, TimeUnit.MILLISECONDS, - 1, TimeUnit.SECONDS, 0, TimeUnit.MILLISECONDS); - - public StreamingEntryReader(ManagedCursorImpl cursor, StreamingDispatcher dispatcher, PersistentTopic topic) { - this.cursor = cursor; - this.dispatcher = dispatcher; - this.topic = topic; - this.topicExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topic.getName()); - this.dispatcherExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(dispatcher.getName()); - } - - /** - * Read entries in streaming way, that said instead of reading with micro batch and send entries to consumer after - * all entries in the batch are read from ledger, this method will fire numEntriesToRead requests to managedLedger - * and send entry to consumer whenever it is read and all entries before it have been sent to consumer. - * @param numEntriesToRead number of entry to read from ledger. - * @param maxReadSizeByte maximum byte will be read from ledger. - * @param ctx Context send along with read request. - */ - public synchronized void asyncReadEntries(int numEntriesToRead, long maxReadSizeByte, Object ctx) { - if (STATE_UPDATER.compareAndSet(this, State.Canceling, State.Canceled)) { - internalCancelReadRequests(); - } - - if (!issuedReads.isEmpty() || !pendingReads.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] There's pending streaming read not completed yet. Not scheduling next read request.", - cursor.getName()); - } - return; - } - - PositionImpl nextReadPosition = (PositionImpl) cursor.getReadPosition(); - ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger(); - // Edge case, when a old ledger is full and new ledger is not yet opened, position can point to next - // position of the last confirmed position, but it'll be an invalid position. So try to update the position. - if (!managedLedger.isValidPosition(nextReadPosition)) { - nextReadPosition = managedLedger.getNextValidPosition(nextReadPosition); - } - boolean hasEntriesToRead = managedLedger.hasMoreEntries(nextReadPosition); - currentReadSizeByte.set(0); - STATE_UPDATER.set(this, State.Issued); - this.maxReadSizeByte = maxReadSizeByte; - for (int c = 0; c < numEntriesToRead; c++) { - PendingReadEntryRequest pendingReadEntryRequest = PendingReadEntryRequest.create(ctx, nextReadPosition); - // Make sure once we start putting request into pending requests queue, we won't put any following request - // to issued requests queue in order to guarantee the order. - if (hasEntriesToRead && managedLedger.hasMoreEntries(nextReadPosition)) { - issuedReads.offer(pendingReadEntryRequest); - } else { - pendingReads.offer(pendingReadEntryRequest); - } - nextReadPosition = managedLedger.getNextValidPosition(nextReadPosition); - } - - // Issue requests. - for (PendingReadEntryRequest request : issuedReads) { - managedLedger.asyncReadEntry(request.position, this, request); - } - - if (!pendingReads.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}} Streaming entry reader has {} pending read requests waiting on new entry." - , cursor.getName(), pendingReads.size()); - } - // If new entries are available after we put request into pending queue, fire read. - // Else register callback with managed ledger to get notify when new entries are available. - if (managedLedger.hasMoreEntries(pendingReads.peek().position)) { - entriesAvailable(); - } else if (managedLedger.isTerminated()) { - dispatcher.notifyConsumersEndOfTopic(); - cleanQueue(pendingReads); - if (issuedReads.size() == 0) { - dispatcher.canReadMoreEntries(true); - } - } else { - managedLedger.addWaitingEntryCallBack(this); - } - } - } - - @Override - public void readEntryComplete(Entry entry, Object ctx) { - // Don't block caller thread, complete read entry with dispatcher dedicated thread. - dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx)); - } - - private void internalReadEntryComplete(Entry entry, Object ctx) { - PendingReadEntryRequest pendingReadEntryRequest = (PendingReadEntryRequest) ctx; - pendingReadEntryRequest.entry = entry; - readFailureBackoff.reduceToHalf(); - Entry readEntry; - // If we have entry to send to dispatcher. - if (!issuedReads.isEmpty() && issuedReads.peek() == pendingReadEntryRequest) { - while (!issuedReads.isEmpty() && issuedReads.peek().entry != null) { - PendingReadEntryRequest firstPendingReadEntryRequest = issuedReads.poll(); - readEntry = firstPendingReadEntryRequest.entry; - currentReadSizeByte.addAndGet(readEntry.getLength()); - //Cancel remaining requests and reset cursor if maxReadSizeByte exceeded. - if (currentReadSizeByte.get() > maxReadSizeByte) { - cancelReadRequests(readEntry.getPosition()); - dispatcher.canReadMoreEntries(false); - STATE_UPDATER.set(this, State.Completed); - return; - } else { - // All request has been completed, mark returned entry as last. - if (issuedReads.isEmpty() && pendingReads.isEmpty()) { - firstPendingReadEntryRequest.isLast = true; - STATE_UPDATER.set(this, State.Completed); - } - dispatcher.readEntryComplete(readEntry, firstPendingReadEntryRequest); - } - } - } else if (!issuedReads.isEmpty() && issuedReads.peek().retry > maxRetry) { - cancelReadRequests(issuedReads.peek().position); - dispatcher.canReadMoreEntries(true); - STATE_UPDATER.set(this, State.Completed); - } - } - - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - // Don't block caller thread, complete read entry fail with dispatcher dedicated thread. - dispatcherExecutor.execute(() -> internalReadEntryFailed(exception, ctx)); - } - - private void internalReadEntryFailed(ManagedLedgerException exception, Object ctx) { - PendingReadEntryRequest pendingReadEntryRequest = (PendingReadEntryRequest) ctx; - PositionImpl readPosition = pendingReadEntryRequest.position; - pendingReadEntryRequest.retry++; - long waitTimeMillis = readFailureBackoff.next(); - if (exception.getCause() instanceof TransactionBufferException.TransactionNotSealedException - || exception.getCause() instanceof ManagedLedgerException.OffloadReadHandleClosedException) { - waitTimeMillis = 1; - if (log.isDebugEnabled()) { - log.debug("[{}] Error reading transaction entries : {}, - Retrying to read in {} seconds", - cursor.getName(), exception.getMessage(), waitTimeMillis / 1000.0); - } - } else if (!(exception instanceof ManagedLedgerException.TooManyRequestsException)) { - log.error("[{} Error reading entries at {} : {} - Retrying to read in {} seconds", cursor.getName(), - readPosition, exception.getMessage(), waitTimeMillis / 1000.0); - } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Got throttled by bookies while reading at {} : {} - Retrying to read in {} seconds", - cursor.getName(), readPosition, exception.getMessage(), waitTimeMillis / 1000.0); - } - } - if (!issuedReads.isEmpty()) { - if (issuedReads.peek().retry > maxRetry) { - cancelReadRequests(issuedReads.peek().position); - dispatcher.canReadMoreEntries(true); - STATE_UPDATER.set(this, State.Completed); - return; - } - if (pendingReadEntryRequest.retry <= maxRetry) { - retryReadRequest(pendingReadEntryRequest, waitTimeMillis); - } - } - } - - // Cancel all issued and pending request and update cursor's read position. - private void cancelReadRequests(Position position) { - if (!issuedReads.isEmpty()) { - cleanQueue(issuedReads); - cursor.seek(position); - } - - if (!pendingReads.isEmpty()) { - cleanQueue(pendingReads); - } - } - - private void internalCancelReadRequests() { - Position readPosition = !issuedReads.isEmpty() ? issuedReads.peek().position : pendingReads.peek().position; - cancelReadRequests(readPosition); - } - - public boolean cancelReadRequests() { - if (STATE_UPDATER.compareAndSet(this, State.Issued, State.Canceling)) { - // Don't block caller thread, complete cancel read with dispatcher dedicated thread. - topicExecutor.execute(() -> { - synchronized (StreamingEntryReader.this) { - if (STATE_UPDATER.compareAndSet(this, State.Canceling, State.Canceled)) { - internalCancelReadRequests(); - } - } - }); - return true; - } - return false; - } - - private void cleanQueue(Queue queue) { - while (!queue.isEmpty()) { - PendingReadEntryRequest pendingReadEntryRequest = queue.poll(); - if (pendingReadEntryRequest.entry != null) { - pendingReadEntryRequest.entry.release(); - pendingReadEntryRequest.recycle(); - } - } - } - - private void retryReadRequest(PendingReadEntryRequest pendingReadEntryRequest, long delay) { - topic.getBrokerService().executor().schedule(() -> { - // Jump again into dispatcher dedicated thread - dispatcherExecutor.execute(() -> { - ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger(); - managedLedger.asyncReadEntry(pendingReadEntryRequest.position, this, pendingReadEntryRequest); - }); - }, delay, TimeUnit.MILLISECONDS); - } - - @Override - public void entriesAvailable() { - dispatcherExecutor.execute(this::internalEntriesAvailable); - } - - private synchronized void internalEntriesAvailable() { - if (log.isDebugEnabled()) { - log.debug("[{}} Streaming entry reader get notification of newly added entries from managed ledger," - + " trying to issued pending read requests.", cursor.getName()); - } - ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger(); - List newlyIssuedRequests = new ArrayList<>(); - if (!pendingReads.isEmpty()) { - // Edge case, when a old ledger is full and new ledger is not yet opened, position can point to next - // position of the last confirmed position, but it'll be an invalid position. So try to update the position. - if (!managedLedger.isValidPosition(pendingReads.peek().position)) { - pendingReads.peek().position = managedLedger.getNextValidPosition(pendingReads.peek().position); - } - while (!pendingReads.isEmpty() && managedLedger.hasMoreEntries(pendingReads.peek().position)) { - PendingReadEntryRequest next = pendingReads.poll(); - issuedReads.offer(next); - newlyIssuedRequests.add(next); - // Need to update the position because when the PendingReadEntryRequest is created, the position could - // be all set to managed ledger's last confirmed position. - if (!pendingReads.isEmpty()) { - pendingReads.peek().position = managedLedger.getNextValidPosition(next.position); - } - } - - for (PendingReadEntryRequest request : newlyIssuedRequests) { - managedLedger.asyncReadEntry(request.position, this, request); - } - - if (!pendingReads.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}} Streaming entry reader has {} pending read requests waiting on new entry." - , cursor.getName(), pendingReads.size()); - } - if (managedLedger.hasMoreEntries(pendingReads.peek().position)) { - entriesAvailable(); - } else { - managedLedger.addWaitingEntryCallBack(this); - } - } - } - } - - protected State getState() { - return STATE_UPDATER.get(this); - } - - enum State { - Issued, Canceling, Canceled, Completed; - } - -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java deleted file mode 100644 index 4d576d9f437f3..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.streamingdispatch; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java deleted file mode 100644 index f86fe0701dc70..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import org.apache.pulsar.broker.service.PersistentDispatcherFailoverConsumerTest; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -/** - * PersistentDispatcherFailoverConsumerTest with {@link StreamingDispatcher} - */ -@Test(groups = "quarantine") -public class PersistentDispatcherFailoverConsumerStreamingDispatcherTest extends PersistentDispatcherFailoverConsumerTest { - - @BeforeMethod(alwaysRun = true) - public void setup() throws Exception { - super.setup(); - pulsarTestContext.getConfig().setStreamingDispatch(true); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java deleted file mode 100644 index 92352cde47ff0..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import org.apache.pulsar.broker.service.PersistentFailoverE2ETest; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -/** - * PersistentFailoverE2ETest with {@link StreamingDispatcher} - */ -@Test(groups = "broker") -public class PersistentFailoverStreamingDispatcherE2ETest extends PersistentFailoverE2ETest { - - @BeforeClass - @Override - protected void setup() throws Exception { - conf.setStreamingDispatch(true); - super.setup(); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java deleted file mode 100644 index ef515cd85bf98..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.apache.pulsar.client.api.DispatcherBlockConsumerTest; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -/** - * DispatcherBlockConsumerTest with {@link StreamingDispatcher} - */ -@Test(groups = "flaky") -public class PersistentStreamingDispatcherBlockConsumerTest extends DispatcherBlockConsumerTest { - - @BeforeMethod - @Override - protected void setup() throws Exception { - super.setup(); - conf.setStreamingDispatch(true); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java deleted file mode 100644 index fc62d84b58634..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.apache.pulsar.client.api.SubscriptionMessageDispatchThrottlingTest; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -/** - * SubscriptionMessageDispatchThrottlingTest with {@link StreamingDispatcher} - */ -@Test(groups = "flaky") -public class PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest - extends SubscriptionMessageDispatchThrottlingTest { - - @BeforeClass - @Override - protected void setup() throws Exception { - super.setup(); - conf.setStreamingDispatch(true); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java deleted file mode 100644 index b4aa4793ba6fc..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import org.apache.pulsar.broker.service.PersistentTopicE2ETest; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.testng.annotations.Test; - -/** - * PersistentTopicE2ETest with {@link StreamingDispatcher} - */ -@Test(groups = "flaky") -public class PersistentTopicStreamingDispatcherE2ETest extends PersistentTopicE2ETest { - - @Override - protected void doInitConf() throws Exception { - super.doInitConf(); - conf.setStreamingDispatch(true); - } - - @Override - @Test - public void testMessageRedelivery() throws Exception { - super.testMessageRedelivery(); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java deleted file mode 100644 index 440cbbe290c4d..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.service.PersistentTopicTest; -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -/** - * PersistentTopicTest with {@link StreamingDispatcher} - */ -@Test(groups = "broker") -public class PersistentTopicStreamingDispatcherTest extends PersistentTopicTest { - - @BeforeMethod(alwaysRun = true) - public void setup() throws Exception { - super.setup(); - ServiceConfiguration config = pulsarTestContext.getConfig(); - config.setTopicLevelPoliciesEnabled(false); - config.setSystemTopicEnabled(false); - config.setStreamingDispatch(true); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java deleted file mode 100644 index 706571eeb712f..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.persistent; - -import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher; -import org.apache.pulsar.client.api.SimpleProducerConsumerTest; -import org.testng.annotations.Test; - -/** - * SimpleProducerConsumerTest with {@link StreamingDispatcher} - */ -@Test(groups = "flaky") -public class SimpleProducerConsumerTestStreamingDispatcherTest extends SimpleProducerConsumerTest { - - @Override - protected void doInitConf() throws Exception { - super.doInitConf(); - conf.setStreamingDispatch(true); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java deleted file mode 100644 index 5eb29b8ef2870..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service.streamingdispatch; - -import static org.awaitility.Awaitility.await; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.bookkeeper.common.util.OrderedExecutor; -import org.apache.bookkeeper.common.util.OrderedScheduler; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.EntryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.test.MockedBookKeeperTestCase; -import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.testng.annotations.Test; - -/** - * Tests for {@link StreamingEntryReader} - */ -@Test(groups = "flaky") -public class StreamingEntryReaderTests extends MockedBookKeeperTestCase { - - private static final Charset Encoding = StandardCharsets.UTF_8; - private PersistentTopic mockTopic; - private StreamingDispatcher mockDispatcher; - private BrokerService mockBrokerService; - private EventLoopGroup eventLoopGroup; - private OrderedExecutor orderedExecutor; - private ManagedLedgerConfig config; - private ManagedLedgerImpl ledger; - private ManagedCursor cursor; - - @Override - protected void setUpTestCase() throws Exception { - eventLoopGroup = new NioEventLoopGroup(1); - orderedExecutor = OrderedScheduler.newSchedulerBuilder() - .numThreads(1) - .name("StreamingEntryReaderTests").build(); - mockTopic = mock(PersistentTopic.class); - mockBrokerService = mock(BrokerService.class); - mockDispatcher = mock(StreamingDispatcher.class); - config = new ManagedLedgerConfig().setMaxEntriesPerLedger(10); - ledger = spy((ManagedLedgerImpl) factory.open("my_test_ledger", config)); - cursor = ledger.openCursor("test"); - when(mockTopic.getBrokerService()).thenReturn(mockBrokerService); - when(mockBrokerService.executor()).thenReturn(eventLoopGroup); - when(mockBrokerService.getTopicOrderedExecutor()).thenReturn(orderedExecutor); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) { - return null; - } - }).when(mockDispatcher).notifyConsumersEndOfTopic(); - } - - @Override - protected void cleanUpTestCase() { - if (eventLoopGroup != null) { - eventLoopGroup.shutdownNow(); - eventLoopGroup = null; - } - if (orderedExecutor != null) { - orderedExecutor.shutdownNow(); - orderedExecutor = null; - } - } - - @Test - public void testCanReadEntryFromMLedgerHappyPath() throws Exception { - AtomicInteger entryCount = new AtomicInteger(0); - Stack positions = new Stack<>(); - - for (int i = 0; i < 150; i++) { - ledger.addEntry(String.format("message-%d", i).getBytes(Encoding)); - } - - StreamingEntryReader streamingEntryReader =new StreamingEntryReader((ManagedCursorImpl) cursor, - mockDispatcher, mockTopic); - - doAnswer((InvocationOnMock invocationOnMock) -> { - Entry entry = invocationOnMock.getArgument(0, Entry.class); - positions.push(entry.getPosition()); - assertEquals(new String(entry.getData()), String.format("message-%d", entryCount.getAndIncrement())); - cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition())); - return null; - } - ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class)); - - streamingEntryReader.asyncReadEntries(50, 700, null); - await().until(() -> entryCount.get() == 50); - // Check cursor's read position has been properly updated - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - streamingEntryReader.asyncReadEntries(50, 700, null); - await().until(() -> entryCount.get() == 100); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - streamingEntryReader.asyncReadEntries(50, 700, null); - await().until(() -> entryCount.get() == 150); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - } - - @Test - public void testCanReadEntryFromMLedgerSizeExceededLimit() throws Exception { - AtomicBoolean readComplete = new AtomicBoolean(false); - Stack positions = new Stack<>(); - List entries = new ArrayList<>(); - int size = "mmmmmmmmmmessage-0".getBytes().length; - for (int i = 0; i < 15; i++) { - ledger.addEntry(String.format("mmmmmmmmmmessage-%d", i).getBytes(Encoding)); - } - - StreamingEntryReader streamingEntryReader = - new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic); - - doAnswer((InvocationOnMock invocationOnMock) -> { - Entry entry = invocationOnMock.getArgument(0, Entry.class); - positions.push(entry.getPosition()); - entries.add(new String(entry.getData())); - cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition())); - return null; - } - ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class)); - - doAnswer((InvocationOnMock invocationOnMock) -> { - readComplete.set(true); - return null; - } - ).when(mockDispatcher).canReadMoreEntries(anyBoolean()); - - PositionImpl position = ledger.getPositionAfterN(ledger.getFirstPosition(), 3, ManagedLedgerImpl.PositionBound.startExcluded); - // Make reading from mledger return out of order. - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class); - executor.schedule(() -> { - cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(), "mmmmmmmmmmessage-2".getBytes()), - invocationOnMock.getArgument(2)); - }, 200, TimeUnit.MILLISECONDS); - return null; - } - }).when(ledger).asyncReadEntry(eq(position), any(), any()); - - // Only 2 entries should be read with this request. - streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null); - await().until(() -> readComplete.get()); - assertEquals(entries.size(), 2); - // Assert cursor's read position has been properly updated to the third entry, since we should only read - // 2 retries with previous request - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - reset(ledger); - readComplete.set(false); - streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null); - await().until(() -> readComplete.get()); - readComplete.set(false); - streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null); - await().until(() -> readComplete.get()); - readComplete.set(false); - streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null); - await().until(() -> readComplete.get()); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - assertEquals(entries.size(), 8); - for (int i = 0; i < entries.size(); i++) { - assertEquals(String.format("mmmmmmmmmmessage-%d", i), entries.get(i)); - } - } - - @Test - public void testCanReadEntryFromMLedgerWaitingForNewEntry() throws Exception { - AtomicInteger entryCount = new AtomicInteger(0); - AtomicBoolean entryProcessed = new AtomicBoolean(false); - Stack positions = new Stack<>(); - List entries = new ArrayList<>(); - for (int i = 0; i < 7; i++) { - ledger.addEntry(String.format("message-%d", i).getBytes(Encoding)); - } - - StreamingEntryReader streamingEntryReader = - new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic); - - doAnswer((InvocationOnMock invocationOnMock) -> { - Entry entry = invocationOnMock.getArgument(0, Entry.class); - positions.push(entry.getPosition()); - entries.add(new String(entry.getData())); - entryCount.getAndIncrement(); - cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition())); - entryProcessed.set(true); - return null; - } - ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class)); - - streamingEntryReader.asyncReadEntries(5, 100, null); - await().until(() -> entryCount.get() == 5); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - streamingEntryReader.asyncReadEntries(5, 100, null); - // We only write 7 entries initially so only 7 entries can be read. - await().until(() -> entryCount.get() == 7); - // Add new entry and await for it to be send to reader. - entryProcessed.set(false); - ledger.addEntry("message-7".getBytes(Encoding)); - await().until(() -> entryProcessed.get()); - assertEquals(entries.size(), 8); - entryProcessed.set(false); - ledger.addEntry("message-8".getBytes(Encoding)); - await().until(() -> entryProcessed.get()); - assertEquals(entries.size(), 9); - entryProcessed.set(false); - ledger.addEntry("message-9".getBytes(Encoding)); - await().until(() -> entryProcessed.get()); - assertEquals(entries.size(), 10); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - for (int i = 0; i < entries.size(); i++) { - assertEquals(String.format("message-%d", i), entries.get(i)); - } - } - - @Test - public void testCanCancelReadEntryRequestAndResumeReading() throws Exception { - Map messages = new HashMap<>(); - AtomicInteger count = new AtomicInteger(0); - Stack positions = new Stack<>(); - List entries = new ArrayList<>(); - - for (int i = 0; i < 20; i++) { - String msg = String.format("message-%d", i); - messages.put(ledger.addEntry(msg.getBytes(Encoding)), msg); - } - - StreamingEntryReader streamingEntryReader = - new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic); - - doAnswer((InvocationOnMock invocationOnMock) -> { - Entry entry = invocationOnMock.getArgument(0, Entry.class); - positions.push(entry.getPosition()); - entries.add(new String(entry.getData())); - cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition())); - return null; - } - ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class)); - - // Only return 5 entries - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) { - AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class); - PositionImpl position = invocationOnMock.getArgument(0, PositionImpl.class); - int c = count.getAndIncrement(); - if (c < 5) { - cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(), - messages.get(position).getBytes()), - invocationOnMock.getArgument(2)); - } - return null; - } - }).when(ledger).asyncReadEntry(any(), any(), any()); - - streamingEntryReader.asyncReadEntries(20, 200, null); - streamingEntryReader.cancelReadRequests(); - await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Canceled); - // Only have 5 entry as we make ledger only return 5 entries and cancel the request. - assertEquals(entries.size(), 5); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - // Clear mock and try to read remaining entries - reset(ledger); - streamingEntryReader.asyncReadEntries(15, 200, null); - streamingEntryReader.cancelReadRequests(); - await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Completed); - // Only have 5 entry as we make ledger only return 5 entries and cancel the request. - assertEquals(entries.size(), 20); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - // Make sure message still returned in order - for (int i = 0; i < entries.size(); i++) { - assertEquals(String.format("message-%d", i), entries.get(i)); - } - } - - @Test - public void testCanHandleExceptionAndRetry() throws Exception { - Map messages = new HashMap<>(); - AtomicBoolean entryProcessed = new AtomicBoolean(false); - AtomicInteger count = new AtomicInteger(0); - Stack positions = new Stack<>(); - List entries = new ArrayList<>(); - for (int i = 0; i < 12; i++) { - String msg = String.format("message-%d", i); - messages.put(ledger.addEntry(msg.getBytes(Encoding)), msg); - } - - StreamingEntryReader streamingEntryReader = - new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic); - - doAnswer((InvocationOnMock invocationOnMock) -> { - Entry entry = invocationOnMock.getArgument(0, Entry.class); - positions.push(entry.getPosition()); - entries.add(new String(entry.getData())); - cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition())); - - if (entries.size() == 6 || entries.size() == 12) { - entryProcessed.set(true); - } - return null; - } - ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class)); - - // Make reading from mledger throw exception randomly. - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class); - PositionImpl position = invocationOnMock.getArgument(0, PositionImpl.class); - int c = count.getAndIncrement(); - if (c >= 3 && c < 5 || c >= 9 && c < 11) { - cb.readEntryFailed(new ManagedLedgerException.TooManyRequestsException("Fake exception."), - invocationOnMock.getArgument(2)); - } else { - cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(), - messages.get(position).getBytes()), - invocationOnMock.getArgument(2)); - } - return null; - } - }).when(ledger).asyncReadEntry(any(), any(), any()); - - streamingEntryReader.asyncReadEntries(6, 100, null); - await().until(() -> entryProcessed.get()); - assertEquals(entries.size(), 6); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - entryProcessed.set(false); - streamingEntryReader.asyncReadEntries(6, 100, null); - await().until(() -> entryProcessed.get()); - assertEquals(entries.size(), 12); - assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek())); - // Make sure message still returned in order - for (int i = 0; i < entries.size(); i++) { - assertEquals(String.format("message-%d", i), entries.get(i)); - } - } - - @Test - public void testWillCancelReadAfterExhaustingRetry() throws Exception { - Map messages = new HashMap<>(); - AtomicInteger count = new AtomicInteger(0); - Stack positions = new Stack<>(); - List entries = new ArrayList<>(); - for (int i = 0; i < 12; i++) { - String msg = String.format("message-%d", i); - messages.put(ledger.addEntry(msg.getBytes(Encoding)), msg); - } - - StreamingEntryReader streamingEntryReader = - new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic); - - doAnswer((InvocationOnMock invocationOnMock) -> { - Entry entry = invocationOnMock.getArgument(0, Entry.class); - positions.push(entry.getPosition()); - cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition())); - entries.add(new String(entry.getData())); - return null; - } - ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class)); - - // Fail after first 3 read. - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class); - PositionImpl position = invocationOnMock.getArgument(0, PositionImpl.class); - int c = count.getAndIncrement(); - if (c >= 3) { - cb.readEntryFailed(new ManagedLedgerException.TooManyRequestsException("Fake exception."), - invocationOnMock.getArgument(2)); - } else { - cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(), - messages.get(position).getBytes()), - invocationOnMock.getArgument(2)); - } - return null; - } - }).when(ledger).asyncReadEntry(any(), any(), any()); - - streamingEntryReader.asyncReadEntries(5, 100, null); - await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Completed); - // Issued 5 read, should only have 3 entries as others were canceled after exhausting retries. - assertEquals(entries.size(), 3); - for (int i = 0; i < entries.size(); i++) { - assertEquals(String.format("message-%d", i), entries.get(i)); - } - reset(ledger); - streamingEntryReader.asyncReadEntries(5, 100, null); - await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Completed); - assertEquals(entries.size(), 8); - for (int i = 0; i < entries.size(); i++) { - assertEquals(String.format("message-%d", i), entries.get(i)); - } - } - -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java index ed85ffd600ec6..4f4affc39d316 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java @@ -1228,7 +1228,6 @@ public void testBacklogConsumerCacheReads() throws Exception { conf.setManagedLedgerMinimumBacklogCursorsForCaching(2); conf.setManagedLedgerMinimumBacklogEntriesForCaching(10); conf.setManagedLedgerCacheEvictionTimeThresholdMillis(60 * 1000); - conf.setStreamingDispatch(false); restartBroker(); final long totalMessages = 200; final int receiverSize = 10; From b0bb5ae1e8606082a0511c71d36b7c6c53d7086d Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 19:20:24 -0500 Subject: [PATCH 206/280] [fix][test] Use delta when comparing doubles in checkLoadReportNicSpeed (#20343) ### Motivation @dave2wave recently encountered this test failure: ``` 20:46:55 [ERROR] Tests run: 2, Failures: 1, Errors: 0, Skipped: 1, Time elapsed: 15.301 s <<< FAILURE! - in org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest 20:46:55 [ERROR] checkLoadReportNicSpeed(org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest) Time elapsed: 0.142 s <<< FAILURE! 20:46:55 java.lang.AssertionError: expected [1.6200000000000004E7] but found [1.62E7] 20:46:55 at org.testng.Assert.fail(Assert.java:99) 20:46:55 at org.testng.Assert.failNotEquals(Assert.java:1037) 20:46:55 at org.testng.Assert.assertEquals(Assert.java:701) 20:46:55 at org.testng.Assert.assertEquals(Assert.java:712) 20:46:55 at org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest.checkLoadReportNicSpeed(LoadReportNetworkLimitTest.java:63) ``` The test failed because we are not providing an acceptable delta for the equality check. It looks like there are other tests where we have strict double or float comparisons. I am going to leave those for now to minimize the diff on this PR. I read through all of the open flaky test issues and none appear to be related to this class of flakiness. ### Modifications * Add a delta to the assertions in the `checkLoadReportNicSpeed` test ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Documentation - [x] `doc-not-needed` --- .../broker/loadbalance/LoadReportNetworkLimitTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java index bec971dfa40e7..ffa2607b6b42e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java @@ -60,12 +60,12 @@ public void checkLoadReportNicSpeed() throws Exception { LoadManagerReport report = admin.brokerStats().getLoadReport(); if (SystemUtils.IS_OS_LINUX) { - assertEquals(report.getBandwidthIn().limit, usableNicCount * 5.4 * 1000 * 1000); - assertEquals(report.getBandwidthOut().limit, usableNicCount * 5.4 * 1000 * 1000); + assertEquals(report.getBandwidthIn().limit, usableNicCount * 5.4 * 1000 * 1000, 0.0001); + assertEquals(report.getBandwidthOut().limit, usableNicCount * 5.4 * 1000 * 1000, 0.0001); } else { // On non-Linux system we don't report the network usage - assertEquals(report.getBandwidthIn().limit, -1.0); - assertEquals(report.getBandwidthOut().limit, -1.0); + assertEquals(report.getBandwidthIn().limit, -1.0, 0.0001); + assertEquals(report.getBandwidthOut().limit, -1.0, 0.0001); } } From e53fcaac3fde564e883fcc63baf74da60371ab5e Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 20:33:16 -0500 Subject: [PATCH 207/280] [improve][sec] Suppress etcd CVE warnings (#20342) --- src/owasp-dependency-check-suppressions.xml | 68 +++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index a59b3b999e2bf..5a595af2c0a79 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -229,6 +229,74 @@ CVE-2021-42550 + + + + 861af62ae22a71d30f401a80049397fe7ff44423 + CVE-2020-15113 + + + + + 663f2ccc0ec7797954c333fa75feeb7d559948b0 + CVE-2020-15113 + + + + + abd0ffcd4e66046057c3bfb34affc0de870a038b + CVE-2020-15113 + + + + + a2e99802fec5586daca0c4daae975fe601a057a8 + CVE-2017-8359 + + + + a2e99802fec5586daca0c4daae975fe601a057a8 + CVE-2020-15113 + + + + a2e99802fec5586daca0c4daae975fe601a057a8 + CVE-2020-7768 + + + + a2e99802fec5586daca0c4daae975fe601a057a8 + CVE-2017-7861 + + + + a2e99802fec5586daca0c4daae975fe601a057a8 + CVE-2017-9431 + + + + a2e99802fec5586daca0c4daae975fe601a057a8 + CVE-2017-7860 + + Date: Thu, 18 May 2023 11:15:53 +0800 Subject: [PATCH 208/280] [fix][broker]Fix deadlock of metadata store (#20189) Motivation: This task loadOrCreatePersistentTopic occupied the event thread of the ZK client so that other ZK tasks could not be finished anymore(Including the task itself), and it calls bundlesCache.synchronous().get(nsname) which is a blocking method. Modification: Since the method getBundle(topic) will eventually call the method bundlesCache.synchronous().get(nsname), use getBundleAsync(topic) instead of getBundle(topic) to avoid blocking the thread. --- .../broker/namespace/NamespaceService.java | 32 +++++++++++-------- .../broker/namespace/OwnershipCache.java | 6 ++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index a0e2bf7534c02..9d8d9e3890a19 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -38,7 +38,9 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; @@ -1137,16 +1139,17 @@ public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) new IllegalArgumentException("Invalid class of NamespaceBundle: " + suName.getClass().getName())); } + /** + * @Deprecated This method is only used in test now. + */ + @Deprecated public boolean isServiceUnitActive(TopicName topicName) { try { - OwnedBundle ownedBundle = ownershipCache.getOwnedBundle(getBundle(topicName)); - if (ownedBundle == null) { - return false; - } - return ownedBundle.isActive(); - } catch (Exception e) { - LOG.warn("Unable to find OwnedBundle for topic - [{}]", topicName, e); - return false; + return isServiceUnitActiveAsync(topicName).get(pulsar.getConfig() + .getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + LOG.warn("Unable to find OwnedBundle for topic in time - [{}]", topicName, e); + throw new RuntimeException(e); } } @@ -1156,12 +1159,13 @@ public CompletableFuture isServiceUnitActiveAsync(TopicName topicName) return getBundleAsync(topicName) .thenCompose(bundle -> loadManager.get().checkOwnershipAsync(Optional.of(topicName), bundle)); } - Optional> res = ownershipCache.getOwnedBundleAsync(getBundle(topicName)); - if (!res.isPresent()) { - return CompletableFuture.completedFuture(false); - } - - return res.get().thenApply(ob -> ob != null && ob.isActive()); + return getBundleAsync(topicName).thenCompose(bundle -> { + Optional> optionalFuture = ownershipCache.getOwnedBundleAsync(bundle); + if (!optionalFuture.isPresent()) { + return CompletableFuture.completedFuture(false); + } + return optionalFuture.get().thenApply(ob -> ob != null && ob.isActive()); + }); } private boolean isNamespaceOwned(NamespaceName fqnn) throws Exception { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index 7d0b5a4147721..86003153714cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -288,10 +288,10 @@ public Optional> getOwnedBundleAsync(NamespaceBun /** * Disable bundle in local cache and on zk. - * - * @param bundle - * @throws Exception + * @Deprecated This is a dangerous method which is currently only used for test, it will occupy the ZK thread. + * Please switch to your own thread after calling this method. */ + @Deprecated public CompletableFuture disableOwnership(NamespaceBundle bundle) { return updateBundleState(bundle, false) .thenCompose(__ -> { From 908d0b3f459167a3c3df2a85c23096df336427da Mon Sep 17 00:00:00 2001 From: Raghavender Mittapalli <14803749+syk-coder@users.noreply.github.com> Date: Thu, 18 May 2023 09:10:25 +0530 Subject: [PATCH 209/280] [fix][broker] Use user-specified bundle size on creating a namespace anti-affinity group with the default local policies (#20327) --- .../apache/pulsar/broker/admin/impl/NamespacesBase.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 8c72d0b0286d3..97029eb5ce128 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -1710,7 +1710,8 @@ protected String internalGetNamespaceAntiAffinityGroup() { try { return getLocalPolicies() .getLocalPolicies(namespaceName) - .orElse(new LocalPolicies()).namespaceAntiAffinityGroup; + .orElseGet(() -> new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()) + , null, null)).namespaceAntiAffinityGroup; } catch (Exception e) { log.error("[{}] Failed to get the antiAffinityGroup of namespace {}", clientAppId(), namespaceName, e); throw new RestException(Status.NOT_FOUND, "Couldn't find namespace policies"); @@ -1760,7 +1761,9 @@ protected List internalGetAntiAffinityNamespaces(String cluster, String throw new RuntimeException(e); } - String storedAntiAffinityGroup = policies.orElse(new LocalPolicies()).namespaceAntiAffinityGroup; + String storedAntiAffinityGroup = policies.orElseGet(() -> + new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()), + null, null)).namespaceAntiAffinityGroup; return antiAffinityGroup.equalsIgnoreCase(storedAntiAffinityGroup); }).collect(Collectors.toList()); From e4f1f095fef7e105cb7a3c5a1a8c2f9449888052 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 18 May 2023 00:27:54 -0500 Subject: [PATCH 210/280] [improve][ws] Use allowTopicOperationAsync for authz checks (#20299) Similar to: https://github.com/apache/pulsar/pull/20142 ### Motivation In https://github.com/apache/pulsar/pull/20142 we changed the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method. Our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, so this change helps us move in the right direction. This PR follows the same logic and updates the WebSocket proxy to remove all calls to the `can*` methods in the `AuthorizationService` ### Modifications * Update `ProducerHandler`, `ConsumerHandler`, and `ReaderHander` in the WebSocket Proxy to call the `AuthorizationService#allowTopicOperationAsync` method. ### Verifying this change This change is trivial. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping PR as I ran tests locally. --- .../pulsar/websocket/ConsumerHandler.java | 20 +++++++++++++++++-- .../pulsar/websocket/ProducerHandler.java | 16 ++++++++++++++- .../pulsar/websocket/ReaderHandler.java | 20 +++++++++++++++++-- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index c988fd1e70ce3..e326f2f02a901 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -19,6 +19,7 @@ package org.apache.pulsar.websocket; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.concurrent.TimeUnit.SECONDS; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Enums; import com.google.common.base.Splitter; @@ -33,6 +34,7 @@ import java.util.concurrent.atomic.LongAdder; import javax.servlet.http.HttpServletRequest; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; @@ -47,6 +49,7 @@ import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.impl.ConsumerBuilderImpl; import org.apache.pulsar.client.impl.TopicMessageIdImpl; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.websocket.data.ConsumerCommand; @@ -467,8 +470,21 @@ protected ConsumerBuilder getConsumerConfiguration(PulsarClient client) @Override protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception { - return service.getAuthorizationService().canConsume(topic, authRole, authenticationData, - this.subscription); + try { + AuthenticationDataSubscription subscription = new AuthenticationDataSubscription(authenticationData, + this.subscription); + return service.getAuthorizationService() + .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription) + .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } catch (InterruptedException e) { + log.warn("Time-out {} sec while checking authorization on {} ", + service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic); + throw e; + } catch (Exception e) { + log.warn("Consumer-client with Role - {} failed to get permissions for topic - {}. {}", authRole, topic, + e.getMessage()); + throw e; + } } public static String extractSubscription(HttpServletRequest request) { diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java index 1dc3f202fe07d..f18d46e285617 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.pulsar.websocket.WebSocketError.FailedToDeserializeFromJSON; import static org.apache.pulsar.websocket.WebSocketError.PayloadEncodingError; import static org.apache.pulsar.websocket.WebSocketError.UnknownError; @@ -45,6 +46,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.websocket.data.ProducerAck; @@ -242,7 +244,19 @@ public long getMsgPublishedCounter() { @Override protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception { - return service.getAuthorizationService().canProduce(topic, authRole, authenticationData); + try { + return service.getAuthorizationService() + .allowTopicOperationAsync(topic, TopicOperation.PRODUCE, authRole, authenticationData) + .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } catch (InterruptedException e) { + log.warn("Time-out {} sec while checking authorization on {} ", + service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic); + throw e; + } catch (Exception e) { + log.warn("Producer-client with Role - {} failed to get permissions for topic - {}. {}", authRole, topic, + e.getMessage()); + throw e; + } } private void sendAckResponse(ProducerAck response) { diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java index 890f426a9bbf3..3d726b9f02f76 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.websocket; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.StringUtils.isNotBlank; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; @@ -28,6 +29,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.MessageId; @@ -38,6 +40,7 @@ import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MultiTopicsReaderImpl; import org.apache.pulsar.client.impl.ReaderImpl; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.websocket.data.ConsumerCommand; import org.apache.pulsar.websocket.data.ConsumerMessage; @@ -310,8 +313,21 @@ protected void updateDeliverMsgStat(long msgSize) { @Override protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception { - return service.getAuthorizationService().canConsume(topic, authRole, authenticationData, - this.subscription); + try { + AuthenticationDataSubscription subscription = new AuthenticationDataSubscription(authenticationData, + this.subscription); + return service.getAuthorizationService() + .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription) + .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } catch (InterruptedException e) { + log.warn("Time-out {} sec while checking authorization on {} ", + service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic); + throw e; + } catch (Exception e) { + log.warn("Consumer-client with Role - {} failed to get permissions for topic - {}. {}", authRole, topic, + e.getMessage()); + throw e; + } } private int getReceiverQueueSize() { From 65f61121e99b504294cf375f319f7946ff4c50fd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 18 May 2023 09:46:52 -0500 Subject: [PATCH 211/280] [cleanup] Catch TimeoutException when logging about time outs (#20349) Relates to: #20299 ### Motivation We have several catch blocks that are dedicated to providing meaningful logs about timeouts. As such, we should catch the `TimeoutException`, not the `InterruptedException`, in order to ensure accurate logs. ### Modifications * Replace `InterruptedException` with `TimeoutException` ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Documentation - [x] `doc-not-needed` --- .../pulsar/broker/authorization/AuthorizationService.java | 7 ++++--- .../java/org/apache/pulsar/broker/rest/TopicsBase.java | 3 ++- .../java/org/apache/pulsar/websocket/ConsumerHandler.java | 3 ++- .../java/org/apache/pulsar/websocket/ProducerHandler.java | 3 ++- .../java/org/apache/pulsar/websocket/ReaderHandler.java | 3 ++- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 0c61219b57a50..6f303e2117fe0 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; @@ -221,7 +222,7 @@ public boolean canProduce(TopicName topicName, String role, AuthenticationDataSo try { return canProduceAsync(topicName, role, authenticationData).get( conf.getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } catch (InterruptedException e) { + } catch (TimeoutException e) { log.warn("Time-out {} sec while checking authorization on {} ", conf.getMetadataStoreOperationTimeoutSeconds(), topicName); throw e; @@ -237,7 +238,7 @@ public boolean canConsume(TopicName topicName, String role, AuthenticationDataSo try { return canConsumeAsync(topicName, role, authenticationData, subscription) .get(conf.getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } catch (InterruptedException e) { + } catch (TimeoutException e) { log.warn("Time-out {} sec while checking authorization on {} ", conf.getMetadataStoreOperationTimeoutSeconds(), topicName); throw e; @@ -263,7 +264,7 @@ public boolean canLookup(TopicName topicName, String role, AuthenticationDataSou try { return canLookupAsync(topicName, role, authenticationData) .get(conf.getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } catch (InterruptedException e) { + } catch (TimeoutException e) { log.warn("Time-out {} sec while checking authorization on {} ", conf.getMetadataStoreOperationTimeoutSeconds(), topicName); throw e; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 7d3aa37fa4ec5..6f3ac7f8c09ca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.core.Response; @@ -771,7 +772,7 @@ && pulsar().getBrokerService().isAuthorizationEnabled()) { isAuthorized = pulsar().getBrokerService().getAuthorizationService() .allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, authParams) .get(config().getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } catch (InterruptedException e) { + } catch (TimeoutException e) { log.warn("Time-out {} sec while checking authorization on {} ", config().getMetadataStoreOperationTimeoutSeconds(), topicName); throw new RestException(Status.INTERNAL_SERVER_ERROR, "Time-out while checking authorization"); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index e326f2f02a901..2ab62b10ee9ee 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -29,6 +29,7 @@ import java.util.Base64; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; @@ -476,7 +477,7 @@ protected Boolean isAuthorized(String authRole, AuthenticationDataSource authent return service.getAuthorizationService() .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription) .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } catch (InterruptedException e) { + } catch (TimeoutException e) { log.warn("Time-out {} sec while checking authorization on {} ", service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic); throw e; diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java index f18d46e285617..5ad1283fe84c4 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java @@ -34,6 +34,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import javax.servlet.http.HttpServletRequest; @@ -248,7 +249,7 @@ protected Boolean isAuthorized(String authRole, AuthenticationDataSource authent return service.getAuthorizationService() .allowTopicOperationAsync(topic, TopicOperation.PRODUCE, authRole, authenticationData) .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } catch (InterruptedException e) { + } catch (TimeoutException e) { log.warn("Time-out {} sec while checking authorization on {} ", service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic); throw e; diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java index 3d726b9f02f76..2f985b2076da2 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.util.Base64; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; @@ -319,7 +320,7 @@ protected Boolean isAuthorized(String authRole, AuthenticationDataSource authent return service.getAuthorizationService() .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription) .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); - } catch (InterruptedException e) { + } catch (TimeoutException e) { log.warn("Time-out {} sec while checking authorization on {} ", service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic); throw e; From c0757254a0ea2625ffc6d97687a715201e20ef2f Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 18 May 2023 23:19:10 +0800 Subject: [PATCH 212/280] [improve][admin] Return BAD_REQUEST on cluster data is null for createCluster (#20346) --- .../java/org/apache/pulsar/broker/admin/impl/ClustersBase.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java index e61a7f20d1d67..5d4ed54c33466 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java @@ -162,6 +162,9 @@ public void createCluster( .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) .thenCompose(__ -> { NamedEntity.checkName(cluster); + if (clusterData == null) { + throw new RestException(Status.BAD_REQUEST, "cluster data is required"); + } try { clusterData.checkPropertiesIfPresent(); } catch (IllegalArgumentException ex) { From d0d626be8f59107d9de624f813dd0d5012cc1311 Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Fri, 19 May 2023 00:21:37 +0900 Subject: [PATCH 213/280] [improve][cli] Allow pulser-client consume create a replicated subscription (#20316) --- .../main/java/org/apache/pulsar/client/cli/CmdConsume.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java index 58ab6360a17bb..0c65604cbe6b8 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java @@ -109,6 +109,9 @@ public class CmdConsume extends AbstractCmdConsume { @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1) private boolean poolMessages = true; + @Parameter(names = {"-rs", "--replicated" }, description = "Whether the subscription status should be replicated") + private boolean replicateSubscriptionState = false; + public CmdConsume() { // Do nothing super(); @@ -156,7 +159,8 @@ private int consume(String topic) { .subscriptionType(subscriptionType) .subscriptionMode(subscriptionMode) .subscriptionInitialPosition(subscriptionInitialPosition) - .poolMessages(poolMessages); + .poolMessages(poolMessages) + .replicateSubscriptionState(replicateSubscriptionState); if (isRegex) { builder.topicsPattern(Pattern.compile(topic)); From 43a989862f548fa3f67708a5fff62eb764af878c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Mikl=C3=B3s?= Date: Thu, 18 May 2023 17:23:55 +0200 Subject: [PATCH 214/280] [improve][fn] Use functions classloader in TopicSchema.newSchemaInstance() to fix ClassNotFoundException when using custom SerDe classes. (targeted for master) (#20115) --- .../pulsar/functions/instance/ContextImpl.java | 2 +- .../apache/pulsar/functions/sink/PulsarSink.java | 2 +- .../pulsar/functions/source/PulsarSource.java | 6 +++--- .../source/SingleConsumerPulsarSource.java | 11 ++++------- .../pulsar/functions/source/TopicSchema.java | 15 ++++++++++++--- .../pulsar/functions/source/TopicSchemaTest.java | 8 +++++--- .../functions/PulsarFunctionsTest.java | 8 ++++---- .../functions/PulsarFunctionsTestBase.java | 2 +- .../functions/java/PulsarFunctionsJavaTest.java | 11 +++++++---- .../java/PulsarWorkerRebalanceDrainTest.java | 15 +++++++++------ .../functions/utils/CommandGenerator.java | 10 +++++----- 11 files changed, 52 insertions(+), 38 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java index d64c5f9b52db6..5cbbcad24c7a2 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java @@ -148,7 +148,7 @@ public ContextImpl(InstanceConfig config, Logger logger, PulsarClient client, this.clientBuilder = clientBuilder; this.client = client; this.pulsarAdmin = pulsarAdmin; - this.topicSchema = new TopicSchema(client); + this.topicSchema = new TopicSchema(client, Thread.currentThread().getContextClassLoader()); this.statsManager = statsManager; this.producerBuilder = (ProducerBuilderImpl) client.newProducer().blockIfQueueFull(true).enableBatching(true) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 8add0a78c5fff..97a0ad0a2ce17 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -349,7 +349,7 @@ public PulsarSink(PulsarClient client, PulsarSinkConfig pulsarSinkConfig, Map buildPulsarSourceConsumerConfig(String t Class typeArg) { PulsarSourceConsumerConfig.PulsarSourceConsumerConfigBuilder consumerConfBuilder = PulsarSourceConsumerConfig.builder().isRegexPattern(conf.isRegexPattern()) - .receiverQueueSize(conf.getReceiverQueueSize()) - .consumerProperties(conf.getConsumerProperties()); + .receiverQueueSize(conf.getReceiverQueueSize()) + .consumerProperties(conf.getConsumerProperties()); Schema schema; if (conf.getSerdeClassName() != null && !conf.getSerdeClassName().isEmpty()) { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java index 426723804cad1..d4d3ea00b9317 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java @@ -44,14 +44,12 @@ public class SingleConsumerPulsarSource extends PulsarSource { private Consumer consumer; private final List> inputConsumers = new LinkedList<>(); - public SingleConsumerPulsarSource(PulsarClient pulsarClient, - SingleConsumerPulsarSourceConfig pulsarSourceConfig, - Map properties, - ClassLoader functionClassLoader) { + public SingleConsumerPulsarSource(PulsarClient pulsarClient, SingleConsumerPulsarSourceConfig pulsarSourceConfig, + Map properties, ClassLoader functionClassLoader) { super(pulsarClient, pulsarSourceConfig, properties, functionClassLoader); this.pulsarClient = pulsarClient; this.pulsarSourceConfig = pulsarSourceConfig; - this.topicSchema = new TopicSchema(pulsarClient); + this.topicSchema = new TopicSchema(pulsarClient, functionClassLoader); this.properties = properties; this.functionClassLoader = functionClassLoader; } @@ -60,8 +58,7 @@ public SingleConsumerPulsarSource(PulsarClient pulsarClient, public void open(Map config, SourceContext sourceContext) throws Exception { log.info("Opening pulsar source with config: {}", pulsarSourceConfig); - Class typeArg = Reflections.loadClass(this.pulsarSourceConfig.getTypeClassName(), - this.functionClassLoader); + Class typeArg = Reflections.loadClass(this.pulsarSourceConfig.getTypeClassName(), this.functionClassLoader); checkArgument(!Void.class.equals(typeArg), "Input type of Pulsar Function cannot be Void"); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index 8ed177cba3b19..d8ae6b19f4a7e 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -19,7 +19,11 @@ package org.apache.pulsar.functions.source; import io.netty.buffer.ByteBuf; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.ByteBuffer; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -47,8 +51,13 @@ public class TopicSchema { private final Map> cachedSchemas = new HashMap<>(); private final PulsarClient client; - public TopicSchema(PulsarClient client) { + private final ClassLoader functionsClassloader; + + public TopicSchema(PulsarClient client, ClassLoader functionsClassloader) { this.client = client; + this.functionsClassloader = AccessController.doPrivileged( + (PrivilegedAction) () -> new URLClassLoader(new URL[0], functionsClassloader) + ); } /** @@ -242,11 +251,11 @@ private Schema newSchemaInstance(String topic, Class clazz, ConsumerCo @SuppressWarnings("unchecked") private Schema newSchemaInstance(String topic, Class clazz, String schemaTypeOrClassName, boolean input) { return newSchemaInstance(topic, clazz, new ConsumerConfig(schemaTypeOrClassName), input, - Thread.currentThread().getContextClassLoader()); + functionsClassloader); } @SuppressWarnings("unchecked") private Schema newSchemaInstance(String topic, Class clazz, ConsumerConfig conf, boolean input) { - return newSchemaInstance(topic, clazz, conf, input, Thread.currentThread().getContextClassLoader()); + return newSchemaInstance(topic, clazz, conf, input, functionsClassloader); } } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java index 506ef67ab0740..c4e77cb3ff85d 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.functions.source; -import static org.testng.Assert.assertEquals; -import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -30,12 +28,16 @@ import org.apache.pulsar.functions.proto.Request; import org.testng.annotations.Test; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; + @Slf4j public class TopicSchemaTest { @Test public void testGetSchema() { - TopicSchema topicSchema = new TopicSchema(null); + TopicSchema topicSchema = new TopicSchema(null, Thread.currentThread().getContextClassLoader()); String TOPIC = "public/default/test"; Schema schema = topicSchema.getSchema(TOPIC + "1", DummyClass.class, Optional.of(SchemaType.JSON)); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 6088628aac52b..1e54764ad5d2b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -24,10 +24,10 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.swagger.util.Json; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -43,8 +43,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - -import io.swagger.util.Json; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -77,8 +75,8 @@ import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.api.examples.AutoSchemaFunction; import org.apache.pulsar.functions.api.examples.AvroSchemaTestFunction; -import org.apache.pulsar.functions.api.examples.MergeTopicFunction; import org.apache.pulsar.functions.api.examples.InitializableFunction; +import org.apache.pulsar.functions.api.examples.MergeTopicFunction; import org.apache.pulsar.functions.api.examples.RecordFunction; import org.apache.pulsar.functions.api.examples.pojo.AvroTestObject; import org.apache.pulsar.functions.api.examples.pojo.Users; @@ -902,6 +900,7 @@ protected void submitFunction(Runtime runtime, String functionName, String functionFile, String functionClass, + Map inputSerdeClassNames, String outputSerdeClassName, Map userConfigs) throws Exception { @@ -916,6 +915,7 @@ protected void submitFunction(Runtime runtime, } generator.setSinkTopic(outputTopicName); generator.setFunctionName(functionName); + generator.setCustomSerDeSourceTopics(inputSerdeClassNames); generator.setOutputSerDe(outputSerdeClassName); if (userConfigs != null) { generator.setUserConfig(userConfigs); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 62aa36da1b6be..288ced63ae5eb 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -54,7 +54,7 @@ public abstract class PulsarFunctionsTestBase extends PulsarTestSuite { public static final String SERDE_JAVA_CLASS = "org.apache.pulsar.functions.api.examples.CustomBaseToBaseFunction"; - public static final String SERDE_OUTPUT_CLASS = + public static final String SERDE_CLASS = "org.apache.pulsar.functions.api.examples.CustomBaseSerde"; public static final String EXCLAMATION_PYTHON_CLASS = diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java index 097a452937d27..939d6e19d1fb1 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java @@ -19,9 +19,9 @@ package org.apache.pulsar.tests.integration.functions.java; import static org.testng.Assert.assertEquals; - import java.util.Collections; - +import java.util.Map; +import org.apache.commons.collections4.map.HashedMap; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.policies.data.FunctionStatus; import org.apache.pulsar.common.policies.data.FunctionStatusUtil; @@ -70,10 +70,13 @@ private void testCustomSerdeFunction() throws Exception { admin.topics().createNonPartitionedTopic(outputTopicName); } + Map inputTopicsSerde = new HashedMap<>(); + inputTopicsSerde.put(inputTopicName, SERDE_CLASS); + String functionName = "test-serde-fn-" + randomName(8); submitFunction( - Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, - SERDE_OUTPUT_CLASS, Collections.singletonMap("serde-topic", outputTopicName) + Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, inputTopicsSerde, + SERDE_CLASS, Collections.singletonMap("serde-topic", outputTopicName) ); // get function info diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java index a6f54349f2689..de29cbc94c295 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.tests.integration.functions.java; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.databind.MappingIterator; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -26,10 +29,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - -import com.fasterxml.jackson.databind.MappingIterator; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.commons.collections4.map.HashedMap; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.FunctionStatus; @@ -41,8 +43,6 @@ import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testng.annotations.Test; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; @Slf4j public abstract class PulsarWorkerRebalanceDrainTest extends PulsarFunctionsTest { @@ -251,9 +251,12 @@ private void createFunctionWorker(String functionName, String topicPrefix) throw admin.topics().createNonPartitionedTopic(outputTopicName); } + Map inputTopicsSerde = new HashedMap<>(); + inputTopicsSerde.put(inputTopicName, SERDE_CLASS); + submitFunction( - Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, - SERDE_OUTPUT_CLASS, Collections.singletonMap(topicPrefix, outputTopicName) + Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, inputTopicsSerde, + SERDE_CLASS, Collections.singletonMap(topicPrefix, outputTopicName) ); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java index 90fac7a055268..adc791fab4dc8 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java @@ -46,7 +46,7 @@ public enum Runtime { private String functionClassName; private String sourceTopic; private String sourceTopicPattern; - private Map customSereSourceTopics; + private Map customSerDeSourceTopics; private String sinkTopic; private String logTopic; private String outputSerDe; @@ -182,8 +182,8 @@ public String generateCreateFunctionCommand(String codeFile) { if (batchBuilder != null) { commandBuilder.append("--batch-builder" + batchBuilder); } - if (customSereSourceTopics != null && !customSereSourceTopics.isEmpty()) { - commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSereSourceTopics) + "\'"); + if (customSerDeSourceTopics != null && !customSerDeSourceTopics.isEmpty()) { + commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSerDeSourceTopics) + "\'"); } if (sinkTopic != null) { commandBuilder.append(" --output " + sinkTopic); @@ -280,8 +280,8 @@ public String generateUpdateFunctionCommand(String codeFile) { if (StringUtils.isNotEmpty(sourceTopic)) { commandBuilder.append(" --inputs " + sourceTopic); } - if (customSereSourceTopics != null && !customSereSourceTopics.isEmpty()) { - commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSereSourceTopics) + "\'"); + if (customSerDeSourceTopics != null && !customSerDeSourceTopics.isEmpty()) { + commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSerDeSourceTopics) + "\'"); } if (batchBuilder != null) { commandBuilder.append("--batch-builder" + batchBuilder); From 2ebb3797c3f371c3ca22cbc8002a8110e3e3fa47 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 18 May 2023 13:13:13 -0500 Subject: [PATCH 215/280] [fix][test] ProxyWithoutServiceDiscoveryTest should enable authz (#20348) --- .../ProxyWithoutServiceDiscoveryTest.java | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java index 6a61decfcc693..9c8e2ba33c9e8 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java @@ -54,13 +54,6 @@ public class ProxyWithoutServiceDiscoveryTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyWithoutServiceDiscoveryTest.class); - - private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem"; - private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; - private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); @@ -70,22 +63,27 @@ protected void setup() throws Exception { // enable tls and auth&auth at broker conf.setAuthenticationEnabled(true); - conf.setAuthorizationEnabled(false); + conf.setAuthorizationEnabled(true); conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - conf.setTlsAllowInsecureConnection(true); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); Set superUserRoles = new HashSet<>(); - superUserRoles.add("superUser"); + superUserRoles.add("admin"); + superUserRoles.add("superproxy"); conf.setSuperUserRoles(superUserRoles); + Set proxyRoles = new HashSet<>(); + proxyRoles.add("superproxy"); + conf.setProxyRoles(proxyRoles); + + conf.setBrokerClientTlsEnabled(true); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); - conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SERVER_KEY_FILE_PATH); + conf.setBrokerClientAuthenticationParameters(String.format("tlsCertFile:%s,tlsKeyFile:%s", + getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"))); Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); @@ -110,14 +108,14 @@ protected void setup() throws Exception { proxyConfig.setTlsEnabledWithBroker(true); // enable tls and auth&auth at proxy - proxyConfig.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - proxyConfig.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - proxyConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH); + proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH); + proxyConfig.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); - proxyConfig.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); - proxyConfig.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + proxyConfig.setBrokerClientAuthenticationParameters(String.format("tlsCertFile:%s,tlsKeyFile:%s", + getTlsFileForClient("superproxy.cert"), getTlsFileForClient("superproxy.key-pk8"))); + proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setAuthenticationProviders(providers); @@ -137,7 +135,7 @@ protected void cleanup() throws Exception { /** * 

    -     * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
+     * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker)
      *
      * 1. client connects to proxy over tls and pass auth-data
      * 2. proxy authenticate client and retrieve client-role
@@ -154,8 +152,8 @@ public void testDiscoveryService() throws Exception {
         log.info("-- Starting {} test --", methodName);
 
         Map authParams = Maps.newHashMap();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         // create a client which connects to proxy over tls and pass authData
@@ -198,10 +196,10 @@ public void testDiscoveryService() throws Exception {
     }
 
     protected final PulsarClient createPulsarClient(Authentication auth, String lookupUrl) throws Exception {
-        admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
-                .allowTlsInsecureConnection(true).authentication(auth).build());
+        admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()).tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
+                .authentication(auth).build());
         return PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).authentication(auth)
                 .enableTls(true).build();
     }
 

From d565c953a7deab1ea93949bfefa44477e51e417f Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Fri, 19 May 2023 11:19:14 -0500
Subject: [PATCH 216/280] [cleanup] Consolidate certs in broker (and some
 proxy) tests (#20353)

---
 .../pulsar/broker/admin/AdminApiTest.java     |   9 +-
 .../broker/admin/v1/V1_AdminApiTest.java      |   9 +-
 .../broker/service/BrokerServiceTest.java     |  41 +++---
 .../pulsar/broker/web/WebServiceTest.java     |  30 +++--
 .../client/api/BrokerServiceLookupTest.java   |  44 ++----
 .../worker/PulsarFunctionTlsTest.java         |  16 ++-
 .../authentication/tls/broker-cert.pem        | 117 ----------------
 .../authentication/tls/broker-key.pem         |  27 ----
 .../resources/authentication/tls/cacert.pem   | 127 ------------------
 .../authentication/tls/client-cert.pem        |  90 -------------
 .../authentication/tls/client-key.pem         |  27 ----
 .../src/test/resources/certificate/client.crt |  20 ---
 .../src/test/resources/certificate/client.csr |  17 ---
 .../src/test/resources/certificate/client.key |  28 ----
 .../src/test/resources/certificate/server.crt |  20 ---
 .../src/test/resources/certificate/server.csr |  17 ---
 .../src/test/resources/certificate/server.key |  28 ----
 .../client/cli/PulsarClientToolTest.java      |   9 +-
 .../server/ProxyServiceTlsStarterTest.java    |  16 +--
 .../pulsar/proxy/server/ProxyTlsTest.java     |  12 +-
 .../proxy/server/ProxyTlsTestWithAuth.java    |   8 +-
 21 files changed, 84 insertions(+), 628 deletions(-)
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/broker-key.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/cacert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/client-cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/client-key.pem
 delete mode 100644 pulsar-broker/src/test/resources/certificate/client.crt
 delete mode 100644 pulsar-broker/src/test/resources/certificate/client.csr
 delete mode 100644 pulsar-broker/src/test/resources/certificate/client.key
 delete mode 100644 pulsar-broker/src/test/resources/certificate/server.crt
 delete mode 100644 pulsar-broker/src/test/resources/certificate/server.csr
 delete mode 100644 pulsar-broker/src/test/resources/certificate/server.key

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index f0ac63f08ae4a..855343e18a24b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -154,9 +154,6 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {
 
     private static final Logger LOG = LoggerFactory.getLogger(AdminApiTest.class);
 
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-
     private MockedPulsarService mockPulsarSetup;
 
     private PulsarService otherPulsar;
@@ -185,8 +182,8 @@ private void applyDefaultConfig() {
         conf.setLoadBalancerEnabled(true);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setMessageExpiryCheckIntervalInMinutes(1);
         conf.setSubscriptionExpiryCheckIntervalInMinutes(1);
         conf.setBrokerDeleteInactiveTopicsEnabled(false);
@@ -203,7 +200,7 @@ private void setupConfigAndStart(java.util.function.Consumer authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         PulsarClient pulsarClient = null;
 
@@ -852,15 +847,15 @@ public void testTlsAuthDisallowInsecure() throws Exception {
         conf.setAuthenticationProviders(providers);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsAllowInsecureConnection(false);
         conf.setNumExecutorThreadPoolSize(5);
         restartBroker();
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         PulsarClient pulsarClient = null;
 
@@ -914,16 +909,16 @@ public void testTlsAuthUseTrustCert() throws Exception {
         conf.setAuthenticationProviders(providers);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsAllowInsecureConnection(false);
-        conf.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setNumExecutorThreadPoolSize(5);
         restartBroker();
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         PulsarClient pulsarClient = null;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
index ca8efe9d1cc79..b069d31dc6e0d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
@@ -63,6 +63,7 @@
 import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.common.util.SecurityUtility;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.asynchttpclient.AsyncHttpClient;
 import org.asynchttpclient.BoundRequestBuilder;
 import org.asynchttpclient.DefaultAsyncHttpClient;
@@ -84,10 +85,17 @@ public class WebServiceTest {
     private PulsarService pulsar;
     private String BROKER_LOOKUP_URL;
     private String BROKER_LOOKUP_URL_TLS;
-    private static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
-    private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-    private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
-    private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
+
+    private final static String CA_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
+    private final static String BROKER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final static String BROKER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final static String CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final static String CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
 
 
     @Test
@@ -351,8 +359,8 @@ private String makeHttpRequest(boolean useTls, boolean useAuth) throws Exception
             if (useTls) {
                 KeyManager[] keyManagers = null;
                 if (useAuth) {
-                    Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
-                    PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
+                    Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(CLIENT_CERT_FILE_PATH);
+                    PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(CLIENT_KEY_FILE_PATH);
 
                     KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
                     ks.load(null, null);
@@ -403,10 +411,10 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut
         config.setAuthenticationProviders(providers);
         config.setAuthorizationEnabled(false);
         config.setSuperUserRoles(roles);
-        config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         config.setTlsAllowInsecureConnection(allowInsecure);
-        config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH);
+        config.setTlsTrustCertsFilePath(allowInsecure ? "" : CA_CERT_FILE_PATH);
         config.setClusterName("local");
         config.setAdvertisedAddress("localhost"); // TLS certificate expects localhost
         config.setMetadataStoreUrl("zk:localhost:2181");
@@ -433,8 +441,8 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut
             serviceUrl = BROKER_URL_BASE_TLS;
 
             Map authParams = new HashMap<>();
-            authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-            authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+            authParams.put("tlsCertFile", CLIENT_CERT_FILE_PATH);
+            authParams.put("tlsKeyFile", CLIENT_KEY_FILE_PATH);
 
             adminBuilder.authentication(AuthenticationTls.class.getName(), authParams).allowTlsInsecureConnection(true);
         }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
index 8597e0a87997a..792f419ee997e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
@@ -31,7 +31,6 @@
 import com.google.common.util.concurrent.MoreExecutors;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.Field;
@@ -41,10 +40,6 @@
 import java.net.URI;
 import java.net.URL;
 import java.net.URLConnection;
-import java.security.KeyStore;
-import java.security.PrivateKey;
-import java.security.SecureRandom;
-import java.security.cert.Certificate;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -62,10 +57,7 @@
 import java.util.stream.Collectors;
 import javax.naming.AuthenticationException;
 import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
 import lombok.Cleanup;
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.PulsarService;
@@ -427,10 +419,6 @@ public void testPartitionTopicLookup() throws Exception {
     @Test
     public void testWebserviceServiceTls() throws Exception {
         log.info("-- Starting {} test --", methodName);
-        final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
-        final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-        final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
-        final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
 
         /**** start broker-2 ****/
         ServiceConfiguration conf2 = new ServiceConfiguration();
@@ -443,12 +431,15 @@ public void testWebserviceServiceTls() throws Exception {
         conf2.setWebServicePort(Optional.of(0));
         conf2.setWebServicePortTls(Optional.of(0));
         conf2.setAdvertisedAddress("localhost");
-        conf2.setTlsAllowInsecureConnection(true);
-        conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf2.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf2.setTlsRequireTrustedClientCertOnConnect(true);
+        conf2.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf2.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf2.setClusterName(conf.getClusterName());
         conf2.setMetadataStoreUrl("zk:localhost:2181");
         conf2.setConfigurationMetadataStoreUrl("zk:localhost:3181");
+        // Not in use, and because TLS is not configured, it will fail to start
+        conf2.setSystemTopicEnabled(false);
 
         @Cleanup
         PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf2);
@@ -457,10 +448,13 @@ public void testWebserviceServiceTls() throws Exception {
         // restart broker1 with tls enabled
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsAllowInsecureConnection(true);
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf.setTlsRequireTrustedClientCertOnConnect(true);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setNumExecutorThreadPoolSize(5);
+        // Not in use, and because TLS is not configured, it will fail to start
+        conf.setSystemTopicEnabled(false);
         stopBroker();
         startBroker();
         pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
@@ -494,18 +488,8 @@ public void testWebserviceServiceTls() throws Exception {
         final String lookupResourceUrl = "/lookup/v2/topic/persistent/my-property/my-ns/my-topic1";
 
         // set client cert_key file
-        KeyManager[] keyManagers = null;
-        Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
-        PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
-        KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
-        ks.load(null, null);
-        ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
-        KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-        kmf.init(ks, "".toCharArray());
-        keyManagers = kmf.getKeyManagers();
-        TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
-        SSLContext sslCtx = SSLContext.getInstance("TLS");
-        sslCtx.init(keyManagers, trustManagers, new SecureRandom());
+        SSLContext sslCtx = SecurityUtility.createSslContext(false, CA_CERT_FILE_PATH,
+                getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"), "");
         HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());
 
         // hit broker2 url
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
index 246d980d6178c..1e8b26beee38a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
@@ -50,6 +50,7 @@
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig;
 import org.apache.pulsar.functions.sink.PulsarSink;
 import org.apache.pulsar.functions.worker.service.WorkerServiceLoader;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
@@ -62,11 +63,16 @@ public class PulsarFunctionTlsTest {
 
     protected static final int BROKER_COUNT = 2;
 
-    private static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    private static final String CA_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
+    private final String TLS_SERVER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final String TLS_SERVER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final String TLS_CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final String TLS_CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
+    private final String CA_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
 
     LocalBookkeeperEnsemble bkEnsemble;
     protected PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT];
diff --git a/pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem b/pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem
deleted file mode 100644
index 8d0a02f24214f..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem
+++ /dev/null
@@ -1,117 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 4098 (0x1002)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Validity
-            Not Before: Feb 17 17:00:44 2021 GMT
-            Not After : Feb 12 17:00:44 2041 GMT
-        Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=localhost/emailAddress=dev@pulsar.apache.org
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:9b:2a:6f:24:02:23:f7:ff:e6:75:61:ca:07:a8:
-                    c0:ab:e9:8d:eb:51:2e:64:f7:9e:9b:d4:b4:be:3a:
-                    fa:f4:6e:c6:92:8f:38:4d:08:cd:89:15:3e:2c:c4:
-                    99:6d:cb:58:80:fc:e0:4d:d6:7d:f6:82:ab:0d:94:
-                    f2:e2:45:c9:d3:15:95:57:0a:6c:86:dc:78:64:3b:
-                    34:4b:01:7c:5d:de:4f:d4:21:1a:5d:27:a0:a5:70:
-                    7a:2e:02:50:e1:19:b4:b9:05:df:99:0d:8b:cc:62:
-                    dc:10:73:fa:72:8b:38:7f:d3:56:54:61:50:bb:92:
-                    ff:09:71:09:c7:bd:04:43:3c:8c:9c:8b:32:d1:05:
-                    04:8a:c6:89:d8:78:56:4d:da:2f:f4:ec:34:37:26:
-                    b5:87:e4:3f:26:c9:41:60:ba:31:10:19:be:f8:0c:
-                    a4:0a:85:19:59:e2:00:5d:b7:c0:bd:d1:2e:fc:a6:
-                    34:8b:85:2a:cc:05:f6:fb:e4:00:e6:74:95:ff:02:
-                    6f:43:7f:39:a7:c2:83:8e:5b:38:40:c9:42:c8:bc:
-                    26:72:36:35:64:c2:54:22:11:87:e8:65:8f:3d:e9:
-                    41:a7:6d:19:88:9a:20:9b:9a:52:e7:d2:cb:b3:e0:
-                    2e:8f:c1:56:54:bc:6d:14:30:73:c5:d7:8e:d0:5a:
-                    5e:cd
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Basic Constraints:
-                CA:FALSE
-            Netscape Cert Type:
-                SSL Server
-            Netscape Comment:
-                OpenSSL Generated Server Certificate
-            X509v3 Subject Key Identifier:
-                49:3C:B2:98:30:CE:7F:79:7A:C6:8B:57:CA:24:9F:12:82:1E:5D:EF
-            X509v3 Authority Key Identifier:
-                keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71
-                DirName:/C=US/ST=California/L=Palo Alto/O=Apache Software Foundation/OU=Pulsar/CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-                serial:52:7B:B4:00:96:60:B4:26:85:BE:01:82:B8:B8:E2:8C:72:EF:5B:90
-
-            X509v3 Key Usage: critical
-                Digital Signature, Key Encipherment
-            X509v3 Extended Key Usage:
-                TLS Web Server Authentication
-    Signature Algorithm: sha256WithRSAEncryption
-         0f:bd:af:39:0c:2c:dc:8f:7e:06:0d:27:df:35:c7:8d:5a:03:
-         68:97:f6:dc:d6:d3:39:0e:b4:76:48:7d:e1:1c:a9:4b:83:fa:
-         52:00:ab:28:93:2d:06:76:0c:14:35:3c:f1:8e:3b:af:c8:d0:
-         27:1f:58:d4:71:22:5f:05:a6:9e:73:c6:a5:5e:2a:e6:fb:eb:
-         fc:73:52:87:ca:8a:2a:f9:1e:5f:e2:b9:bd:01:27:9f:7c:61:
-         a6:97:ad:a0:ab:4e:fb:cc:fa:c8:77:6a:65:1b:ae:60:5e:fb:
-         97:14:8c:40:d7:96:c6:2c:64:59:c0:52:52:7c:2d:98:4b:f4:
-         72:da:83:f7:c6:4f:32:42:ce:df:02:dd:5f:eb:58:42:f9:62:
-         a1:9a:05:ef:13:48:27:af:a3:7f:23:eb:e0:dc:1d:8f:96:2a:
-         88:47:f7:e4:75:6f:a9:15:f6:44:f1:6d:39:3a:2c:df:a7:82:
-         cc:7e:aa:9c:1c:c0:a7:7d:68:31:4a:4e:21:b8:9f:17:90:4b:
-         f1:68:23:ef:a7:53:fc:a9:a8:35:6b:8f:4c:5e:d4:ea:b0:8a:
-         27:9a:86:89:ce:f2:5d:03:35:80:fc:45:e8:87:66:0f:32:b5:
-         2a:f5:1b:79:0e:09:8b:90:40:20:fb:e3:27:8a:c9:92:c1:53:
-         97:10:5a:8c:50:ef:02:46:7e:ec:68:c8:1e:26:66:0e:1d:d6:
-         6c:82:e7:38:14:e8:cb:45:77:29:5f:2c:1a:9d:d7:54:21:8a:
-         cf:0f:b7:0c:ae:fe:d6:fb:fb:c3:07:3e:33:df:59:25:1c:73:
-         d4:87:73:14:b4:76:16:8a:3f:82:05:7b:42:0a:55:0c:79:24:
-         3c:58:31:3f:e0:3e:9f:4e:d0:0e:fd:77:b7:13:2c:d3:d0:46:
-         cc:80:09:0f:50:56:8b:6e:6e:91:b2:5b:c8:2f:4d:86:dc:72:
-         00:de:08:0d:5e:3e:96:1f:12:7d:3b:0d:4d:71:d5:c8:a8:06:
-         ba:00:23:ec:10:4c:a4:c3:6f:bc:f0:d7:b1:cf:57:3f:3b:79:
-         db:80:87:35:c7:4e:7f:bb:38:30:0a:9f:fe:5a:86:f5:97:ce:
-         24:38:79:fd:a0:dc:0b:82:11:a1:ea:0c:e9:16:65:e0:c0:54:
-         80:ad:6e:55:18:ac:27:35:3a:b0:20:70:62:8e:5d:a2:33:53:
-         8c:ce:f9:ee:a1:27:cb:db:e5:9a:5e:e6:f7:80:93:84:63:04:
-         26:58:ab:23:bb:94:80:d0:a0:55:a2:8a:ed:bc:0f:c3:41:d2:
-         26:a5:b9:8d:8a:45:e8:a1:fc:e8:ee:7a:64:93:ed:d6:ef:a2:
-         51:d7:c9:0a:31:39:35:4a
------BEGIN CERTIFICATE-----
-MIIGPDCCBCSgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwgaYxCzAJBgNVBAYTAlVT
-MRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNV
-BAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIx
-EjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5h
-cGFjaGUub3JnMB4XDTIxMDIxNzE3MDA0NFoXDTQxMDIxMjE3MDA0NFowgZIxCzAJ
-BgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMSMwIQYDVQQKDBpBcGFjaGUg
-U29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQDDAls
-b2NhbGhvc3QxJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJsqbyQCI/f/5nVhygeowKvp
-jetRLmT3npvUtL46+vRuxpKPOE0IzYkVPizEmW3LWID84E3WffaCqw2U8uJFydMV
-lVcKbIbceGQ7NEsBfF3eT9QhGl0noKVwei4CUOEZtLkF35kNi8xi3BBz+nKLOH/T
-VlRhULuS/wlxCce9BEM8jJyLMtEFBIrGidh4Vk3aL/TsNDcmtYfkPybJQWC6MRAZ
-vvgMpAqFGVniAF23wL3RLvymNIuFKswF9vvkAOZ0lf8Cb0N/OafCg45bOEDJQsi8
-JnI2NWTCVCIRh+hljz3pQadtGYiaIJuaUufSy7PgLo/BVlS8bRQwc8XXjtBaXs0C
-AwEAAaOCAYQwggGAMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCG
-SAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUw
-HQYDVR0OBBYEFEk8spgwzn95esaLV8oknxKCHl3vMIHmBgNVHSMEgd4wgduAFNKy
-PbGkfEhLNuGn3tj8upK6p8RxoYGspIGpMIGmMQswCQYDVQQGEwJVUzETMBEGA1UE
-CAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpBcGFj
-aGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQD
-DAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9y
-Z4IUUnu0AJZgtCaFvgGCuLjijHLvW5AwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQAPva85DCzcj34GDSffNceN
-WgNol/bc1tM5DrR2SH3hHKlLg/pSAKsoky0GdgwUNTzxjjuvyNAnH1jUcSJfBaae
-c8alXirm++v8c1KHyooq+R5f4rm9ASeffGGml62gq077zPrId2plG65gXvuXFIxA
-15bGLGRZwFJSfC2YS/Ry2oP3xk8yQs7fAt1f61hC+WKhmgXvE0gnr6N/I+vg3B2P
-liqIR/fkdW+pFfZE8W05Oizfp4LMfqqcHMCnfWgxSk4huJ8XkEvxaCPvp1P8qag1
-a49MXtTqsIonmoaJzvJdAzWA/EXoh2YPMrUq9Rt5DgmLkEAg++MnismSwVOXEFqM
-UO8CRn7saMgeJmYOHdZsguc4FOjLRXcpXywanddUIYrPD7cMrv7W+/vDBz4z31kl
-HHPUh3MUtHYWij+CBXtCClUMeSQ8WDE/4D6fTtAO/Xe3EyzT0EbMgAkPUFaLbm6R
-slvIL02G3HIA3ggNXj6WHxJ9Ow1NcdXIqAa6ACPsEEykw2+88Nexz1c/O3nbgIc1
-x05/uzgwCp/+Wob1l84kOHn9oNwLghGh6gzpFmXgwFSArW5VGKwnNTqwIHBijl2i
-M1OMzvnuoSfL2+WaXub3gJOEYwQmWKsju5SA0KBVoortvA/DQdImpbmNikXoofzo
-7npkk+3W76JR18kKMTk1Sg==
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/broker-key.pem b/pulsar-broker/src/test/resources/authentication/tls/broker-key.pem
deleted file mode 100644
index ee03e754beee1..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/broker-key.pem
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEAmypvJAIj9//mdWHKB6jAq+mN61EuZPeem9S0vjr69G7Gko84
-TQjNiRU+LMSZbctYgPzgTdZ99oKrDZTy4kXJ0xWVVwpshtx4ZDs0SwF8Xd5P1CEa
-XSegpXB6LgJQ4Rm0uQXfmQ2LzGLcEHP6cos4f9NWVGFQu5L/CXEJx70EQzyMnIsy
-0QUEisaJ2HhWTdov9Ow0Nya1h+Q/JslBYLoxEBm++AykCoUZWeIAXbfAvdEu/KY0
-i4UqzAX2++QA5nSV/wJvQ385p8KDjls4QMlCyLwmcjY1ZMJUIhGH6GWPPelBp20Z
-iJogm5pS59LLs+Auj8FWVLxtFDBzxdeO0FpezQIDAQABAoIBAG9pk63mP49l1kM4
-eQjw2Y9WvslVXBuxVNiNbU4eKW1zUO+RGJrvlC027JLWg1g7pwvPBvu85GspPcsd
-xRxFgfonyDhcSrq2+Vb2z8B/i54W73jgX/69YnMIBSKeFRbcD1C+7+MEv/l8jojd
-zdmLL4FQ7O7fhUl57dgIqz4Y8UOYyyBsPpz3pzJLFEb5rE/ajqmFzyl+dO+8140B
-niQ0+7+tAK0njX8OC0WN844GkO24WPCfWhUFrYGkfLq498eRUCWM2YP2tAJ+Uxnh
-v3K9icDwOX6PJXYlbvNEUCE+t60NoDYHcMpfzUdFEhBYpKadfKE/RFFcu0vAZ+aR
-y24oAuECgYEAyPLYXWIs88pPHQhSf2DAMRref5eeV+XA6Dy/P+z8z0bA7I6X9dl6
-AK6rRKGJl9HI7c/Gky6P10fymopYopNkClXm7SBTLKx0vfjil0U6Mx5ZsfDspE3q
-0o9MJKVgobCxVZlLErU55XzktKwjlv2UvDX7VuxRndqN9qdf+YSMb9kCgYEAxayx
-sOrJcPZVfy3Ohy5CeStF+E2dtfcKB7M7xZxZqykVy+6J1XjXHmp1L7Wpi0ju57Hi
-l2ZqKasHDwtlLOnfSTbvC47hsa1ydnoFTjJBObR1wS43oVkyV0AHid4w81ddOWPC
-H0ZmhvNe7pUxm5crpxsY6hAAraJ4Hej23MOxghUCgYEAip26UvCeQa2U1VogTm3X
-Jgh641kbiVabs5fz9Yzs966+9m+Gs7jJSB81Vap415mHGUTyniTIZKDk4WX9rmgt
-4lNPcNOTjIWKImHFLMQ8WXbeOLkRBGYbThQ7WiwadG8GZR3Rg54vyfZVbawxAL78
-ErjKIDP0OQfCVhsvQVgF6EECgYEAlQ2P+xA/Dv+gHkLjDUmTdBxuKToVZqU9merL
-cklfz9EuD1Tx99ajltq9PFll25IGGw0mB/WAraS5sN1tz/0VkfZrL7LwefKIcc+2
-em0og6OQezcnWXGRpPqx9IJnNMY2lFSlhsGmA7I1bf9vpZvKnbmwAqZIbKUqn5sP
-sg2ZprUCgYEApAVD+9wXfZE/YDHVZX1k6p38ORqjq/04AJkL/LmUW5DL5to1+1KQ
-Q438HzMtYIq7aZyzWmlF6DmyN5mxKKK3yY79p0rvdV74AoT+ucDzM3ge0Md7liCs
-0GwNnDSiPzdau738UoIKc1VbF7dMDL3LzqnfrBUCr7nXRbR3BHHuqws=
------END RSA PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/cacert.pem b/pulsar-broker/src/test/resources/authentication/tls/cacert.pem
deleted file mode 100644
index 6abfc2d80c123..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/cacert.pem
+++ /dev/null
@@ -1,127 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number:
-            52:7b:b4:00:96:60:b4:26:85:be:01:82:b8:b8:e2:8c:72:ef:5b:90
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Validity
-            Not Before: Feb 17 16:43:44 2021 GMT
-            Not After : Feb 12 16:43:44 2041 GMT
-        Subject: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (4096 bit)
-                Modulus:
-                    00:b1:3c:7d:ab:4a:54:72:37:2a:92:94:0a:66:46:
-                    af:8c:ed:f4:2e:f3:87:1a:d0:c7:9d:23:35:1b:61:
-                    74:69:ca:f7:f5:3e:95:9c:86:f2:21:34:f8:0b:ed:
-                    45:76:22:ec:75:52:c0:67:db:2f:ba:da:25:3f:e1:
-                    5b:ac:da:15:dd:a5:75:24:b2:12:f0:b0:ce:fd:ab:
-                    44:06:a9:09:f6:b0:8e:8f:83:53:16:69:fa:9c:cc:
-                    00:fa:dd:13:f3:da:fd:f2:bf:88:8e:c4:f8:1a:6f:
-                    ab:4d:f8:32:81:80:7e:51:7a:99:2d:94:cd:f3:5d:
-                    1c:58:b2:44:f1:96:12:46:56:bd:60:8f:65:32:b7:
-                    d4:4b:7b:f3:23:88:2d:9b:a4:c4:c9:52:ea:9f:66:
-                    c1:74:be:4b:91:c6:b9:57:ec:c1:cc:81:bb:03:d5:
-                    fa:a0:46:4f:9a:a7:3e:3c:27:26:2b:97:eb:69:53:
-                    04:75:50:97:d6:0d:90:b1:37:9f:64:df:70:4d:d9:
-                    b3:e3:b7:cc:76:50:d9:3c:9b:4c:ac:e9:26:2e:cf:
-                    ac:47:42:14:b7:60:00:0a:de:42:47:66:0c:c7:7a:
-                    b9:4d:f4:fb:c2:6a:45:78:ec:b0:b4:ce:b3:1f:50:
-                    25:96:13:0c:55:0a:e0:d6:76:f7:1f:e1:16:e6:41:
-                    d6:72:6a:49:17:12:d9:05:8f:dc:56:b6:31:b3:b7:
-                    9c:e3:d8:a9:99:8a:1d:3b:9d:d9:59:44:ee:46:88:
-                    11:5f:ab:fa:38:a9:8b:d2:23:15:8b:af:1a:de:66:
-                    ba:7d:51:95:37:94:91:aa:01:01:d7:83:19:4b:5d:
-                    8d:f4:18:39:ef:e3:32:d0:62:c8:12:50:4e:91:c2:
-                    ac:58:73:68:bb:92:20:fc:14:e5:1a:86:bd:40:4c:
-                    94:e0:7d:0d:9c:08:57:ae:00:44:38:94:a3:3d:64:
-                    99:43:f8:e3:12:90:14:0f:5d:63:e2:c6:07:ea:d0:
-                    4c:8e:cf:e0:ae:34:be:86:4f:fc:58:e2:ea:f5:23:
-                    82:37:96:02:57:1b:b4:29:ca:fd:68:a0:48:79:e8:
-                    31:97:9a:5a:0e:2b:b4:b0:84:bb:57:4e:5f:4f:a7:
-                    43:45:97:d7:de:05:fc:2f:6c:3e:f5:53:26:56:a3:
-                    a5:da:52:69:57:8e:a0:4b:27:50:f9:ad:6e:76:a6:
-                    29:cc:06:94:dd:d0:ac:c6:18:22:a0:e2:bb:ed:d5:
-                    e4:97:f7:ac:23:df:75:30:41:97:07:3f:d3:12:8e:
-                    c5:a4:ef:ce:40:e8:3b:57:24:19:33:1b:ee:8a:0e:
-                    dd:0c:70:f2:1a:87:35:d9:71:d8:18:a7:9c:47:db:
-                    93:51:c3
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Subject Key Identifier:
-                D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71
-            X509v3 Authority Key Identifier:
-                keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71
-
-            X509v3 Basic Constraints: critical
-                CA:TRUE
-            X509v3 Key Usage: critical
-                Digital Signature, Certificate Sign, CRL Sign
-    Signature Algorithm: sha256WithRSAEncryption
-         14:3d:7c:15:86:de:aa:5a:30:5d:d4:f2:bc:5f:10:d2:af:fe:
-         91:d7:ee:f3:b8:5f:ce:e4:c9:b2:01:c3:16:da:66:8e:7e:b1:
-         c1:e3:30:ff:1d:73:d0:9c:20:3d:54:32:57:ae:07:80:4a:24:
-         6e:7e:32:a3:e7:23:4d:5c:31:54:8b:c1:1b:c5:bc:20:5d:43:
-         62:93:e0:2e:a7:01:77:39:cf:fd:ec:4c:57:09:4f:2b:ad:ac:
-         b6:c0:be:5a:a3:ea:12:ac:5a:7f:60:23:81:bb:9a:fa:5f:7a:
-         67:a9:31:c3:34:af:db:ff:32:22:83:40:c2:7d:2f:39:5e:8a:
-         29:44:73:5f:6e:b4:f4:a2:ae:60:1f:8e:ef:91:9a:49:bb:a6:
-         90:2b:e0:44:95:24:8b:37:90:18:2d:41:32:8a:8e:07:8d:ea:
-         75:62:b8:9c:ec:73:6f:12:54:23:6d:40:00:74:c7:d3:fb:b7:
-         95:06:7d:cc:6d:8e:2c:d0:8b:11:06:8a:b7:43:1a:d7:e9:98:
-         f4:c6:ef:ad:2a:75:08:fb:07:8f:20:36:7a:86:1a:cf:f7:d6:
-         96:ad:ed:71:59:d1:81:56:18:8d:98:c2:c0:44:e5:29:7a:7c:
-         c0:e3:d7:fb:b8:f5:b2:50:53:8a:cf:38:ff:99:aa:bb:28:51:
-         60:e8:05:91:e1:ee:86:90:90:9b:87:60:63:38:cf:54:a5:82:
-         74:0f:40:b5:d2:6a:c5:a9:98:22:59:4e:fb:a5:81:e2:7b:0e:
-         3f:71:f3:24:17:1e:c5:89:fc:ae:ed:f3:69:65:02:b8:1e:98:
-         bc:37:c6:25:36:f8:ca:99:60:8e:13:3b:33:ec:91:b3:eb:04:
-         6d:41:97:3e:35:c0:97:ed:66:12:25:44:23:f3:2e:fa:9c:2e:
-         c2:ba:dd:f3:63:d7:5b:b2:72:03:4d:3b:fb:5e:29:d6:5c:02:
-         32:93:47:d1:4c:77:4a:58:c5:aa:81:ab:67:84:80:81:14:28:
-         e1:db:11:16:6d:31:50:7a:47:b2:a8:2d:15:a1:c4:63:1b:ce:
-         d5:e1:d7:57:dc:1a:71:e0:55:9f:6d:fb:be:e6:99:e8:89:be:
-         2c:e0:19:5e:cd:02:79:52:ee:93:56:9f:dc:d7:de:31:9b:2a:
-         c8:91:48:a0:c7:44:7d:72:32:27:c3:2b:d8:e8:6b:94:67:b5:
-         1d:9d:99:25:23:d9:24:b5:ed:4b:f2:18:2d:88:f5:d4:36:bb:
-         53:8c:a8:b1:7f:05:13:d7:8d:89:9d:55:33:90:bc:60:99:cf:
-         05:ba:bd:cb:c5:61:f9:c5:1a:f7:46:9c:40:90:dd:83:aa:7a:
-         1f:ab:5c:10:8d:26:27:1e
------BEGIN CERTIFICATE-----
-MIIGPzCCBCegAwIBAgIUUnu0AJZgtCaFvgGCuLjijHLvW5AwDQYJKoZIhvcNAQEL
-BQAwgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQH
-DAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9u
-MQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3
-DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMB4XDTIxMDIxNzE2NDM0NFoXDTQx
-MDIxMjE2NDM0NFowgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh
-MRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBG
-b3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEk
-MCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMIICIjANBgkqhkiG
-9w0BAQEFAAOCAg8AMIICCgKCAgEAsTx9q0pUcjcqkpQKZkavjO30LvOHGtDHnSM1
-G2F0acr39T6VnIbyITT4C+1FdiLsdVLAZ9svutolP+FbrNoV3aV1JLIS8LDO/atE
-BqkJ9rCOj4NTFmn6nMwA+t0T89r98r+IjsT4Gm+rTfgygYB+UXqZLZTN810cWLJE
-8ZYSRla9YI9lMrfUS3vzI4gtm6TEyVLqn2bBdL5Lkca5V+zBzIG7A9X6oEZPmqc+
-PCcmK5fraVMEdVCX1g2QsTefZN9wTdmz47fMdlDZPJtMrOkmLs+sR0IUt2AACt5C
-R2YMx3q5TfT7wmpFeOywtM6zH1AllhMMVQrg1nb3H+EW5kHWcmpJFxLZBY/cVrYx
-s7ec49ipmYodO53ZWUTuRogRX6v6OKmL0iMVi68a3ma6fVGVN5SRqgEB14MZS12N
-9Bg57+My0GLIElBOkcKsWHNou5Ig/BTlGoa9QEyU4H0NnAhXrgBEOJSjPWSZQ/jj
-EpAUD11j4sYH6tBMjs/grjS+hk/8WOLq9SOCN5YCVxu0Kcr9aKBIeegxl5paDiu0
-sIS7V05fT6dDRZfX3gX8L2w+9VMmVqOl2lJpV46gSydQ+a1udqYpzAaU3dCsxhgi
-oOK77dXkl/esI991MEGXBz/TEo7FpO/OQOg7VyQZMxvuig7dDHDyGoc12XHYGKec
-R9uTUcMCAwEAAaNjMGEwHQYDVR0OBBYEFNKyPbGkfEhLNuGn3tj8upK6p8RxMB8G
-A1UdIwQYMBaAFNKyPbGkfEhLNuGn3tj8upK6p8RxMA8GA1UdEwEB/wQFMAMBAf8w
-DgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQAUPXwVht6qWjBd1PK8
-XxDSr/6R1+7zuF/O5MmyAcMW2maOfrHB4zD/HXPQnCA9VDJXrgeASiRufjKj5yNN
-XDFUi8EbxbwgXUNik+AupwF3Oc/97ExXCU8rray2wL5ao+oSrFp/YCOBu5r6X3pn
-qTHDNK/b/zIig0DCfS85XoopRHNfbrT0oq5gH47vkZpJu6aQK+BElSSLN5AYLUEy
-io4Hjep1Yric7HNvElQjbUAAdMfT+7eVBn3MbY4s0IsRBoq3QxrX6Zj0xu+tKnUI
-+wePIDZ6hhrP99aWre1xWdGBVhiNmMLAROUpenzA49f7uPWyUFOKzzj/maq7KFFg
-6AWR4e6GkJCbh2BjOM9UpYJ0D0C10mrFqZgiWU77pYHiew4/cfMkFx7Fifyu7fNp
-ZQK4Hpi8N8YlNvjKmWCOEzsz7JGz6wRtQZc+NcCX7WYSJUQj8y76nC7Cut3zY9db
-snIDTTv7XinWXAIyk0fRTHdKWMWqgatnhICBFCjh2xEWbTFQekeyqC0VocRjG87V
-4ddX3Bpx4FWfbfu+5pnoib4s4BlezQJ5Uu6TVp/c194xmyrIkUigx0R9cjInwyvY
-6GuUZ7UdnZklI9kkte1L8hgtiPXUNrtTjKixfwUT142JnVUzkLxgmc8Fur3LxWH5
-xRr3RpxAkN2Dqnofq1wQjSYnHg==
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/client-cert.pem b/pulsar-broker/src/test/resources/authentication/tls/client-cert.pem
deleted file mode 100644
index 45f3cde215fe5..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/client-cert.pem
+++ /dev/null
@@ -1,90 +0,0 @@
-Certificate:
-    Data:
-        Version: 1 (0x0)
-        Serial Number: 4097 (0x1001)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Validity
-            Not Before: Feb 17 16:56:55 2021 GMT
-            Not After : Feb 12 16:56:55 2041 GMT
-        Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=admin/emailAddress=dev@pulsar.apache.org
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:ab:61:f5:12:b1:e1:ae:19:01:3e:59:4a:c6:ca:
-                    00:0c:96:e8:76:3a:83:20:d9:af:3a:e1:11:20:12:
-                    e0:e4:d0:70:8f:4b:7b:af:e1:89:ef:9b:c5:a9:c2:
-                    ed:ae:24:8d:bb:42:6e:ec:59:11:3f:f5:63:59:61:
-                    18:9f:70:b6:76:88:e2:ca:79:15:cc:fb:9c:5e:5c:
-                    bb:a1:d7:f0:d8:11:d4:17:34:1e:81:7e:0b:0e:05:
-                    be:5d:fa:d6:46:af:e1:95:d8:a0:5d:c5:2f:d9:a9:
-                    8f:69:64:49:95:f7:42:16:6a:84:2b:2e:af:91:73:
-                    3d:b6:d4:44:56:9a:61:43:49:15:22:ae:90:5d:04:
-                    29:90:4e:b2:41:34:73:3e:a2:48:05:1c:bc:8e:1b:
-                    0b:c1:d5:df:56:32:40:e9:91:a2:7b:de:31:2b:67:
-                    f1:8e:d6:c5:c0:87:57:70:29:f9:af:db:57:a0:2e:
-                    8c:30:0a:a7:47:39:33:4c:d7:2d:32:aa:48:29:bd:
-                    c4:48:c5:58:52:07:c4:99:b1:cc:66:da:ac:28:4d:
-                    c1:bc:1f:44:3f:a3:63:61:bd:ff:48:61:76:04:b2:
-                    7d:1c:6e:9c:ee:82:bb:f7:60:1c:7a:a0:98:be:2d:
-                    70:43:2f:64:bf:d2:0f:20:25:f7:c7:7d:70:05:b8:
-                    2e:bf
-                Exponent: 65537 (0x10001)
-    Signature Algorithm: sha256WithRSAEncryption
-         1c:31:b8:0f:a1:03:28:a0:da:31:ec:34:ce:e0:fd:01:99:9d:
-         9b:ad:f8:03:5d:20:85:18:de:ca:b5:ea:61:c9:3b:65:42:9c:
-         e5:21:73:d2:06:41:4b:a9:3a:fb:7f:ff:45:f3:5a:4a:ab:5a:
-         86:cd:57:6a:5f:13:c0:ae:7e:ad:5c:6e:c3:c4:e7:b7:d3:14:
-         bf:86:fe:f2:d1:70:0e:fc:98:50:a7:fe:53:62:5a:2d:f5:63:
-         2c:ee:4a:7c:dd:32:3e:d1:52:3a:1f:15:38:4b:2a:4a:ee:27:
-         a9:d8:92:a8:33:92:83:c9:3a:09:5a:01:66:0e:68:da:8f:82:
-         c0:18:cc:78:ea:c5:db:09:7c:2f:61:c3:51:f8:58:7a:27:d7:
-         92:c0:ff:f8:29:d7:a0:e9:54:17:8d:48:a8:ff:5e:92:ee:81:
-         6c:37:90:1c:93:28:8c:d2:f5:b1:20:96:d3:1d:0f:c0:7f:db:
-         0c:6d:65:7f:3a:55:e5:c9:9a:ad:09:91:a5:57:cb:fc:bf:df:
-         69:bd:6b:87:94:5b:d0:cf:3b:8b:48:41:3d:56:b6:1d:3f:e7:
-         f6:b6:58:f7:54:2a:dd:da:60:68:db:9b:70:04:8b:19:c3:44:
-         bf:1d:b4:28:b9:f8:ea:ad:d3:1a:6e:64:72:b1:61:6a:f3:e1:
-         d4:68:56:7b:0e:ad:4c:53:1e:d2:2e:1c:bc:b7:82:59:af:65:
-         d2:fd:ef:89:7c:34:8f:51:a1:4e:9d:7e:dc:c7:97:68:ea:aa:
-         e5:67:ed:be:dc:38:74:0e:c3:6f:fd:08:62:54:d8:1f:15:d1:
-         25:fc:21:f6:8c:f9:2f:65:5e:07:b9:e9:56:ba:48:14:5c:0d:
-         18:ba:f8:83:54:5b:b6:27:0c:36:2c:20:29:9c:c2:68:c5:3a:
-         0f:a5:d6:5f:7c:aa:f9:a6:2a:2b:69:c5:b1:39:e7:1c:02:31:
-         5b:f5:82:de:c9:4e:8d:33:dc:94:02:44:0a:44:95:75:7b:a1:
-         e7:ee:92:fc:35:93:73:8c:22:c1:32:ea:39:17:ca:d0:87:fc:
-         4d:8e:04:f8:59:66:d3:14:3f:59:ad:76:14:20:16:7b:77:4f:
-         94:58:f8:85:5c:ba:b3:69:ed:7f:75:54:9a:1a:88:21:5d:04:
-         57:87:85:e2:d4:0e:1b:61:7f:5d:36:dc:72:a1:9d:0b:c8:ce:
-         19:69:49:fa:1b:bb:3f:3d:1b:4d:81:42:95:4e:d8:0b:04:d1:
-         08:6d:15:b3:ae:52:41:12:ff:e1:90:c4:7d:52:88:55:8b:87:
-         83:06:48:8b:fc:3a:a7:47:0e:6c:a8:4c:9e:b0:aa:da:50:f5:
-         97:97:98:3e:9d:18:ef:43
------BEGIN CERTIFICATE-----
-MIIEqzCCApMCAhABMA0GCSqGSIb3DQEBCwUAMIGmMQswCQYDVQQGEwJVUzETMBEG
-A1UECAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpB
-cGFjaGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYD
-VQQDDAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hl
-Lm9yZzAeFw0yMTAyMTcxNjU2NTVaFw00MTAyMTIxNjU2NTVaMIGOMQswCQYDVQQG
-EwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEjMCEGA1UECgwaQXBhY2hlIFNvZnR3
-YXJlIEZvdW5kYXRpb24xDzANBgNVBAsMBlB1bHNhcjEOMAwGA1UEAwwFYWRtaW4x
-JDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCCASIwDQYJKoZI
-hvcNAQEBBQADggEPADCCAQoCggEBAKth9RKx4a4ZAT5ZSsbKAAyW6HY6gyDZrzrh
-ESAS4OTQcI9Le6/hie+bxanC7a4kjbtCbuxZET/1Y1lhGJ9wtnaI4sp5Fcz7nF5c
-u6HX8NgR1Bc0HoF+Cw4Fvl361kav4ZXYoF3FL9mpj2lkSZX3QhZqhCsur5FzPbbU
-RFaaYUNJFSKukF0EKZBOskE0cz6iSAUcvI4bC8HV31YyQOmRonveMStn8Y7WxcCH
-V3Ap+a/bV6AujDAKp0c5M0zXLTKqSCm9xEjFWFIHxJmxzGbarChNwbwfRD+jY2G9
-/0hhdgSyfRxunO6Cu/dgHHqgmL4tcEMvZL/SDyAl98d9cAW4Lr8CAwEAATANBgkq
-hkiG9w0BAQsFAAOCAgEAHDG4D6EDKKDaMew0zuD9AZmdm634A10ghRjeyrXqYck7
-ZUKc5SFz0gZBS6k6+3//RfNaSqtahs1Xal8TwK5+rVxuw8Tnt9MUv4b+8tFwDvyY
-UKf+U2JaLfVjLO5KfN0yPtFSOh8VOEsqSu4nqdiSqDOSg8k6CVoBZg5o2o+CwBjM
-eOrF2wl8L2HDUfhYeifXksD/+CnXoOlUF41IqP9eku6BbDeQHJMojNL1sSCW0x0P
-wH/bDG1lfzpV5cmarQmRpVfL/L/fab1rh5Rb0M87i0hBPVa2HT/n9rZY91Qq3dpg
-aNubcASLGcNEvx20KLn46q3TGm5kcrFhavPh1GhWew6tTFMe0i4cvLeCWa9l0v3v
-iXw0j1GhTp1+3MeXaOqq5Wftvtw4dA7Db/0IYlTYHxXRJfwh9oz5L2VeB7npVrpI
-FFwNGLr4g1RbticMNiwgKZzCaMU6D6XWX3yq+aYqK2nFsTnnHAIxW/WC3slOjTPc
-lAJECkSVdXuh5+6S/DWTc4wiwTLqORfK0If8TY4E+Flm0xQ/Wa12FCAWe3dPlFj4
-hVy6s2ntf3VUmhqIIV0EV4eF4tQOG2F/XTbccqGdC8jOGWlJ+hu7Pz0bTYFClU7Y
-CwTRCG0Vs65SQRL/4ZDEfVKIVYuHgwZIi/w6p0cObKhMnrCq2lD1l5eYPp0Y70M=
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/client-key.pem b/pulsar-broker/src/test/resources/authentication/tls/client-key.pem
deleted file mode 100644
index e12697c966a9c..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/client-key.pem
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEoQIBAAKCAQEAq2H1ErHhrhkBPllKxsoADJbodjqDINmvOuERIBLg5NBwj0t7
-r+GJ75vFqcLtriSNu0Ju7FkRP/VjWWEYn3C2dojiynkVzPucXly7odfw2BHUFzQe
-gX4LDgW+XfrWRq/hldigXcUv2amPaWRJlfdCFmqEKy6vkXM9ttREVpphQ0kVIq6Q
-XQQpkE6yQTRzPqJIBRy8jhsLwdXfVjJA6ZGie94xK2fxjtbFwIdXcCn5r9tXoC6M
-MAqnRzkzTNctMqpIKb3ESMVYUgfEmbHMZtqsKE3BvB9EP6NjYb3/SGF2BLJ9HG6c
-7oK792AceqCYvi1wQy9kv9IPICX3x31wBbguvwIDAQABAoIBAQCcwbSPrPRncaeZ
-h8LFoO36le16dnqKCZIloMcxNxNNNvo9lyVC8mBgMXLSm+Eab4TTyyf6Nl14ytJc
-ZltHOqkqMnp+B9LQ8zNLfDaDCijY+TWtI5bjio5B/S7qdwyXCzii/slv+3SQ+m6a
-T4ifCtH//t11QfaEa4v/NphrPjnIeAgB681bk8nKdRop84ar+51lgbHoAza+wv+8
-e+aK3Od8r4yD19ZoPiMg0o4t2cEi8kupVgjsuZVtcvF9Q6QLYV17BFYEHqYjcr18
-N1EJ96f2FLO6cwEM+cG4n8gHjfDGRcDlhT9Cum1kDpg4J88auVUXnrDyi5Dcv1Pz
-6EC+ZmXBAoGBAOHUSUDMkbEePKDaM3Z+4jLqZWc3UZhxQLnqg5l7phdQ6iSogQX9
-1LpZCJ+lOMTHBCnaTCoQpuSHgYgraVkD4KG6nzC423oDesd/xNvlfW3TRsmwZWbL
-khdcdBSoVy3Kbv1v8kxw0NlcR68qo1XYfmFCAITcFHdxDz/jGStydlR9AoGBAMJH
-gyPenL595X8t47R93rkGOIx5cVf5YrDIZCByp4K44Tf9OqZHbky7jSPSSbur10mI
-pypRq5EcZ/cudU4w4gGaMauczt5Dgvlqd3T+GTZY3jO8bxi66gvzYTbigAxaJWcY
-Uafiv5W9ldRKsY3pyCL8ubg38Ed2cSaS2wGd/SDrAn8NO2MPaO0gc6UZx688QjL+
-yL0oTxV42Snxusv7MkOJGjSd8UGeGEFeqdjXgdbRsNeNnDzaOh+NRGNSlziU/qUq
-1MR/FlXF0G5hQhtGxyuSQ87iAnPukf79X21tyG9TP4lBUE3iLLoQAlgw606muQiu
-qi9dmYeZeAZst+HBqfNFAoGADg6qmH/VC5uEbY1eeoLZCL5AfTmUT+9FitEVHZvu
-LvE9qpVyFvH4Mykm7z6aAzBN5Y4zukYqiddqVmJQLpYu5DrJ+UbhWQe9hFqFxjtU
-i7Amc8vgpgNwR+kWUahV547mQe1qiyFHB4iuPKwi6MfPqWhr775sbl9NlKLvodBS
-rn0CgYBDrLH6ehNV/RnJIVZYQD6YcocYdYFy4u76mCYKmEP57XmstZHZXQgiRwbK
-Oy2Yg/qieKtSMjstgHFK6ZNYIR37l9J9Lh9aeal61+wW2dsGEy29Rhg01FpvKReq
-wCHz3tneUyaOhq9m0gKMOpWYcO+FBX1/2K5Gwj8FgEpu9r2b3w==
------END RSA PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/certificate/client.crt b/pulsar-broker/src/test/resources/certificate/client.crt
deleted file mode 100644
index 2d7d156866a86..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/client.crt
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDVjCCAj4CCQCtw/UnTFDT7DANBgkqhkiG9w0BAQUFADBtMQswCQYDVQQGEwJB
-VTETMBEGA1UECAwKU29tZS1TdGF0ZTEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MSEw
-HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDzANBgNVBAMMBmNsaWVu
-dDAeFw0xNjA2MjAwMTQ1NDZaFw0yNjA2MTgwMTQ1NDZaMG0xCzAJBgNVBAYTAkFV
-MRMwEQYDVQQIDApTb21lLVN0YXRlMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxITAf
-BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQV5F3Au9FWXIYPdWqiX
-Rk5gdVmVkDuuFK4ZoOd8inoJpB3PPkpmpgoVkKQHDFhgx3ODGWIUgo+n6QDsJxY4
-ygHfVeggQgek8iUfteYVsIcHS0bjkhIij/3ihC301FkiqbrV069oLvUXLKcv3zxG
-mdBAiz0k4xGZhFieVRvQCLY9syUUxmQ/3Cv42lDY8a1gTw4CRRx/hCfDvXCKhOT4
-bMwUIDZfHB3JoDh3Thp8FLz0nTrRF75mSQJ/OdcafIm0Xoz2Otp/CSxLS+U1lLvG
-05crWTDe0om7NW4mK4CqGCFq5gUw7eIzaeO7Q5Qez9XGTMzkgIDTMvNYGGEeJhhm
-NQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQAKXy4g6hljY5MpO8mbZh+uJHq6NEUs
-4dr7OKDDWc39AROZsGf2eFUmHOjmRSw7VHpguGKI+rFRELVffpg/VvMh5apu+DBf
-jhxtDNceAyh5uugPNUJHXyeikBDYW8bAzUU3DmMldPkTZWcGjurmyhDQ1TtK2YJe
-RMFBXw5aAzdJMNi6OfXDH/ZX32hrb482yghDZj+ndnm0FefmLbFTQRMF8/fIHb1W
-kqNHwIaapZwH6j/MJy/TRFYcJunrBUYT9zVjY46k3GU0ex/Bn7T4pg9gzgFGZJhn
-jQQFKliIC84thCzdlPkrLduLY8tmlDKpLXatbEQ+s1MmNOURm6irPp6g
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/certificate/client.csr b/pulsar-broker/src/test/resources/certificate/client.csr
deleted file mode 100644
index e01f33ef073f6..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/client.csr
+++ /dev/null
@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICsjCCAZoCAQAwbTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
-FTATBgNVBAcMDERlZmF1bHQgQ2l0eTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0
-cyBQdHkgTHRkMQ8wDQYDVQQDDAZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IB
-DwAwggEKAoIBAQCpBXkXcC70VZchg91aqJdGTmB1WZWQO64Urhmg53yKegmkHc8+
-SmamChWQpAcMWGDHc4MZYhSCj6fpAOwnFjjKAd9V6CBCB6TyJR+15hWwhwdLRuOS
-EiKP/eKELfTUWSKputXTr2gu9Rcspy/fPEaZ0ECLPSTjEZmEWJ5VG9AItj2zJRTG
-ZD/cK/jaUNjxrWBPDgJFHH+EJ8O9cIqE5PhszBQgNl8cHcmgOHdOGnwUvPSdOtEX
-vmZJAn851xp8ibRejPY62n8JLEtL5TWUu8bTlytZMN7Sibs1biYrgKoYIWrmBTDt
-4jNp47tDlB7P1cZMzOSAgNMy81gYYR4mGGY1AgMBAAGgADANBgkqhkiG9w0BAQUF
-AAOCAQEAk3eueaq/gonBzKH75oWHlqPbMZQFk4NXqx8h24ZfkCzPEFPyDM+jdQxv
-8vDtyWq+fizqAQmGrM7WPHgnTbmZyovfmwuKwtTlkD/8t7XpTmm9fYspbL4WzdP1
-y8/Vug09te+rni+v+kjk5b9IceEy6kLvXuzirE6c4LunAm+thrr5gWmsx1pyDiq7
-W2M15UZrm/paaCg6cVaMFdXCRZP+g1P4NcgDUe2TyFbLlhOJNtX3DJRZWEhrkEYK
-mRz2tJuiuitCzheAgRrFXepRagHKYffNSas1n/2kIc9QpZ8654kxsAzEwL7CnHd/
-SHbMS9dfP+uM6DACwcvngSOBMJ9KMg==
------END CERTIFICATE REQUEST-----
diff --git a/pulsar-broker/src/test/resources/certificate/client.key b/pulsar-broker/src/test/resources/certificate/client.key
deleted file mode 100644
index 34fc701c5257d..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/client.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCpBXkXcC70VZch
-g91aqJdGTmB1WZWQO64Urhmg53yKegmkHc8+SmamChWQpAcMWGDHc4MZYhSCj6fp
-AOwnFjjKAd9V6CBCB6TyJR+15hWwhwdLRuOSEiKP/eKELfTUWSKputXTr2gu9Rcs
-py/fPEaZ0ECLPSTjEZmEWJ5VG9AItj2zJRTGZD/cK/jaUNjxrWBPDgJFHH+EJ8O9
-cIqE5PhszBQgNl8cHcmgOHdOGnwUvPSdOtEXvmZJAn851xp8ibRejPY62n8JLEtL
-5TWUu8bTlytZMN7Sibs1biYrgKoYIWrmBTDt4jNp47tDlB7P1cZMzOSAgNMy81gY
-YR4mGGY1AgMBAAECggEAcJj3yVhvv0/BhY8+CCYl2K1f7u1GCLbpSleNNTbhLbMM
-9yrwo/OWnGg9Y4USOPQrTNOz81X2id+/oSZ/K67PGCvVJ3qi+rny9WkrzdbAfkAF
-6O0Jr4arRbeBjkK7Rjc3M1EHH6VLx3R5AsNBzfpuogss5FVQXICd/5+1oscLeLEx
-/Fn+51IEn9FUg5vr7ElG51f+zPxexcWHLNoqGjTEIGGtI8/CfTzD9tBV4sIjf/Nc
-Zzfs9XYrChfcrS0U1zDa+L7c5gYfoN6M08sBiuZlhyyO9wgzPlp+XnsrSFv6hUta
-0scjAbN4bh+orQn6zgFN/sjkQnraWXW7pKFLyTR/IQKBgQDVju4IbhE9XRweNgXi
-s3BuGV+HsuFffEf0904/zCuCUcScGb5WCz5+KtlFJ//YxfocHVZajH+4GdCGbWim
-m+H3XvRpWgfK/aBNOXu5ueLbnPYyPjTrcpKRsomeoiV+Jz1tv5PQElwzCiCzVvQf
-fMyhQT16YIsFQAGJzQMBEHWODQKBgQDKnKps3sKSR3ycUtIxCVXUir7p52qst0Pm
-bPO8JrcRKZP2z8MJB96+DcQFzrxj7t5DDktkYEsFOPPuIeUsYXsY+MKHs4hEQVCz
-hpDJJNQ8s+SV8TLzKpinZEmLIjslLbn2rQrpqybPg84VxqX3qqM8IrXhMf77aGj6
-QHqvQwHWyQKBgQDF1RVO+9++j82ncvY6z22coKath5leIjxqgtqbISFBJUxUK0j2
-Xo4yxLDnbqmE/8m1V7wSP8tlGYzhquLiTM+kn/Mc0Ukc0503TMQABmJQfXRYkOXn
-IwkCLXltWdoPpnwyeeGNRCTjJ0OpvyiBLtRFobE498xxPZzvMdrRlpS/1QKBgQCo
-wmMleUnBQ2/kWQugMnFeLg6kjs+IesFAnYFKN0kGL4aB7j06OWbrEFY0rCS4bA6O
-9coQGjCCchSjRXI4TB2XCCQnmX8nsuuADNZt45Iv2XrM9XEFn3Y0/tBO5j0zU2nw
-r+NGC/uwns050BMPPf7mqNarctQ6HZZK0wgdEQfoGQKBgC+pbkQv9cn68TsiaJ3w
-tvNRTXCIAAH4Vtn9Cp+63ao+kXn94BJqQF99i58kJpG4ol6wbCHUoC6fHgxUh5HB
-JB0HjC2eCMgn4acAQg0sPW6l35KX36yYxtrL7eosB/yBYum0XAwmboNjEhlCZkOs
-YOpSsn61g7xqqrt40Spb5vUn
------END PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/certificate/server.crt b/pulsar-broker/src/test/resources/certificate/server.crt
deleted file mode 100644
index 59b651be2a406..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/server.crt
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDLjCCAhYCCQDn/Yvym+FMsDANBgkqhkiG9w0BAQUFADBZMQswCQYDVQQGEwJB
-VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0
-cyBQdHkgTHRkMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTYwNjEzMjIyMTQ2WhcN
-MjYwNjExMjIyMTQ2WjBZMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0
-ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRIwEAYDVQQDEwls
-b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCs29IuzZvk
-OGUkS/wqKzd/h2esqjCSjw4SLLbeh1GA3UEvh1k9+eRiYwJG1yCOHmcsp4A8Du99
-8xbgeihpWWw7pjL5VVky3ciuvHyz1Cc6bKRps/GzVJBwFP0gzHnK8bUM86U52yGT
-1DepD/Y2lURy0igdVcAMjGweMwoTmiaVcwZexfYuEef+jz3fmpmOwP9rboIA9rQr
-mTbLzzkbAwZXdl+bRvIefIjIazIzTOs8tJWrhFaTJUgBhhLjFIwTdpS+n+FqOu8J
-92K+PvKjIeJ3kmnZyRHK7uidlAn0g/DK+co1sX3zORPCWeg21K+/vVHTj91zARNb
-O9hVS4bqqsw9AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBACE0WBuTbHcPtYKv2ZMS
-mYk9jvtAhmWHQ6tNqV8CmS2AsrzZdWglGaqIRsm5slkD2BGeQS+BesTArUuENTmP
-r9kJSecdiiB8aWtLbhoCSH3QR6IW/b5UVl6sR5OIh7SkNTjMSUSDnMEVLNGyKZGS
-gCGVbDf3n5KhOTnwqguELRykynKFt2LVksBia9+88lUtiRHpbyClo/KVWltJlaww
-PT0WEpwqVUcHmwrR3MTzJDEPvIplSgxdaDmFGYS1YKm9T/wQd+t/0DbXMmfJXBbd
-FVUnB6o7qJVU9N2Tbaj9NbCtwz5nTZG4A5kRXWHVjZsn5WzLuS/me3rDXjwlfB2p
-ipY=
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/certificate/server.csr b/pulsar-broker/src/test/resources/certificate/server.csr
deleted file mode 100644
index 8782222c5ab46..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/server.csr
+++ /dev/null
@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICnjCCAYYCAQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx
-ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJbG9j
-YWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArNvSLs2b5Dhl
-JEv8Kis3f4dnrKowko8OEiy23odRgN1BL4dZPfnkYmMCRtcgjh5nLKeAPA7vffMW
-4HooaVlsO6Yy+VVZMt3Irrx8s9QnOmykabPxs1SQcBT9IMx5yvG1DPOlOdshk9Q3
-qQ/2NpVEctIoHVXADIxsHjMKE5omlXMGXsX2LhHn/o8935qZjsD/a26CAPa0K5k2
-y885GwMGV3Zfm0byHnyIyGsyM0zrPLSVq4RWkyVIAYYS4xSME3aUvp/hajrvCfdi
-vj7yoyHid5Jp2ckRyu7onZQJ9IPwyvnKNbF98zkTwlnoNtSvv71R04/dcwETWzvY
-VUuG6qrMPQIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAEPHySnpf3E/7tZsiDka
-rqdB/sU7fdqjyV0iy0cuKQkU8WYrsE7bHkqMYc8CiIDfWhIGW5Jnzups2O6eH0Sx
-2BS21ARFiNGC1UfY1HSV2zrTNh3RqQa3YsXzv9vvdQ/gjsqGDuGDIc1yAA+Ytdja
-3rhIzEVqBhiLzg+M2+gW1zs+Kqj0Zo0pLB2uqhdZJmjxBb2FCli50vCVEhqIS3RO
-KTE+AJfxThWIeahFyVaskaEGkS6NVr2JihV0elbKolH19k2UzRTVn7p3Ixh5ojuW
-gtU/90vOy/SDkSRmCWMqgkUKJ2oeImleHdrvwNyrzvrLWRAz6R5yGQJwji9kKpHD
-FK0=
------END CERTIFICATE REQUEST-----
diff --git a/pulsar-broker/src/test/resources/certificate/server.key b/pulsar-broker/src/test/resources/certificate/server.key
deleted file mode 100644
index 6da70f5aec3b5..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/server.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCs29IuzZvkOGUk
-S/wqKzd/h2esqjCSjw4SLLbeh1GA3UEvh1k9+eRiYwJG1yCOHmcsp4A8Du998xbg
-eihpWWw7pjL5VVky3ciuvHyz1Cc6bKRps/GzVJBwFP0gzHnK8bUM86U52yGT1Dep
-D/Y2lURy0igdVcAMjGweMwoTmiaVcwZexfYuEef+jz3fmpmOwP9rboIA9rQrmTbL
-zzkbAwZXdl+bRvIefIjIazIzTOs8tJWrhFaTJUgBhhLjFIwTdpS+n+FqOu8J92K+
-PvKjIeJ3kmnZyRHK7uidlAn0g/DK+co1sX3zORPCWeg21K+/vVHTj91zARNbO9hV
-S4bqqsw9AgMBAAECggEAd/LuDeZFZ/+uR5qmuAhXMZqfWZSbsges5vW6S/6wkvB1
-vGp6heQzFAbKXKgJgjUcuULeXE6s58RYuppqEnin/1hcBOKxy/dUu9Q14H+2XPdo
-u6TPcvaaZ/xYjnr1hNtnHD6yB8zEpxVbLmjSHJxF7Dti9MA9TTfgCrC2LFYKsicD
-/5AQyHuwpHyTL3Iiwv4Qtks/SD2a3fu8lD0yTQwA/hY6/0ieXxXd9tZV5a6GSA0P
-nieol1byfuX7Q5fb8ggPd9u9K1mVZTBRKiE5w+uU4Ic2IkBmZX5ZuRS+vFplpLsY
-YpFPvzFmpNkpK2SdYjJ+V4tkJsFHmOaFRgW/0QB2DQKBgQDeQMSZBQlPUrgRdWHN
-OyvTcrSvXzg5DbaIj39tgdNZ6PYns/thD0n707KGRJOChIyYiiKxLxzLWdPUxqQO
-rNLUV9IkMVc/QZR8RUqGc2BxmPOxAprhzeOhLsyqP/sgtxRHAnLqmkXuHYoxvTZ6
-LFCRCZBpEJrutGxl3s/x+sfkuwKBgQDHGwnSmvArpL8ZY1dV4xKNkxifCBnNmqAl
-TKHPW3odN9nkMECEt1XUIioUUKXUsiAZNp5xa/v1DEyJ4f2T20QKcAGbS18b1M5W
-axIoH3IhyLo74tuo0fthgq5bzypfFOlIjo7F9mpEky/461RWmoNAAlp9+FkDi48C
-KwjAk39/ZwKBgQDXFJqs8sDFsOlMi+nvsHmDERhmNqG0JN8mXKgWk3KzKc09MuHs
-Vd1lBMNZSHfv8NIWtGdKTKty5yUmXm1ZfkoxECPevpkOMCq/8FZksrb8d+YswLae
-Gp9U1nNdtrkSOdo3tdj7y/wsqQ2ZgOB9bvEwyq6j3lvw8U2NcAiQxf44DQKBgBHb
-lPf0uZHQhutKA61KXoGgLdclrNrKAY8W3nRwqfUw6zQSN9cvcl1Cay/DQ/xdtY9N
-XMyjeMezwLGlOU8nnWSqQxqgmfkvDwqlM82xdFUfYcS5RiZQHxHR3L2TSSOaBoph
-buDGhyV7ZhQXV0slNJxrGZ6uxZ0RyVPSdEiBcjAFAoGBAJqZ6uCVHpv/FwZVggu7
-Xb9EIxZnLSmXwaXFpJoMZpRpKb8cSTTJbgSMv3Dq2LcNKYXdNBhgKgPSc/XipXt9
-ZdT36KWipV+PzW691kUiWHtA8/+E0LCi4Y7rlcBMz9PgDNXK4XMMZOVKxDqPcHSJ
-P6y01ku7T2X+abUiJ334Hg6G
------END PRIVATE KEY-----
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java
index 8d416125fd1b3..c401f3d0bea64 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java
@@ -328,9 +328,8 @@ public void testArgs() throws Exception {
         PulsarClientTool pulsarClientTool = new PulsarClientTool(new Properties());
         final String url = "pulsar+ssl://localhost:6651";
         final String authPlugin = "org.apache.pulsar.client.impl.auth.AuthenticationTls";
-        final String authParams = "tlsCertFile:pulsar-broker/src/test/resources/authentication/tls/client-cert.pem," +
-                "tlsKeyFile:pulsar-broker/src/test/resources/authentication/tls/client-key.pem";
-        final String tlsTrustCertsFilePath = "pulsar/pulsar-broker/src/test/resources/authentication/tls/cacert.pem";
+        final String authParams = String.format("tlsCertFile:%s,tlsKeyFile:%s", getTlsFileForClient("admin.cert"),
+                getTlsFileForClient("admin.key-pk8"));
         final String message = "test msg";
         final int numberOfMessages = 1;
         final String topicName = getTopicWithRandomSuffix("test-topic");
@@ -338,11 +337,11 @@ public void testArgs() throws Exception {
         String[] args = {"--url", url,
                 "--auth-plugin", authPlugin,
                 "--auth-params", authParams,
-                "--tlsTrustCertsFilePath", tlsTrustCertsFilePath,
+                "--tlsTrustCertsFilePath", CA_CERT_FILE_PATH,
                 "produce", "-m", message,
                 "-n", Integer.toString(numberOfMessages), topicName};
         pulsarClientTool.jcommander.parse(args);
-        assertEquals(pulsarClientTool.rootParams.getTlsTrustCertsFilePath(), tlsTrustCertsFilePath);
+        assertEquals(pulsarClientTool.rootParams.getTlsTrustCertsFilePath(), CA_CERT_FILE_PATH);
         assertEquals(pulsarClientTool.rootParams.getAuthParams(), authParams);
         assertEquals(pulsarClientTool.rootParams.getAuthPluginClassName(), authPlugin);
         assertEquals(pulsarClientTool.rootParams.getServiceURL(), url);
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java
index 0bc7c525384df..01c06fbf52f4e 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java
@@ -47,10 +47,6 @@
 import static org.testng.Assert.assertTrue;
 
 public class ProxyServiceTlsStarterTest extends MockedPulsarServiceBaseTest {
-
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem";
-    private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem";
     private ProxyServiceStarter serviceStarter;
     private String serviceUrl;
     private int webPort;
@@ -63,14 +59,14 @@ protected void setup() throws Exception {
         serviceStarter.getConfig().setBrokerServiceURL(pulsar.getBrokerServiceUrl());
         serviceStarter.getConfig().setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls());
         serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress());
-        serviceStarter.getConfig().setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        serviceStarter.getConfig().setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
         serviceStarter.getConfig().setServicePort(Optional.empty());
         serviceStarter.getConfig().setServicePortTls(Optional.of(0));
         serviceStarter.getConfig().setWebServicePort(Optional.of(0));
         serviceStarter.getConfig().setTlsEnabledWithBroker(true);
         serviceStarter.getConfig().setWebSocketServiceEnabled(true);
-        serviceStarter.getConfig().setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        serviceStarter.getConfig().setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        serviceStarter.getConfig().setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        serviceStarter.getConfig().setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
         serviceStarter.getConfig().setBrokerProxyAllowedTargetPorts("*");
         serviceStarter.start();
         serviceUrl = serviceStarter.getProxyService().getServiceUrlTls();
@@ -79,8 +75,8 @@ protected void setup() throws Exception {
 
     protected void doInitConf() throws Exception {
         super.doInitConf();
-        this.conf.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        this.conf.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        this.conf.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        this.conf.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
     }
 
     @Override
@@ -94,7 +90,7 @@ protected void cleanup() throws Exception {
     public void testProducer() throws Exception {
         @Cleanup
         PulsarClient client = PulsarClient.builder().serviceUrl(serviceUrl)
-                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .build();
 
         @Cleanup
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java
index e1cf62aafa8a3..64b0cd6b1a610 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java
@@ -43,10 +43,6 @@
 
 public class ProxyTlsTest extends MockedPulsarServiceBaseTest {
 
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem";
-    private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem";
-
     private ProxyService proxyService;
     private ProxyConfiguration proxyConfig = new ProxyConfiguration();
 
@@ -61,8 +57,8 @@ protected void setup() throws Exception {
         proxyConfig.setWebServicePort(Optional.of(0));
         proxyConfig.setWebServicePortTls(Optional.of(0));
         proxyConfig.setTlsEnabledWithBroker(false);
-        proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        proxyConfig.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
         proxyConfig.setMetadataStoreUrl(DUMMY_VALUE);
         proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE);
 
@@ -87,7 +83,7 @@ public void testProducer() throws Exception {
         @Cleanup
         PulsarClient client = PulsarClient.builder()
                 .serviceUrl(proxyService.getServiceUrlTls())
-                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).build();
+                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
         Producer producer = client.newProducer(Schema.BYTES).topic("persistent://sample/test/local/topic").create();
 
         for (int i = 0; i < 10; i++) {
@@ -100,7 +96,7 @@ public void testPartitions() throws Exception {
         @Cleanup
         PulsarClient client = PulsarClient.builder()
                 .serviceUrl(proxyService.getServiceUrlTls())
-                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).build();
+                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
         TenantInfoImpl tenantInfo = createDefaultTenantInfo();
         admin.tenants().createTenant("sample", tenantInfo);
         admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2);
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
index d5b70dfa03756..f77c0eeb2d41c 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
@@ -35,10 +35,6 @@
 
 public class ProxyTlsTestWithAuth extends MockedPulsarServiceBaseTest {
 
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem";
-    private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem";
-
     private ProxyService proxyService;
     private ProxyConfiguration proxyConfig = new ProxyConfiguration();
 
@@ -63,8 +59,8 @@ protected void setup() throws Exception {
         proxyConfig.setWebServicePort(Optional.of(0));
         proxyConfig.setWebServicePortTls(Optional.of(0));
         proxyConfig.setTlsEnabledWithBroker(true);
-        proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        proxyConfig.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
         proxyConfig.setMetadataStoreUrl(DUMMY_VALUE);
         proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE);
         proxyConfig.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2");

From b88c48ac17a3a034abc0b975ed3ddd06e51c1c47 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sat, 20 May 2023 00:28:35 +0800
Subject: [PATCH 217/280] [improve][broker] Copy BrokerEntryMetadata when
 rebatchMessage (#20337)

---
 .../pulsar/client/impl/RawBatchConverter.java | 21 +++++++++-
 .../pulsar/client/impl/RawReaderTest.java     | 40 +++++++++++++++++++
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
index 4d718f71a2e23..4809ce1a04807 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
@@ -19,7 +19,9 @@
 package org.apache.pulsar.client.impl;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.pulsar.common.protocol.Commands.magicBrokerEntryMetadata;
 import io.netty.buffer.ByteBuf;
+import io.netty.buffer.CompositeByteBuf;
 import io.netty.buffer.Unpooled;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -92,6 +94,14 @@ public static Optional rebatchMessage(RawMessage msg,
         checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
 
         ByteBuf payload = msg.getHeadersAndPayload();
+        int readerIndex = payload.readerIndex();
+        ByteBuf brokerMeta = null;
+        if (payload.getShort(readerIndex) == magicBrokerEntryMetadata) {
+            payload.skipBytes(Short.BYTES);
+            int brokerEntryMetadataSize = payload.readInt();
+            payload.readerIndex(readerIndex);
+            brokerMeta = payload.readRetainedSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES);
+        }
         MessageMetadata metadata = Commands.parseMessageMetadata(payload);
         ByteBuf batchBuffer = PulsarByteBufAllocator.DEFAULT.buffer(payload.capacity());
 
@@ -139,8 +149,15 @@ public static Optional rebatchMessage(RawMessage msg,
 
                 ByteBuf metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c,
                                                                                   metadata, compressedPayload);
-                Optional result = Optional.of(new RawMessageImpl(msg.getMessageIdData(),
-                                                                             metadataAndPayload));
+
+                if (brokerMeta != null) {
+                    CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeDirectBuffer();
+                    compositeByteBuf.addComponents(true, brokerMeta, metadataAndPayload);
+                    metadataAndPayload = compositeByteBuf;
+                }
+
+                Optional result =
+                        Optional.of(new RawMessageImpl(msg.getMessageIdData(), metadataAndPayload));
                 metadataAndPayload.release();
                 compressedPayload.release();
                 return result;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
index fb09cd9951885..a201ef104e7b3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
@@ -39,6 +39,7 @@
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.RawMessage;
 import org.apache.pulsar.client.api.RawReader;
+import org.apache.pulsar.common.api.proto.BrokerEntryMetadata;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
@@ -57,6 +58,11 @@ public class RawReaderTest extends MockedPulsarServiceBaseTest {
     @BeforeMethod
     @Override
     public void setup() throws Exception {
+        conf.setBrokerEntryMetadataInterceptors(org.assertj.core.util.Sets.newTreeSet(
+                "org.apache.pulsar.common.intercept.AppendBrokerTimestampMetadataInterceptor",
+                "org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor"
+        ));
+        conf.setExposingBrokerEntryMetadataToClientEnabled(true);
         super.internalSetup();
 
         admin.clusters().createCluster("test",
@@ -384,6 +390,40 @@ public void testBatchingRebatch() throws Exception {
         }
     }
 
+    @Test
+    public void testBatchingRebatchWithBrokerEntryMetadata() throws Exception {
+        String topic = "persistent://my-property/my-ns/my-raw-topic";
+
+        try (Producer producer = pulsarClient.newProducer().topic(topic)
+                .maxPendingMessages(3)
+                .enableBatching(true)
+                .batchingMaxMessages(3)
+                .batchingMaxPublishDelay(1, TimeUnit.HOURS)
+                .messageRoutingMode(MessageRoutingMode.SinglePartition)
+                .create()) {
+            producer.newMessage().key("key1").value("my-content-1".getBytes()).sendAsync();
+            producer.newMessage().key("key2").value("my-content-2".getBytes()).sendAsync();
+            producer.newMessage().key("key3").value("my-content-3".getBytes()).send();
+        }
+
+        RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
+        try (RawMessage m1 = reader.readNextAsync().get()) {
+            RawMessage m2 = RawBatchConverter.rebatchMessage(m1, (key, id) -> key.equals("key2")).get();
+            BrokerEntryMetadata brokerEntryMetadata =
+                    Commands.parseBrokerEntryMetadataIfExist(m2.getHeadersAndPayload());
+            Assert.assertNotNull(brokerEntryMetadata);
+            Assert.assertEquals(brokerEntryMetadata.getIndex(), 2);
+            Assert.assertTrue(brokerEntryMetadata.getBrokerTimestamp() < System.currentTimeMillis());
+            List> idsAndKeys = RawBatchConverter.extractIdsAndKeysAndSize(m2);
+            Assert.assertEquals(idsAndKeys.size(), 1);
+            Assert.assertEquals(idsAndKeys.get(0).getMiddle(), "key2");
+            m2.close();
+            Assert.assertEquals(m1.getHeadersAndPayload().refCnt(), 1);
+        } finally {
+            reader.closeAsync().get();
+        }
+    }
+
     @Test
     public void testAcknowledgeWithProperties() throws Exception {
         int numKeys = 10;

From 1e664b7f550ffa28d3c810f3b7d6d625d5905eb3 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Sat, 20 May 2023 03:38:24 +0800
Subject: [PATCH 218/280] [fix] [broker] In Key_Shared mode: remove unnecessary
 mechanisms of message skip to avoid unnecessary consumption stuck (#20335)

### Motivation
- https://github.com/apache/pulsar/pull/7105 provide a mechanism to avoid a stuck consumer affecting the consumption of other consumers:
  - if all consumers can not accept more messages, stop delivering messages to the client.
  - if one consumer can not accept more messages, just read new messages and deliver them to other consumers.
- https://github.com/apache/pulsar/pull/7553 provide a mechanism to fix the issue of lost order of consumption: If the consumer cannot accept any more messages, skip the consumer for the next round of message delivery because there may be messages with the same key in the replay queue.
- https://github.com/apache/pulsar/pull/10762 provide a mechanism to fix the issue of lost order of consumption: If there have any messages with the same key in the replay queue, do not deliver the new messages to this consumer.

https://github.com/apache/pulsar/pull/10762 and https://github.com/apache/pulsar/pull/7553 do the same thing and https://github.com/apache/pulsar/pull/10762 is better than https://github.com/apache/pulsar/pull/7553 , so https://github.com/apache/pulsar/pull/7553 is unnecessary.

### Modifications
remove the mechanism provided by https://github.com/apache/pulsar/pull/7553 to avoid unnecessary consumption stuck.
---
 ...tStickyKeyDispatcherMultipleConsumers.java | 24 +------------------
 1 file changed, 1 insertion(+), 23 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index 1a8c6e180a2a2..8f05530f58bfa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -71,17 +71,12 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
      */
     private final LinkedHashMap recentlyJoinedConsumers;
 
-    private final Set stuckConsumers;
-    private final Set nextStuckConsumers;
-
     PersistentStickyKeyDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor,
             Subscription subscription, ServiceConfiguration conf, KeySharedMeta ksm) {
         super(topic, cursor, subscription, ksm.isAllowOutOfOrderDelivery());
 
         this.allowOutOfOrderDelivery = ksm.isAllowOutOfOrderDelivery();
         this.recentlyJoinedConsumers = allowOutOfOrderDelivery ? null : new LinkedHashMap<>();
-        this.stuckConsumers = new HashSet<>();
-        this.nextStuckConsumers = new HashSet<>();
         this.keySharedMode = ksm.getKeySharedMode();
         switch (this.keySharedMode) {
         case AUTO_SPLIT:
@@ -226,8 +221,6 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
             }
         }
 
-        nextStuckConsumers.clear();
-
         final Map> groupedEntries = localGroupedEntries.get();
         groupedEntries.clear();
         final Map> consumerStickyKeyHashesMap = new HashMap<>();
@@ -318,14 +311,11 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
         // acquire message-dispatch permits for already delivered messages
         acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent);
 
-        stuckConsumers.clear();
-
         if (totalMessagesSent == 0 && (recentlyJoinedConsumers == null || recentlyJoinedConsumers.isEmpty())) {
             // This means, that all the messages we've just read cannot be dispatched right now.
             // This condition can only happen when:
             //  1. We have consumers ready to accept messages (otherwise the would not haven been triggered)
             //  2. All keys in the current set of messages are routing to consumers that are currently busy
-            //     and stuck is not caused by stuckConsumers
             //
             // The solution here is to move on and read next batch of messages which might hopefully contain
             // also keys meant for other consumers.
@@ -334,10 +324,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
             // ahead in the stream while the new consumers are not ready to accept the new messages,
             // therefore would be most likely only increase the distance between read-position and mark-delete
             // position.
-            if (!nextStuckConsumers.isEmpty()) {
-                isDispatcherStuckOnReplays = true;
-                stuckConsumers.addAll(nextStuckConsumers);
-            }
+            isDispatcherStuckOnReplays = true;
             return true;
         }  else if (currentThreadKeyNumber == 0) {
             return true;
@@ -348,8 +335,6 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
     private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List entries, int maxMessages,
             ReadType readType, Set stickyKeyHashes) {
         if (maxMessages == 0) {
-            // the consumer was stuck
-            nextStuckConsumers.add(consumer);
             return 0;
         }
         if (readType == ReadType.Normal && stickyKeyHashes != null
@@ -366,13 +351,6 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List en
         // At this point, all the old messages were already consumed and this consumer
         // is now ready to receive any message
         if (maxReadPosition == null) {
-            // stop to dispatch by stuckConsumers
-            if (stuckConsumers.contains(consumer)) {
-                if (log.isDebugEnabled()) {
-                    log.debug("[{}] stop to dispatch by stuckConsumers, consumer: {}", name, consumer);
-                }
-                return 0;
-            }
             // The consumer has not recently joined, so we can send all messages
             return maxMessages;
         }

From eae4bd069e3af60c2a8815453dc057c768dce4a9 Mon Sep 17 00:00:00 2001
From: Paul Gier 
Date: Sat, 20 May 2023 11:34:07 -0500
Subject: [PATCH 219/280] [fix][broker] update rest endpoint method names to
 avoid swagger conflict (#20359)

Signed-off-by: Paul Gier 
---
 .../pulsar/broker/admin/v2/Namespaces.java    |  6 ++--
 src/gen-swagger.sh                            | 33 -------------------
 2 files changed, 3 insertions(+), 36 deletions(-)
 delete mode 100755 src/gen-swagger.sh

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
index 55766c173f71f..b4f1194f92f0a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
@@ -2519,7 +2519,7 @@ public void getMaxTopicsPerNamespace(@Suspended final AsyncResponse asyncRespons
     @ApiOperation(value = "Set maxTopicsPerNamespace config on a namespace.")
     @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
             @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), })
-    public void setInactiveTopicPolicies(@PathParam("tenant") String tenant,
+    public void setMaxTopicsPerNamespace(@PathParam("tenant") String tenant,
                                          @PathParam("namespace") String namespace,
                                          @ApiParam(value = "Number of maximum topics for specific namespace",
                                                  required = true) int maxTopicsPerNamespace) {
@@ -2529,10 +2529,10 @@ public void setInactiveTopicPolicies(@PathParam("tenant") String tenant,
 
     @DELETE
     @Path("/{tenant}/{namespace}/maxTopicsPerNamespace")
-    @ApiOperation(value = "Set maxTopicsPerNamespace config on a namespace.")
+    @ApiOperation(value = "Remove maxTopicsPerNamespace config on a namespace.")
     @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
             @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), })
-    public void setInactiveTopicPolicies(@PathParam("tenant") String tenant,
+    public void removeMaxTopicsPerNamespace(@PathParam("tenant") String tenant,
                                          @PathParam("namespace") String namespace) {
         validateNamespaceName(tenant, namespace);
         internalRemoveMaxTopicsPerNamespace();
diff --git a/src/gen-swagger.sh b/src/gen-swagger.sh
deleted file mode 100755
index 6402dca261408..0000000000000
--- a/src/gen-swagger.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-PULSAR_PATH=$(git rev-parse --show-toplevel)
-
-cd $PULSAR_PATH
-
-echo "Generating swagger json file for master ..."
-mvn -am -pl pulsar-broker install -DskipTests -Pswagger
-echo "Swagger json file is generated for master."
-
-mkdir -p site2/website/static/swagger/master/
-
-cp pulsar-broker/target/docs/swagger*.json site2/website/static/swagger/master/
-echo "Copied swagger json file for master."
-

From aa7decc5b75894e864f3c5962c5cffad255abf25 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Sat, 20 May 2023 09:34:27 -0700
Subject: [PATCH 220/280] [fix][broker]
 managedLedger.getConfig().getProperties().putAll(properties) NPE (#20361)

---
 .../apache/pulsar/broker/admin/impl/PersistentTopicsBase.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index ed7cd70c6411c..e0f168eb8d842 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -699,7 +699,11 @@ private CompletableFuture internalUpdateNonPartitionedTopicProperties(Map<
 
                     @Override
                     public void updatePropertiesComplete(Map properties, Object ctx) {
+                        if (managedLedger.getConfig().getProperties() == null) {
+                            managedLedger.getConfig().setProperties(new HashMap<>());
+                        }
                         managedLedger.getConfig().getProperties().putAll(properties);
+
                         future.complete(null);
                     }
 

From a9f2f28891b4c0f16c31e0b5ec8667e396479e90 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Sun, 21 May 2023 17:50:44 -0700
Subject: [PATCH 221/280] [improve][broker] Gracefully shut down load balancer
 extension (#20315)

### Motivation

Load balancer extension needs to shut down gracefully, especially when shutting down the leader broker. When the leader broker closes the leader election service too late, service unit lookups to the leader broker could fail during the shutdown. This could delay client reconnection time.

Lookup failure logs
```
(shutdown starts)
pulsar-broker-8 pulsar-broker 2023-04-22T00:19:52,630+0000 [pulsar-service-shutdown] INFO  org.apache.pulsar.broker.PulsarService - Closing PulsarService
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,690+0000 [pulsar-io-4-5] INFO  org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [pulsar-18-9] Closed connection [id: 0x907e74b0, L:/10.0.13.6:35838 ! R:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local/10.0.18.18:6650] -- Will try again in 0.1 s
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,691+0000 [pulsar-io-4-5] INFO  org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [reader-30e6b40dd9] Closed connection [id: 0x907e74b0, L:/10.0.13.6:35838 ! R:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local/10.0.18.18:6650] -- Will try again in 0.1 s

(znode is gone)
pulsar-broker-8 pulsar-broker 2023-04-22T00:19:52,652+0000 [pulsar-load-manager-1-1] INFO  org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl - BrokerRegistry detected the broker:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 registry has been deleted.

(lookup failure)
org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [reader-30e6b40dd9] Could not get connection to broker: org.apache.pulsar.client.api.PulsarClientException$ConnectException: Disconnected from server at pulsar-broker-3.pulsar-broker.pulsar.svc.cluster.local/10.0.13.6:6650 -- Will try again in 0.193 s
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,827+0000 [main-EventThread] INFO  org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup null for topic persistent://pulsar/system/loadbalancer-service-unit-state with error Failed to look up a broker registry:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 for bundle:pulsar/system/0x40000000_0x50000000
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,827+0000 [main-EventThread] INFO  org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup null for topic persistent://pulsar/system/loadbalancer-service-unit-state with error Failed to look up a broker registry:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 for bundle:pulsar/system/0x40000000_0x50000000

(leader election service has been closed)
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:55,569+0000 [pulsar-load-manager-1-1] INFO  org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl - This broker:pulsar-broker-3.pulsar-broker.pulsar.svc.cluster.local:8080 plays the leader now.

```

### Modifications
During the shutdown flow, when calling loadbalancer.disableBroker(), the load balancer extension gracefully gives up the owned bundles to other brokers. After that, it closes the leader election service and removes its register in the metadata store.
---
 .../apache/pulsar/broker/PulsarService.java   |  22 +++-
 .../extensions/ExtensibleLoadManagerImpl.java |  17 +++
 .../ExtensibleLoadManagerWrapper.java         |   2 +-
 .../channel/ServiceUnitStateChannel.java      |   5 +
 .../channel/ServiceUnitStateChannelImpl.java  | 112 +++++++++++++++---
 .../ExtensibleLoadManagerImplTest.java        |  45 +++++++
 .../channel/ServiceUnitStateChannelTest.java  |  65 ++++++----
 7 files changed, 222 insertions(+), 46 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index f833674ceeb0d..62d4634fa2d62 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -382,6 +382,17 @@ public void closeMetadataServiceSession() throws Exception {
         localMetadataStore.close();
     }
 
+    private void closeLeaderElectionService() throws Exception {
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService().close();
+        } else {
+            if (this.leaderElectionService != null) {
+                this.leaderElectionService.close();
+                this.leaderElectionService = null;
+            }
+        }
+    }
+
     @Override
     public void close() throws PulsarServerException {
         try {
@@ -502,10 +513,7 @@ public CompletableFuture closeAsync() {
                 this.bkClientFactory = null;
             }
 
-            if (this.leaderElectionService != null) {
-                this.leaderElectionService.close();
-                this.leaderElectionService = null;
-            }
+            closeLeaderElectionService();
 
             if (adminClient != null) {
                 adminClient.close();
@@ -1316,7 +1324,11 @@ public boolean isRunning() {
      * @return a reference of the current LeaderElectionService instance.
      */
     public LeaderElectionService getLeaderElectionService() {
-        return this.leaderElectionService;
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            return ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService();
+        } else {
+            return this.leaderElectionService;
+        }
     }
 
     /**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 4ebf537f7a8a8..cbaed8ee8f94f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -26,6 +26,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -380,12 +381,22 @@ public CompletableFuture> assign(Optional> selectAsync(ServiceUnitId bundle) {
+        return selectAsync(bundle, Collections.emptySet());
+    }
+
+    public CompletableFuture> selectAsync(ServiceUnitId bundle,
+                                                           Set excludeBrokerSet) {
         BrokerRegistry brokerRegistry = getBrokerRegistry();
         return brokerRegistry.getAvailableBrokerLookupDataAsync()
                 .thenCompose(availableBrokers -> {
                     LoadManagerContext context = this.getContext();
 
                     Map availableBrokerCandidates = new HashMap<>(availableBrokers);
+                    if (!excludeBrokerSet.isEmpty()) {
+                        for (String exclude : excludeBrokerSet) {
+                            availableBrokerCandidates.remove(exclude);
+                        }
+                    }
 
                     // Filter out brokers that do not meet the rules.
                     List filterPipeline = getBrokerFilterPipeline();
@@ -685,4 +696,10 @@ private void monitor() {
             log.error("Failed to get the channel ownership.", e);
         }
     }
+
+    public void disableBroker() throws Exception {
+        serviceUnitStateChannel.cleanOwnerships();
+        leaderElectionService.close();
+        brokerRegistry.unregister();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java
index 1eabbe620e213..18e949537dedb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java
@@ -74,7 +74,7 @@ public CompletableFuture checkOwnershipAsync(Optional to
 
     @Override
     public void disableBroker() throws Exception {
-        this.loadManager.getBrokerRegistry().unregister();
+        this.loadManager.disableBroker();
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index 4782a31fe0c56..6e75fe91a914f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -206,4 +206,9 @@ public interface ServiceUnitStateChannel extends Closeable {
      * Cancels the ownership monitor.
      */
     void cancelOwnershipMonitor();
+
+    /**
+     * Cleans the service unit ownerships from the current broker's channel.
+     */
+    void cleanOwnerships();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index b4c4e7fd5d42a..6246c26e57bcc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -91,7 +91,6 @@
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
 import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceBundles;
-import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicDomain;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.stats.Metrics;
@@ -110,6 +109,10 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
 
     public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD;
     private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec
+
+    private static final int OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS = 5000;
+    private static final int OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS = 100;
+    private static final int OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS = 3000;
     public static final long VERSION_ID_INIT = 1; // initial versionId
     private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60;
     public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins
@@ -257,6 +260,11 @@ public void cancelOwnershipMonitor() {
         }
     }
 
+    @Override
+    public void cleanOwnerships() {
+        doCleanup(lookupServiceAddress);
+    }
+
     public synchronized void start() throws PulsarServerException {
         if (!validateChannelState(LeaderElectionServiceStarted, false)) {
             throw new IllegalStateException("Invalid channel state:" + channelState.name());
@@ -284,7 +292,7 @@ public synchronized void start() throws PulsarServerException {
                 }
             }
             PulsarClusterMetadataSetup.createNamespaceIfAbsent
-                    (pulsar.getPulsarResources(), NamespaceName.SYSTEM_NAMESPACE, config.getClusterName());
+                    (pulsar.getPulsarResources(), SYSTEM_NAMESPACE, config.getClusterName());
 
             producer = pulsar.getClient().newProducer(schema)
                     .enableBatching(true)
@@ -694,6 +702,8 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) {
         if (isTargetBroker(data.dstBroker())) {
             log(null, serviceUnit, data, null);
             lastOwnEventHandledAt = System.currentTimeMillis();
+        } else if (data.force() && isTargetBroker(data.sourceBroker())) {
+            closeServiceUnit(serviceUnit);
         }
     }
 
@@ -1108,21 +1118,23 @@ private void scheduleCleanup(String broker, long delayInSecs) {
 
 
     private ServiceUnitStateData getOverrideInactiveBrokerStateData(ServiceUnitStateData orphanData,
-                                                                    String selectedBroker) {
+                                                                    String selectedBroker,
+                                                                    String inactiveBroker) {
         if (orphanData.state() == Splitting) {
             return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker,
                     Map.copyOf(orphanData.splitServiceUnitToDestBroker()),
                     true, getNextVersionId(orphanData));
         } else {
-            return new ServiceUnitStateData(Owned, selectedBroker, true, getNextVersionId(orphanData));
+            return new ServiceUnitStateData(Owned, selectedBroker, inactiveBroker,
+                    true, getNextVersionId(orphanData));
         }
     }
 
-    private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData) {
-
-        Optional selectedBroker = selectBroker(serviceUnit);
+    private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, String inactiveBroker) {
+        Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker);
         if (selectedBroker.isPresent()) {
-            var override = getOverrideInactiveBrokerStateData(orphanData, selectedBroker.get());
+            var override = getOverrideInactiveBrokerStateData(
+                    orphanData, selectedBroker.get(), inactiveBroker);
             log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}",
                     serviceUnit, orphanData, override);
             publishOverrideEventAsync(serviceUnit, orphanData, override)
@@ -1140,26 +1152,69 @@ private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanDa
         }
     }
 
+    private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) {
+        long started = System.currentTimeMillis();
+        while (System.currentTimeMillis() - started < maxWaitTimeInMillis) {
+            boolean cleaned = true;
+            for (var etr : tableview.entrySet()) {
+                var serviceUnit = etr.getKey();
+                var data = etr.getValue();
+
+                if (excludeSystemTopics && serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+                    continue;
+                }
+
+                if (data.state() == Owned && broker.equals(data.dstBroker())) {
+                    cleaned = false;
+                    break;
+                }
+            }
+            if (cleaned) {
+                try {
+                    MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS);
+                } catch (InterruptedException e) {
+                    log.warn("Interrupted while gracefully waiting for the cleanup convergence.");
+                }
+                break;
+            } else {
+                try {
+                    MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS);
+                } catch (InterruptedException e) {
+                    log.warn("Interrupted while delaying the next service unit clean-up. Cleaning broker:{}",
+                            lookupServiceAddress);
+                }
+            }
+        }
+    }
 
-    private void doCleanup(String broker)  {
+    private synchronized void doCleanup(String broker)  {
         long startTime = System.nanoTime();
         log.info("Started ownership cleanup for the inactive broker:{}", broker);
         int orphanServiceUnitCleanupCnt = 0;
         long totalCleanupErrorCntStart = totalCleanupErrorCnt.get();
 
+        Map orphanSystemServiceUnits = new HashMap<>();
         for (var etr : tableview.entrySet()) {
             var stateData = etr.getValue();
             var serviceUnit = etr.getKey();
             var state = state(stateData);
             if (StringUtils.equals(broker, stateData.dstBroker())) {
                 if (isActiveState(state)) {
-                    overrideOwnership(serviceUnit, stateData);
+                    if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+                        orphanSystemServiceUnits.put(serviceUnit, stateData);
+                    } else {
+                        overrideOwnership(serviceUnit, stateData, broker);
+                    }
                     orphanServiceUnitCleanupCnt++;
                 }
 
             } else if (StringUtils.equals(broker, stateData.sourceBroker())) {
                 if (isInFlightState(state)) {
-                    overrideOwnership(serviceUnit, stateData);
+                    if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+                        orphanSystemServiceUnits.put(serviceUnit, stateData);
+                    } else {
+                        overrideOwnership(serviceUnit, stateData, broker);
+                    }
                     orphanServiceUnitCleanupCnt++;
                 }
             }
@@ -1168,14 +1223,33 @@ private void doCleanup(String broker)  {
         try {
             producer.flush();
         } catch (PulsarClientException e) {
-            log.error("Failed to flush the in-flight messages.", e);
+            log.error("Failed to flush the in-flight non-system bundle override messages.", e);
         }
 
+
         if (orphanServiceUnitCleanupCnt > 0) {
+            // System bundles can contain this channel's system topic and other important system topics.
+            // Cleaning such system bundles(closing the system topics) together with the non-system bundles
+            // can cause the cluster to be temporarily unstable.
+            // Hence, we clean the non-system bundles first and gracefully wait for them.
+            // After that, we clean the system bundles, if any.
+            waitForCleanups(broker, true, OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS);
             this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt;
             this.totalInactiveBrokerCleanupCnt++;
         }
 
+        // clean system bundles in the end
+        for (var orphanSystemServiceUnit : orphanSystemServiceUnits.entrySet()) {
+            log.info("Overriding orphan system service unit:{}", orphanSystemServiceUnit.getKey());
+            overrideOwnership(orphanSystemServiceUnit.getKey(), orphanSystemServiceUnit.getValue(), broker);
+        }
+
+        try {
+            producer.flush();
+        } catch (PulsarClientException e) {
+            log.error("Failed to flush the in-flight system bundle override messages.", e);
+        }
+
         double cleanupTime = TimeUnit.NANOSECONDS
                 .toMillis((System.nanoTime() - startTime));
 
@@ -1194,9 +1268,9 @@ private void doCleanup(String broker)  {
 
     }
 
-    private Optional selectBroker(String serviceUnit) {
+    private Optional selectBroker(String serviceUnit, String inactiveBroker) {
         try {
-            return loadManager.selectAsync(getNamespaceBundle(serviceUnit))
+            return loadManager.selectAsync(getNamespaceBundle(serviceUnit), Set.of(inactiveBroker))
                     .get(inFlightStateWaitingTimeInMillis, MILLISECONDS);
         } catch (Throwable e) {
             log.error("Failed to select a broker for serviceUnit:{}", serviceUnit);
@@ -1204,8 +1278,10 @@ private Optional selectBroker(String serviceUnit) {
         return Optional.empty();
     }
 
-    private Optional getRollForwardStateData(String serviceUnit, long nextVersionId) {
-        Optional selectedBroker = selectBroker(serviceUnit);
+    private Optional getRollForwardStateData(String serviceUnit,
+                                                                   String inactiveBroker,
+                                                                   long nextVersionId) {
+        Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker);
         if (selectedBroker.isEmpty()) {
             return Optional.empty();
         }
@@ -1220,7 +1296,7 @@ private Optional getOverrideInFlightStateData(
         var state = orphanData.state();
         switch (state) {
             case Assigning: {
-                return getRollForwardStateData(serviceUnit, nextVersionId);
+                return getRollForwardStateData(serviceUnit, orphanData.dstBroker(), nextVersionId);
             }
             case Splitting: {
                 return Optional.of(new ServiceUnitStateData(Splitting,
@@ -1233,7 +1309,7 @@ private Optional getOverrideInFlightStateData(
                     // rollback to the src
                     return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId));
                 } else {
-                    return getRollForwardStateData(serviceUnit, nextVersionId);
+                    return getRollForwardStateData(serviceUnit, orphanData.sourceBroker(), nextVersionId);
                 }
             }
             default: {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index b71eeb4745b87..b131aff68d909 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -880,6 +880,51 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
         assertEquals(actual, expected);
     }
 
+    @Test
+    public void testDisableBroker() throws Exception {
+        // Test rollback to modular load manager.
+        ServiceConfiguration defaultConf = getDefaultConf();
+        defaultConf.setAllowAutoTopicCreation(true);
+        defaultConf.setForceDeleteNamespaceAllowed(true);
+        defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+        defaultConf.setLoadBalancerSheddingEnabled(false);
+        try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) {
+            var pulsar3 = additionalPulsarTestContext.getPulsarService();
+            ExtensibleLoadManagerImpl ternaryLoadManager = spy((ExtensibleLoadManagerImpl)
+                    FieldUtils.readField(pulsar3.getLoadManager().get(), "loadManager", true));
+            String topic = "persistent://public/default/test";
+
+            String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic);
+            TopicName topicName = TopicName.get("test");
+            NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
+            if (!pulsar3.getBrokerServiceUrl().equals(lookupResult1)) {
+                admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(),
+                        pulsar3.getLookupServiceAddress());
+                lookupResult1 = pulsar2.getAdminClient().lookups().lookupTopic(topic);
+            }
+            String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic);
+            String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic);
+
+            assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl());
+            assertEquals(lookupResult1, lookupResult2);
+            assertEquals(lookupResult1, lookupResult3);
+
+
+            assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            assertTrue(ternaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+
+            ternaryLoadManager.disableBroker();
+
+            assertFalse(ternaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            if (primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()) {
+                assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            } else {
+                assertTrue(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            }
+        }
+    }
+
     private static abstract class MockBrokerFilter implements BrokerFilter {
 
         @Override
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 77c80187a63e1..cb26c460f0a03 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -515,7 +515,7 @@ public void transferTestWhenDestBrokerFails()
 
         // recovered, check the monitor update state : Assigned -> Owned
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         FieldUtils.writeDeclaredField(channel2, "producer", producer, true);
         FieldUtils.writeDeclaredField(channel1,
                 "inFlightStateWaitingTimeInMillis", 1 , true);
@@ -713,8 +713,8 @@ public void handleBrokerDeletionEventTest()
 
         var cleanupJobs1 = getCleanupJobs(channel1);
         var cleanupJobs2 = getCleanupJobs(channel2);
-        var leaderCleanupJobs = spy(cleanupJobs1);
-        var followerCleanupJobs = spy(cleanupJobs2);
+        var leaderCleanupJobsTmp = spy(cleanupJobs1);
+        var followerCleanupJobsTmp = spy(cleanupJobs2);
         var leaderChannel = channel1;
         var followerChannel = channel2;
         String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get();
@@ -723,10 +723,12 @@ public void handleBrokerDeletionEventTest()
         if (leader.equals(lookupServiceAddress2)) {
             leaderChannel = channel2;
             followerChannel = channel1;
-            var tmp = followerCleanupJobs;
-            followerCleanupJobs = leaderCleanupJobs;
-            leaderCleanupJobs = tmp;
+            var tmp = followerCleanupJobsTmp;
+            followerCleanupJobsTmp = leaderCleanupJobsTmp;
+            leaderCleanupJobsTmp = tmp;
         }
+        final var leaderCleanupJobs = leaderCleanupJobsTmp;
+        final var followerCleanupJobs = followerCleanupJobsTmp;
         FieldUtils.writeDeclaredField(leaderChannel, "cleanupJobs", leaderCleanupJobs,
                 true);
         FieldUtils.writeDeclaredField(followerChannel, "cleanupJobs", followerCleanupJobs,
@@ -735,7 +737,7 @@ public void handleBrokerDeletionEventTest()
         var owner1 = channel1.getOwnerAsync(bundle1);
         var owner2 = channel2.getOwnerAsync(bundle2);
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         assertTrue(owner1.get().isEmpty());
         assertTrue(owner2.get().isEmpty());
 
@@ -769,9 +771,11 @@ public void handleBrokerDeletionEventTest()
         verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
 
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
 
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -797,8 +801,12 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(2)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(1, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(1, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -814,8 +822,12 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(2)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -833,8 +845,11 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(1, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(1, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -852,8 +867,11 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 2,
                 0,
@@ -878,8 +896,11 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 2,
                 0,
@@ -1101,7 +1122,7 @@ public void assignTestWhenDestBrokerProducerFails()
         FieldUtils.writeDeclaredField(channel2,
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         channel1.publishAssignEventAsync(bundle, lookupServiceAddress2);
         // channel1 is broken. the assign won't be complete.
         waitUntilState(channel1, bundle);
@@ -1440,7 +1461,7 @@ public void testOverrideInactiveBrokerStateData()
 
         // test stable metadata state
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         leaderChannel.handleMetadataSessionEvent(SessionReestablished);
         followerChannel.handleMetadataSessionEvent(SessionReestablished);
         FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp",
@@ -1505,7 +1526,7 @@ public void testOverrideOrphanStateData()
 
         // test stable metadata state
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         FieldUtils.writeDeclaredField(leaderChannel, "inFlightStateWaitingTimeInMillis",
                 -1, true);
         FieldUtils.writeDeclaredField(followerChannel, "inFlightStateWaitingTimeInMillis",

From b7f0004313ea4565717cc6d3c0b99aee5c079c6c Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Mon, 22 May 2023 08:51:10 +0800
Subject: [PATCH 222/280] [fix][broker] Fix broker load manager class filter
 NPE (#20350)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation
When upgrading the pulsar version and changing the pulsar load manager to `ExtensibleLoadManagerImpl` it might cause NPE. The root cause is the old version of pulsar does not contain the `loadManagerClassName` field.
```
2023-05-18T05:42:50,557+0000 [pulsar-io-4-1] INFO  org.apache.pulsar.broker.service.ServerCnx - [/127.0.0.6:51345] connected with role=[pulsarinstance-v3-0-n@test.dev](mailto:pulsarinstance-v3-0-n@test.dev) using authMethod=token, clientVersion=Pulsar Go 0.9.0, clientProtocolVersion=18, proxyVersion=null
2023-05-18T05:42:50,558+0000 [pulsar-io-4-1] WARN  org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup [pulsarinstance-v3-0-n@test.dev](mailto:pulsarinstance-v3-0-n@test.dev) for topic persistent://xxx with error java.lang.NullPointerException: Cannot invoke “String.equals(Object)” because the return value of “org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData.getLoadManagerClassName()” is null
java.util.concurrent.CompletionException: java.lang.NullPointerException: Cannot invoke “String.equals(Object)” because the return value of “org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData.getLoadManagerClassName()” is null
	at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315) ~[?:?]
	at java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1194) ~[?:?]
	at java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309) ~[?:?]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.selectAsync(ExtensibleLoadManagerImpl.java:385) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.lambda$assign$6(ExtensibleLoadManagerImpl.java:336) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187) ~[?:?]
	at java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309) ~[?:?]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.lambda$assign$10(ExtensibleLoadManagerImpl.java:333) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap$Section.put(ConcurrentOpenHashMap.java:409) ~[io.streamnative-pulsar-common-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap.computeIfAbsent(ConcurrentOpenHashMap.java:243) ~[io.streamnative-pulsar-common-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.assign(ExtensibleLoadManagerImpl.java:327) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper.findBrokerServiceUrl(ExtensibleLoadManagerWrapper.java:66) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.namespace.NamespaceService.lambda$getBrokerServiceUrlAsync$0(NamespaceService.java:191) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
```

### Modifications

* Add null check when using`getLoadManagerClassName`.
* Add test to cover this case.
* Add `RedirectManager` unit test.
---
 .../filter/BrokerLoadManagerClassFilter.java  |   5 +-
 .../extensions/manager/RedirectManager.java   |  17 ++-
 .../impl/BrokerLoadManagerClassFilter.java    |   5 +-
 .../BrokerLoadManagerClassFilterTest.java     |   3 +-
 .../manager/RedirectManagerTest.java          | 111 ++++++++++++++++++
 .../BrokerLoadManagerClassFilterTest.java     |   6 +
 6 files changed, 139 insertions(+), 8 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
index 4ee28a5225a0d..07109b277ae98 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.filter;
 
 import java.util.Map;
+import java.util.Objects;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
@@ -43,7 +44,9 @@ public Map filter(
         }
         brokers.entrySet().removeIf(entry -> {
             BrokerLookupData v = entry.getValue();
-            return !v.getLoadManagerClassName().equals(context.brokerConfiguration().getLoadManagerClassName());
+            // The load manager class name can be null if the cluster has old version of broker.
+            return !Objects.equals(v.getLoadManagerClassName(),
+                    context.brokerConfiguration().getLoadManagerClassName());
         });
         return brokers;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
index 4aff77937a5b4..3455b333b0ae7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
@@ -19,9 +19,11 @@
 package org.apache.pulsar.broker.loadbalance.extensions.manager;
 
 import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT;
+import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -48,6 +50,12 @@ public RedirectManager(PulsarService pulsar) {
         this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class);
     }
 
+    @VisibleForTesting
+    public RedirectManager(PulsarService pulsar, LockManager brokerLookupDataLockManager) {
+        this.pulsar = pulsar;
+        this.brokerLookupDataLockManager = brokerLookupDataLockManager;
+    }
+
     public CompletableFuture> getAvailableBrokerLookupDataAsync() {
         return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenCompose(availableBrokers -> {
             Map map = new ConcurrentHashMap<>();
@@ -69,7 +77,7 @@ public CompletableFuture> getAvailableBrokerLookup
 
     public CompletableFuture> findRedirectLookupResultAsync() {
         String currentLMClassName = pulsar.getConfiguration().getLoadManagerClassName();
-        boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfig(), log);
+        boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log);
         return getAvailableBrokerLookupDataAsync().thenApply(lookupDataMap -> {
             if (lookupDataMap.isEmpty()) {
                 String errorMsg = "No available broker found.";
@@ -89,9 +97,10 @@ public CompletableFuture> findRedirectLookupResultAsync()
                 log.warn(errorMsg);
                 throw new IllegalStateException(errorMsg);
             }
-            if (latestServiceLookupData.get().getLoadManagerClassName().equals(currentLMClassName)) {
+
+            if (Objects.equals(latestServiceLookupData.get().getLoadManagerClassName(), currentLMClassName)) {
                 if (debug) {
-                    log.info("We don't need to redirect, current load manager class name: {}",
+                    log.info("No need to redirect, current load manager class name: {}",
                             currentLMClassName);
                 }
                 return Optional.empty();
@@ -99,7 +108,7 @@ public CompletableFuture> findRedirectLookupResultAsync()
             var serviceLookupDataObj = latestServiceLookupData.get();
             var candidateBrokers = new ArrayList();
             lookupDataMap.forEach((key, value) -> {
-                if (value.getLoadManagerClassName().equals(serviceLookupDataObj.getLoadManagerClassName())) {
+                if (Objects.equals(value.getLoadManagerClassName(), serviceLookupDataObj.getLoadManagerClassName())) {
                     candidateBrokers.add(value);
                 }
             });
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
index 5d6a56ba86960..13e3fdc537e79 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.loadbalance.impl;
 
+import java.util.Objects;
 import java.util.Set;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerFilter;
@@ -32,8 +33,8 @@ public void filter(Set brokers, BundleData bundleToAssign,
                        LoadData loadData,
                        ServiceConfiguration conf) throws BrokerFilterException {
         loadData.getBrokerData().forEach((key, value) -> {
-            if (!value.getLocalData().getLoadManagerClassName()
-                    .equals(conf.getLoadManagerClassName())) {
+            // The load manager class name can be null if the cluster has old version of broker.
+            if (!Objects.equals(value.getLocalData().getLoadManagerClassName(), conf.getLoadManagerClassName())) {
                 brokers.remove(key);
             }
         });
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
index 0169b57fe993e..4aef87cf63aa8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
@@ -44,7 +44,8 @@ public void test() throws BrokerFilterException {
                 "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()),
                 "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()),
                 "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()),
-                "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName())
+                "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()),
+                "broker5", getLookupData("3.0.0", null)
         );
 
         Map result = filter.filter(new HashMap<>(originalBrokers), null, context);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java
new file mode 100644
index 0000000000000..cbf77b59d5ad6
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.manager;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
+import org.apache.pulsar.broker.lookup.LookupResult;
+import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener;
+import org.testng.annotations.Test;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+
+/**
+ * Unit test {@link RedirectManager}.
+ */
+public class RedirectManagerTest {
+
+    @Test
+    public void testFindRedirectLookupResultAsync() throws ExecutionException, InterruptedException {
+        PulsarService pulsar = mock(PulsarService.class);
+        ServiceConfiguration configuration = new ServiceConfiguration();
+        when(pulsar.getConfiguration()).thenReturn(configuration);
+        RedirectManager redirectManager = spy(new RedirectManager(pulsar, null));
+
+        configuration.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
+        configuration.setLoadBalancerDebugModeEnabled(true);
+
+        // Test 1: No load manager class name found.
+        doReturn(CompletableFuture.completedFuture(
+                new HashMap<>(){{
+                    put("broker-1", getLookupData("broker-1", null, 10));
+                    put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 1));
+                }}
+        )).when(redirectManager).getAvailableBrokerLookupDataAsync();
+
+        // Should redirect to broker-1, since broker-1 has the latest load manager, even though the class name is null.
+        Optional lookupResult = redirectManager.findRedirectLookupResultAsync().get();
+        assertTrue(lookupResult.isPresent());
+        assertTrue(lookupResult.get().getLookupData().getBrokerUrl().contains("broker-1"));
+
+        // Test 2: Should redirect to broker-1, since the latest broker are using ExtensibleLoadManagerImpl
+        doReturn(CompletableFuture.completedFuture(
+                new HashMap<>(){{
+                    put("broker-1", getLookupData("broker-1", ExtensibleLoadManagerImpl.class.getName(), 10));
+                    put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 1));
+                }}
+        )).when(redirectManager).getAvailableBrokerLookupDataAsync();
+
+        lookupResult = redirectManager.findRedirectLookupResultAsync().get();
+        assertTrue(lookupResult.isPresent());
+        assertTrue(lookupResult.get().getLookupData().getBrokerUrl().contains("broker-1"));
+
+
+        // Test 3: Should not redirect, since current broker are using ModularLoadManagerImpl
+        doReturn(CompletableFuture.completedFuture(
+                new HashMap<>(){{
+                    put("broker-1", getLookupData("broker-1", ExtensibleLoadManagerImpl.class.getName(), 10));
+                    put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 100));
+                }}
+        )).when(redirectManager).getAvailableBrokerLookupDataAsync();
+
+        lookupResult = redirectManager.findRedirectLookupResultAsync().get();
+        assertFalse(lookupResult.isPresent());
+    }
+
+
+    public BrokerLookupData getLookupData(String broker, String loadManagerClassName, long startTimeStamp) {
+        String webServiceUrl = "http://" + broker + ":8080";
+        String webServiceUrlTls = "https://" + broker + ":8081";
+        String pulsarServiceUrl = "pulsar://" + broker + ":6650";
+        String pulsarServiceUrlTls = "pulsar+ssl://" + broker + ":6651";
+        Map advertisedListeners = new HashMap<>();
+        Map protocols = new HashMap<>(){{
+            put("kafka", "9092");
+        }};
+        return new BrokerLookupData(
+                webServiceUrl, webServiceUrlTls, pulsarServiceUrl,
+                pulsarServiceUrlTls, advertisedListeners, protocols, true, true,
+                loadManagerClassName, startTimeStamp, "3.0.0");
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
index 856bbac029226..56332111f935b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
@@ -46,8 +46,12 @@ public void test() throws BrokerFilterException {
 
         LocalBrokerData localBrokerData1 = new LocalBrokerData();
         localBrokerData1.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+
+        LocalBrokerData localBrokerData2 = new LocalBrokerData();
+        localBrokerData2.setLoadManagerClassName(null);
         loadData.getBrokerData().put("broker1", new BrokerData(localBrokerData));
         loadData.getBrokerData().put("broker2", new BrokerData(localBrokerData1));
+        loadData.getBrokerData().put("broker3", new BrokerData(localBrokerData2));
 
         ServiceConfiguration conf = new ServiceConfiguration();
         conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
@@ -55,6 +59,7 @@ public void test() throws BrokerFilterException {
         Set brokers = new HashSet<>(){{
             add("broker1");
             add("broker2");
+            add("broker3");
         }};
         filter.filter(brokers, null, loadData, conf);
 
@@ -64,6 +69,7 @@ public void test() throws BrokerFilterException {
         brokers = new HashSet<>(){{
             add("broker1");
             add("broker2");
+            add("broker3");
         }};
         conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         filter.filter(brokers, null, loadData, conf);

From 548fd22b32bef8d64b4733954de0e2945d3895b0 Mon Sep 17 00:00:00 2001
From: Paul Gier 
Date: Mon, 22 May 2023 14:31:12 -0500
Subject: [PATCH 223/280] [improve][cli] pulsar-perf: refactor to reduce code
 duplication (#19279)

---
 .../pulsar/testclient/PerfClientUtils.java    |   1 -
 .../testclient/PerformanceBaseArguments.java  |  69 +++++++++++-
 .../testclient/PerformanceConsumer.java       | 105 ++++--------------
 .../testclient/PerformanceProducer.java       |  61 +---------
 .../pulsar/testclient/PerformanceReader.java  |  67 +++--------
 .../PerformanceTopicListArguments.java        |  67 +++++++++++
 .../testclient/PerformanceTransaction.java    |  24 +---
 .../testclient/PerfClientUtilsTest.java       |  32 ++----
 .../PerformanceBaseArgumentsTest.java         |   3 +-
 9 files changed, 186 insertions(+), 243 deletions(-)
 create mode 100644 pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java

diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java
index e40e9610bf42f..f9e5d5ee7e6e1 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java
@@ -119,5 +119,4 @@ public static PulsarAdminBuilder createAdminBuilderFromArguments(PerformanceBase
         return pulsarAdminBuilder;
     }
 
-
 }
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java
index ce402884d5ced..5ae79fb0bf9a4 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java
@@ -20,20 +20,26 @@
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.pulsar.testclient.PerfClientUtils.exit;
+import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+import java.io.File;
 import java.io.FileInputStream;
 import java.util.Properties;
 import lombok.SneakyThrows;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.client.api.ProxyProtocol;
 
-
+/**
+ * PerformanceBaseArguments contains common CLI arguments and parsing logic available to all sub-commands.
+ * Sub-commands should create Argument subclasses and override the `validate` method as necessary.
+ */
 public abstract class PerformanceBaseArguments {
 
-    @Parameter(names = { "-h", "--help" }, description = "Help message", help = true)
+    @Parameter(names = { "-h", "--help" }, description = "Print help message", help = true)
     boolean help;
 
-    @Parameter(names = { "-cf", "--conf-file" }, description = "Configuration file")
+    @Parameter(names = { "-cf", "--conf-file" }, description = "Pulsar configuration file")
     public String confFile;
 
     @Parameter(names = { "-u", "--service-url" }, description = "Pulsar Service URL")
@@ -94,6 +100,9 @@ public abstract class PerformanceBaseArguments {
     @Parameter(names = { "--proxy-protocol" }, description = "Proxy protocol to select type of routing at proxy.")
     ProxyProtocol proxyProtocol = null;
 
+    @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true)
+    public String deprecatedAuthPluginClassName;
+
     public abstract void fillArgumentsFromProperties(Properties prop);
 
     @SneakyThrows
@@ -165,4 +174,58 @@ public void fillArgumentsFromProperties() {
         fillArgumentsFromProperties(prop);
     }
 
+    /**
+     * Validate the CLI arguments.  Default implementation provides validation for the common arguments.
+     * Each subclass should call super.validate() and provide validation code specific to the sub-command.
+     * @throws Exception
+     */
+    public void validate() throws Exception {
+        if (confFile != null && !confFile.isBlank()) {
+            File configFile = new File(confFile);
+            if (!configFile.exists()) {
+                throw new Exception("config file '" + confFile + "', does not exist");
+            }
+            if (configFile.isDirectory()) {
+                throw new Exception("config file '" + confFile + "', is a directory");
+            }
+        }
+    }
+
+    /**
+     * Parse the command line args.
+     * @param cmdName used for the help message
+     * @param args String[] of CLI args
+     * @throws ParameterException If there is a problem parsing the arguments
+     */
+    public void parseCLI(String cmdName, String[] args) {
+        JCommander jc = new JCommander(this);
+        jc.setProgramName(cmdName);
+
+        try {
+            jc.parse(args);
+        } catch (ParameterException e) {
+            System.out.println("error: " + e.getMessage());
+            jc.usage();
+            PerfClientUtils.exit(1);
+        }
+
+        if (help) {
+            jc.usage();
+            PerfClientUtils.exit(0);
+        }
+
+        fillArgumentsFromProperties();
+
+        if (isBlank(authPluginClassName) && !isBlank(deprecatedAuthPluginClassName)) {
+            authPluginClassName = deprecatedAuthPluginClassName;
+        }
+
+        try {
+            validate();
+        } catch (Exception e) {
+            System.out.println("error: " + e.getMessage());
+            PerfClientUtils.exit(1);
+        }
+    }
+
 }
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
index 11a23ca05e0b1..59dabc9302622 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
@@ -18,11 +18,8 @@
  */
 package org.apache.pulsar.testclient;
 
-import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -86,14 +83,7 @@ public class PerformanceConsumer {
     private static final Recorder cumulativeRecorder = new Recorder(MAX_LATENCY, 5);
 
     @Parameters(commandDescription = "Test pulsar consumer performance.")
-    static class Arguments extends PerformanceBaseArguments {
-
-        @Parameter(description = "persistent://prop/ns/my-topic", required = true)
-        public List topic;
-
-        @Parameter(names = { "-t", "--num-topics" }, description = "Number of topics",
-                validateWith = PositiveNumberParameterValidator.class)
-        public int numTopics = 1;
+    static class Arguments extends PerformanceTopicListArguments {
 
         @Parameter(names = { "-n", "--num-consumers" }, description = "Number of consumers (per subscription), only "
                 + "one consumer is allowed when subscriptionType is Exclusive",
@@ -143,9 +133,6 @@ static class Arguments extends PerformanceBaseArguments {
                 description = "Number of messages to consume in total. If <= 0, it will keep consuming")
         public long numMessages = 0;
 
-        @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true)
-        public String deprecatedAuthPluginClassName;
-
         @Parameter(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages")
         private int maxPendingChunkedMessage = 0;
 
@@ -199,79 +186,35 @@ static class Arguments extends PerformanceBaseArguments {
         @Override
         public void fillArgumentsFromProperties(Properties prop) {
         }
-    }
-
-    public static void main(String[] args) throws Exception {
-        final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf consume");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
 
-        if (isBlank(arguments.authPluginClassName) && !isBlank(arguments.deprecatedAuthPluginClassName)) {
-            arguments.authPluginClassName = arguments.deprecatedAuthPluginClassName;
-        }
-
-        for (String arg : arguments.topic) {
-            if (arg.startsWith("-")) {
-                System.out.printf("invalid option: '%s'\nTo use a topic with the name '%s', "
-                        + "please use a fully qualified topic name\n", arg, arg);
-                jc.usage();
-                PerfClientUtils.exit(1);
+        @Override
+        public void validate() throws Exception {
+            super.validate();
+            if (subscriptionType == SubscriptionType.Exclusive && numConsumers > 1) {
+                throw new Exception("Only one consumer is allowed when subscriptionType is Exclusive");
             }
-        }
 
-        if (arguments.topic != null && arguments.topic.size() != arguments.numTopics) {
-            // keep compatibility with the previous version
-            if (arguments.topic.size() == 1) {
-                String prefixTopicName = TopicName.get(arguments.topic.get(0)).toString().trim();
-                List defaultTopics = new ArrayList<>();
-                for (int i = 0; i < arguments.numTopics; i++) {
-                    defaultTopics.add(String.format("%s-%d", prefixTopicName, i));
+            if (subscriptions != null && subscriptions.size() != numSubscriptions) {
+                // keep compatibility with the previous version
+                if (subscriptions.size() == 1) {
+                    if (subscriberName == null) {
+                        subscriberName = subscriptions.get(0);
+                    }
+                    List defaultSubscriptions = new ArrayList<>();
+                    for (int i = 0; i < numSubscriptions; i++) {
+                        defaultSubscriptions.add(String.format("%s-%d", subscriberName, i));
+                    }
+                    subscriptions = defaultSubscriptions;
+                } else {
+                    throw new Exception("The size of subscriptions list should be equal to --num-subscriptions");
                 }
-                arguments.topic = defaultTopics;
-            } else {
-                System.out.println("The size of topics list should be equal to --num-topics");
-                jc.usage();
-                PerfClientUtils.exit(1);
             }
         }
+    }
 
-        if (arguments.subscriptionType == SubscriptionType.Exclusive && arguments.numConsumers > 1) {
-            System.out.println("Only one consumer is allowed when subscriptionType is Exclusive");
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.subscriptions != null && arguments.subscriptions.size() != arguments.numSubscriptions) {
-            // keep compatibility with the previous version
-            if (arguments.subscriptions.size() == 1) {
-                if (arguments.subscriberName == null) {
-                    arguments.subscriberName = arguments.subscriptions.get(0);
-                }
-                List defaultSubscriptions = new ArrayList<>();
-                for (int i = 0; i < arguments.numSubscriptions; i++) {
-                    defaultSubscriptions.add(String.format("%s-%d", arguments.subscriberName, i));
-                }
-                arguments.subscriptions = defaultSubscriptions;
-            } else {
-                System.out.println("The size of subscriptions list should be equal to --num-subscriptions");
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-        arguments.fillArgumentsFromProperties();
+    public static void main(String[] args) throws Exception {
+        final Arguments arguments = new Arguments();
+        arguments.parseCLI("pulsar-perf consume", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
@@ -449,7 +392,7 @@ public static void main(String[] args) throws Exception {
         }
 
         for (int i = 0; i < arguments.numTopics; i++) {
-            final TopicName topicName = TopicName.get(arguments.topic.get(i));
+            final TopicName topicName = TopicName.get(arguments.topics.get(i));
 
             log.info("Adding {} consumers per subscription on topic {}", arguments.numConsumers, topicName);
 
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
index 0c56f1d5c736d..6513f0684b243 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
@@ -24,9 +24,7 @@
 import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES;
 import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES;
 import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS;
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -103,10 +101,7 @@ public class PerformanceProducer {
     private static IMessageFormatter messageFormatter = null;
 
     @Parameters(commandDescription = "Test pulsar producer performance.")
-    static class Arguments extends PerformanceBaseArguments {
-
-        @Parameter(description = "persistent://prop/ns/my-topic", required = true)
-        public List topics;
+    static class Arguments extends PerformanceTopicListArguments {
 
         @Parameter(names = { "-threads", "--num-test-threads" }, description = "Number of test threads",
                 validateWith = PositiveNumberParameterValidator.class)
@@ -118,10 +113,6 @@ static class Arguments extends PerformanceBaseArguments {
         @Parameter(names = { "-s", "--size" }, description = "Message size (bytes)")
         public int msgSize = 1024;
 
-        @Parameter(names = { "-t", "--num-topic" }, description = "Number of topics",
-                validateWith = PositiveNumberParameterValidator.class)
-        public int numTopics = 1;
-
         @Parameter(names = { "-n", "--num-producers" }, description = "Number of producers (per topic)",
                 validateWith = PositiveNumberParameterValidator.class)
         public int numProducers = 1;
@@ -139,9 +130,6 @@ static class Arguments extends PerformanceBaseArguments {
         @Parameter(names = { "-au", "--admin-url" }, description = "Pulsar Admin URL")
         public String adminURL;
 
-        @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true)
-        public String deprecatedAuthPluginClassName;
-
         @Parameter(names = { "-ch",
                 "--chunking" }, description = "Should split the message and publish in chunks if message size is "
                 + "larger than allowed max size")
@@ -272,52 +260,7 @@ public void fillArgumentsFromProperties(Properties prop) {
     public static void main(String[] args) throws Exception {
 
         final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf produce");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (isBlank(arguments.authPluginClassName) && !isBlank(arguments.deprecatedAuthPluginClassName)) {
-            arguments.authPluginClassName = arguments.deprecatedAuthPluginClassName;
-        }
-
-        for (String arg : arguments.topics) {
-            if (arg.startsWith("-")) {
-                System.out.printf("invalid option: '%s'\nTo use a topic with the name '%s', "
-                        + "please use a fully qualified topic name\n", arg, arg);
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-
-        if (arguments.topics != null && arguments.topics.size() != arguments.numTopics) {
-            // keep compatibility with the previous version
-            if (arguments.topics.size() == 1) {
-                String prefixTopicName = arguments.topics.get(0);
-                List defaultTopics = new ArrayList<>();
-                for (int i = 0; i < arguments.numTopics; i++) {
-                    defaultTopics.add(String.format("%s%s%d", prefixTopicName, arguments.separator, i));
-                }
-                arguments.topics = defaultTopics;
-            } else {
-                System.out.println("The size of topics list should be equal to --num-topic");
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-
-        arguments.fillArgumentsFromProperties();
+        arguments.parseCLI("pulsar-perf produce", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java
index 78d8e5f591569..ed5cc37644a31 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java
@@ -18,9 +18,7 @@
  */
 package org.apache.pulsar.testclient;
 
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -62,15 +60,7 @@ public class PerformanceReader {
     private static Recorder cumulativeRecorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5);
 
     @Parameters(commandDescription = "Test pulsar reader performance.")
-    static class Arguments extends PerformanceBaseArguments {
-
-
-        @Parameter(description = "persistent://prop/ns/my-topic", required = true)
-        public List topic;
-
-        @Parameter(names = { "-t", "--num-topics" }, description = "Number of topics",
-                validateWith = PositiveNumberParameterValidator.class)
-        public int numTopics = 1;
+    static class Arguments extends PerformanceTopicListArguments {
 
         @Parameter(names = { "-r", "--rate" }, description = "Simulate a slow message reader (rate in msg/s)")
         public double rate = 0;
@@ -102,51 +92,21 @@ public void fillArgumentsFromProperties(Properties prop) {
                 useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
             }
         }
+        @Override
+        public void validate() throws Exception {
+            super.validate();
+            if (startMessageId != "earliest" && startMessageId != "latest"
+                    && (startMessageId.split(":")).length != 2) {
+                String errMsg = String.format("invalid start message ID '%s', must be either either 'earliest', "
+                        + "'latest' or a specific message id by using 'lid:eid'", startMessageId);
+                throw new Exception(errMsg);
+            }
+        }
     }
 
     public static void main(String[] args) throws Exception {
         final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf read");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        for (String arg : arguments.topic) {
-            if (arg.startsWith("-")) {
-                System.out.printf("invalid option: '%s'\nTo use a topic with the name '%s', "
-                        + "please use a fully qualified topic name\n", arg, arg);
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-
-        if (arguments.topic != null && arguments.topic.size() != arguments.numTopics) {
-            // keep compatibility with the previous version
-            if (arguments.topic.size() == 1) {
-                String prefixTopicName = arguments.topic.get(0);
-                List defaultTopics = new ArrayList<>();
-                for (int i = 0; i < arguments.numTopics; i++) {
-                    defaultTopics.add(String.format("%s-%d", prefixTopicName, i));
-                }
-                arguments.topic = defaultTopics;
-            } else {
-                System.out.println("The size of topics list should be equal to --num-topics");
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-        arguments.fillArgumentsFromProperties();
+        arguments.parseCLI("pulsar-perf read", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
@@ -202,7 +162,7 @@ public static void main(String[] args) throws Exception {
                 .startMessageId(startMessageId);
 
         for (int i = 0; i < arguments.numTopics; i++) {
-            final TopicName topicName = TopicName.get(arguments.topic.get(i));
+            final TopicName topicName = TopicName.get(arguments.topics.get(i));
 
             futures.add(readerBuilder.clone().topic(topicName.toString()).createAsync());
         }
@@ -230,7 +190,6 @@ public void run() {
             timer.schedule(timoutTask, arguments.testTime * 1000);
         }
 
-
         long oldTime = System.nanoTime();
         Histogram reportHistogram = null;
 
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java
new file mode 100644
index 0000000000000..a2f8b6af08282
--- /dev/null
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.testclient;
+
+import com.beust.jcommander.Parameter;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.pulsar.common.naming.TopicName;
+
+/**
+ * PerformanceTopicListArguments provides common topic list arguments which are used
+ * by the consumer, producer, and reader commands, but not by the transaction test command.
+ */
+public abstract class PerformanceTopicListArguments extends PerformanceBaseArguments {
+
+    @Parameter(description = "persistent://prop/ns/my-topic", required = true)
+    public List topics;
+
+    @Parameter(names = { "-t", "--num-topics", "--num-topic" }, description = "Number of topics.  Must match"
+            + "the given number of topic arguments.",
+            validateWith = PositiveNumberParameterValidator.class)
+    public int numTopics = 1;
+
+    @Override
+    public void validate() throws Exception {
+        super.validate();
+        for (String arg : topics) {
+            if (arg.startsWith("-")) {
+                String errMsg = String.format("invalid option: '%s', to use a topic with the name '%s', "
+                        + "please use a fully qualified topic name", arg, arg);
+                throw new Exception(errMsg);
+            }
+        }
+
+        if (topics.size() != numTopics) {
+            // keep compatibility with the previous version
+            if (topics.size() == 1) {
+                String prefixTopicName = TopicName.get(topics.get(0)).toString().trim();
+                List defaultTopics = new ArrayList<>();
+                for (int i = 0; i < numTopics; i++) {
+                    defaultTopics.add(String.format("%s-%d", prefixTopicName, i));
+                }
+                topics = defaultTopics;
+            } else {
+                String errMsg = String.format("the number of topic names (%d) must be equal to --num-topics (%d)",
+                        topics.size(), numTopics);
+                throw new Exception(errMsg);
+            }
+        }
+    }
+}
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java
index a1495a617fb9c..469e6ab1f3fd6 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java
@@ -19,9 +19,7 @@
 package org.apache.pulsar.testclient;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -70,7 +68,6 @@
 
 public class PerformanceTransaction {
 
-
     private static final LongAdder totalNumEndTxnOpFailed = new LongAdder();
     private static final LongAdder totalNumEndTxnOpSuccess = new LongAdder();
     private static final LongAdder numTxnOpSuccess = new LongAdder();
@@ -92,9 +89,8 @@ public class PerformanceTransaction {
     private static final Recorder messageSendRCumulativeRecorder =
             new Recorder(TimeUnit.SECONDS.toMicros(120000), 5);
 
-
     @Parameters(commandDescription = "Test pulsar transaction performance.")
-    static class Arguments  extends PerformanceBaseArguments {
+    static class Arguments extends PerformanceBaseArguments {
 
         @Parameter(names = "--topics-c", description = "All topics that need ack for a transaction", required =
                 true)
@@ -187,26 +183,10 @@ public void fillArgumentsFromProperties(Properties prop) {
     public static void main(String[] args)
             throws IOException, PulsarAdminException, ExecutionException, InterruptedException {
         final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf transaction");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-        arguments.fillArgumentsFromProperties();
+        arguments.parseCLI("pulsar-perf transaction", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
-
         ObjectMapper m = new ObjectMapper();
         ObjectWriter w = m.writerWithDefaultPrettyPrinter();
         log.info("Starting Pulsar perf transaction with config: {}", w.writeValueAsString(arguments));
diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java
index 3a8bba1bccb90..3d734b1f910ea 100644
--- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java
+++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java
@@ -55,11 +55,7 @@ public void close() throws IOException {
     @Test
     public void testClientCreation() throws Exception {
 
-        final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-            @Override
-            public void fillArgumentsFromProperties(Properties prop) {
-            }
-        };
+        final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
         args.tlsHostnameVerificationEnable = true;
         args.authPluginClassName = MyAuth.class.getName();
@@ -99,11 +95,7 @@ public void fillArgumentsFromProperties(Properties prop) {
     @Test
     public void testClientCreationWithProxy() throws Exception {
 
-        final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-            @Override
-            public void fillArgumentsFromProperties(Properties prop) {
-            }
-        };
+        final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
         args.serviceURL = "pulsar+ssl://my-pulsar:6651";
         args.proxyServiceURL = "pulsar+ssl://my-proxy-pulsar:4443";
@@ -126,11 +118,7 @@ public void testClientCreationWithProxyDefinedInConfFile() throws Exception {
                     + "proxyServiceUrl=pulsar+ssl://my-proxy-pulsar:4443\n"
                     + "proxyProtocol=SNI");
 
-            final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-                @Override
-                public void fillArgumentsFromProperties(Properties prop) {
-                }
-            };
+            final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
             args.confFile = testConf.toString();
             args.fillArgumentsFromProperties();
@@ -155,11 +143,7 @@ public void testClientCreationWithEmptyProxyPropertyInConfFile() throws Exceptio
                     + "proxyServiceUrl=\n"
                     + "proxyProtocol=");
 
-            final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-                @Override
-                public void fillArgumentsFromProperties(Properties prop) {
-                }
-            };
+            final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
             args.confFile = testConf.toString();
             args.fillArgumentsFromProperties();
@@ -174,4 +158,10 @@ public void fillArgumentsFromProperties(Properties prop) {
             Files.deleteIfExists(testConf);
         }
     }
-}
\ No newline at end of file
+}
+
+class PerformanceArgumentsTestDefault extends PerformanceBaseArguments {
+    @Override
+    public void fillArgumentsFromProperties(Properties prop) {
+    }
+}
diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java
index 6c60cbd90f8d5..42c93be343074 100644
--- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java
+++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java
@@ -158,5 +158,4 @@ public void fillArgumentsFromProperties(Properties prop) {
             tempConfigFile.delete();
         }
     }
-
-}
\ No newline at end of file
+}

From e008de9466f452dd92997a11621df3e30b024ece Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 23 May 2023 22:11:30 +0300
Subject: [PATCH 224/280] [fix][broker] Change some static fields referencing
 mutable objects to ordinary instance fields in BrokerService (#20372)

---
 .../apache/pulsar/broker/admin/impl/BrokersBase.java |  9 ++++-----
 .../apache/pulsar/broker/service/BrokerService.java  | 12 ++++++------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
index 3328fa9715b99..b367ce7aad955 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
@@ -53,7 +53,6 @@
 import org.apache.pulsar.broker.admin.AdminResource;
 import org.apache.pulsar.broker.loadbalance.LeaderBroker;
 import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.web.RestException;
@@ -254,7 +253,7 @@ public void getAllDynamicConfigurations(@Suspended AsyncResponse asyncResponse)
             @ApiResponse(code = 403, message = "You don't have admin permission to get configuration")})
     public void getDynamicConfigurationName(@Suspended AsyncResponse asyncResponse) {
         validateSuperUserAccessAsync()
-                .thenAccept(__ -> asyncResponse.resume(BrokerService.getDynamicConfiguration()))
+                .thenAccept(__ -> asyncResponse.resume(pulsar().getBrokerService().getDynamicConfiguration()))
                 .exceptionally(ex -> {
                     LOG.error("[{}] Failed to get all dynamic configuration names.", clientAppId(), ex);
                     resumeAsyncResponseExceptionally(asyncResponse, ex);
@@ -287,11 +286,11 @@ public void getRuntimeConfiguration(@Suspended AsyncResponse asyncResponse) {
      */
     private synchronized CompletableFuture persistDynamicConfigurationAsync(
             String configName, String configValue) {
-        if (!BrokerService.validateDynamicConfiguration(configName, configValue)) {
+        if (!pulsar().getBrokerService().validateDynamicConfiguration(configName, configValue)) {
             return FutureUtil
                     .failedFuture(new RestException(Status.PRECONDITION_FAILED, " Invalid dynamic-config value"));
         }
-        if (BrokerService.isDynamicConfiguration(configName)) {
+        if (pulsar().getBrokerService().isDynamicConfiguration(configName)) {
             return dynamicConfigurationResources().setDynamicConfigurationWithCreateAsync(old -> {
                 Map configurationMap = old.orElseGet(Maps::newHashMap);
                 configurationMap.put(configName, configValue);
@@ -512,7 +511,7 @@ private CompletableFuture healthCheckRecursiveReadNext(Reader read
     }
 
     private CompletableFuture internalDeleteDynamicConfigurationOnMetadataAsync(String configName) {
-        if (!BrokerService.isDynamicConfiguration(configName)) {
+        if (!pulsar().getBrokerService().isDynamicConfiguration(configName)) {
             throw new RestException(Status.PRECONDITION_FAILED, " Can't update non-dynamic configuration");
         } else {
             return dynamicConfigurationResources().setDynamicConfigurationAsync(old -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 33e5500d623d2..663d013dc7439 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -214,7 +214,7 @@ public class BrokerService implements Closeable {
     private final OrderedExecutor topicOrderedExecutor;
     // offline topic backlog cache
     private final ConcurrentOpenHashMap offlineTopicStatCache;
-    private static final ConcurrentOpenHashMap dynamicConfigurationMap =
+    private final ConcurrentOpenHashMap dynamicConfigurationMap =
             prepareDynamicConfigurationMap();
     private final ConcurrentOpenHashMap> configRegisteredListeners;
 
@@ -253,10 +253,10 @@ public class BrokerService implements Closeable {
 
     public static final String MANAGED_LEDGER_PATH_ZNODE = "/managed-ledgers";
 
-    private static final LongAdder totalUnackedMessages = new LongAdder();
+    private final LongAdder totalUnackedMessages = new LongAdder();
     private final int maxUnackedMessages;
     public final int maxUnackedMsgsPerDispatcher;
-    private static final AtomicBoolean blockedDispatcherOnHighUnackedMsgs = new AtomicBoolean(false);
+    private final AtomicBoolean blockedDispatcherOnHighUnackedMsgs = new AtomicBoolean(false);
     private final ConcurrentOpenHashSet blockedDispatchers;
     private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
@@ -2938,7 +2938,7 @@ public DelayedDeliveryTrackerFactory getDelayedDeliveryTrackerFactory() {
         return delayedDeliveryTrackerFactory;
     }
 
-    public static List getDynamicConfiguration() {
+    public List getDynamicConfiguration() {
         return dynamicConfigurationMap.keys();
     }
 
@@ -2951,11 +2951,11 @@ public Map getRuntimeConfiguration() {
         return configMap;
     }
 
-    public static boolean isDynamicConfiguration(String key) {
+    public boolean isDynamicConfiguration(String key) {
         return dynamicConfigurationMap.containsKey(key);
     }
 
-    public static boolean validateDynamicConfiguration(String key, String value) {
+    public boolean validateDynamicConfiguration(String key, String value) {
         if (dynamicConfigurationMap.containsKey(key) && dynamicConfigurationMap.get(key).validator != null) {
             return dynamicConfigurationMap.get(key).validator.test(value);
         }

From 120f229ffd4a2ce264b3a95a6731d948e2d88323 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 23 May 2023 14:19:12 -0500
Subject: [PATCH 225/280] [fix] Increase timeout on
 ManagedLedgerCompressionTest flaky test (#20352)

---
 .../pulsar/broker/service/ManagedLedgerCompressionTest.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
index 1ecb0d8f5f7a0..fb60ef97320fb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
@@ -50,7 +50,7 @@ protected void cleanup() throws Exception {
         super.internalCleanup();
     }
 
-    @Test(timeOut = 1000 * 30)
+    @Test(timeOut = 1000 * 60)
     public void testRestartBrokerEnableManagedLedgerInfoCompression() throws Exception {
         String topic = newTopicName();
         @Cleanup

From 2e6928a02037087e3809fe35c55fd738bb229ee7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Wed, 24 May 2023 03:36:10 +0200
Subject: [PATCH 226/280] [improve][client] Swallow Conscrypt
 ClassNotFoundException (#20371)

---
 .../java/org/apache/pulsar/common/util/SecurityUtility.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java
index 12ab9ae0b0bc9..f0023ce5a42dd 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java
@@ -124,7 +124,9 @@ private static Provider loadConscryptProvider() {
             conscryptClazz = Class.forName("org.conscrypt.Conscrypt");
             conscryptClazz.getMethod("checkAvailability").invoke(null);
         } catch (Throwable e) {
-            if (e.getCause() instanceof UnsatisfiedLinkError) {
+            if (e instanceof ClassNotFoundException) {
+                log.warn("Conscrypt isn't available in the classpath. Using JDK default security provider.");
+            } else if (e.getCause() instanceof UnsatisfiedLinkError) {
                 log.warn("Conscrypt isn't available for {} {}. Using JDK default security provider.",
                         System.getProperty("os.name"), System.getProperty("os.arch"));
             } else {

From 1080ad5c787bb317347b3f1f12b78ba3dec49757 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Tue, 23 May 2023 21:26:49 -0700
Subject: [PATCH 227/280] [fix][broker] pre-create non-partitioned system
 topics for load balance extension (#20370)

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation

We need to create system topics without partitions explicitly. Currently, we do not support partitioned system topics.

### Modifications

 create system topics without partitions explicitly
---
 .../extensions/ExtensibleLoadManagerImpl.java   | 17 +++++++++++++++++
 .../channel/ServiceUnitStateChannelImpl.java    |  2 ++
 .../ExtensibleLoadManagerImplTest.java          |  2 ++
 .../channel/ServiceUnitStateChannelTest.java    |  2 ++
 4 files changed, 23 insertions(+)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index cbaed8ee8f94f..531ab18938a1e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -79,6 +79,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
+import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceName;
@@ -213,6 +214,19 @@ public static boolean debug(ServiceConfiguration config, Logger log) {
         return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
     }
 
+    public static void createSystemTopic(PulsarService pulsar, String topic) throws PulsarServerException {
+        try {
+            pulsar.getAdminClient().topics().createNonPartitionedTopic(topic);
+            log.info("Created topic {}.", topic);
+        } catch (PulsarAdminException.ConflictException ex) {
+            if (debug(pulsar.getConfiguration(), log)) {
+                log.info("Topic {} already exists.", topic, ex);
+            }
+        } catch (PulsarAdminException e) {
+            throw new PulsarServerException(e);
+        }
+    }
+
     @Override
     public void start() throws PulsarServerException {
         if (this.started) {
@@ -247,6 +261,9 @@ public void start() throws PulsarServerException {
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies);
         this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper));
 
+        createSystemTopic(pulsar, BROKER_LOAD_DATA_STORE_TOPIC);
+        createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC);
+
         try {
             this.brokerLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 6246c26e57bcc..489a00851057b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -294,6 +294,8 @@ public synchronized void start() throws PulsarServerException {
             PulsarClusterMetadataSetup.createNamespaceIfAbsent
                     (pulsar.getPulsarResources(), SYSTEM_NAMESPACE, config.getClusterName());
 
+            ExtensibleLoadManagerImpl.createSystemTopic(pulsar, TOPIC);
+
             producer = pulsar.getClient().newProducer(schema)
                     .enableBatching(true)
                     .compressionType(MSG_COMPRESSION_TYPE)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index b131aff68d909..498f48d16d2cd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -97,6 +97,7 @@
 import org.apache.pulsar.common.policies.data.BundlesData;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
@@ -129,6 +130,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest {
     @Override
     public void setup() throws Exception {
         conf.setForceDeleteNamespaceAllowed(true);
+        conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED);
         conf.setAllowAutoTopicCreation(true);
         conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index cb26c460f0a03..1263170bec495 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -86,6 +86,7 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.TypedMessageBuilder;
 import org.apache.pulsar.client.impl.TableViewImpl;
+import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.NotificationType;
@@ -129,6 +130,7 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
     @Override
     protected void setup() throws Exception {
         conf.setAllowAutoTopicCreation(true);
+        conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED);
         conf.setLoadBalancerDebugModeEnabled(true);
         conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10);
         super.internalSetup(conf);

From f2ab623095a54968859c1db041d1f24c54dee20a Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 24 May 2023 13:07:07 +0800
Subject: [PATCH 228/280] [fix][broker] Fix memory leak when rebatchMessage
 (#20369)

---
 .../java/org/apache/pulsar/client/impl/RawBatchConverter.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
index 4809ce1a04807..54d2ff867a629 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
@@ -100,7 +100,7 @@ public static Optional rebatchMessage(RawMessage msg,
             payload.skipBytes(Short.BYTES);
             int brokerEntryMetadataSize = payload.readInt();
             payload.readerIndex(readerIndex);
-            brokerMeta = payload.readRetainedSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES);
+            brokerMeta = payload.readSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES);
         }
         MessageMetadata metadata = Commands.parseMessageMetadata(payload);
         ByteBuf batchBuffer = PulsarByteBufAllocator.DEFAULT.buffer(payload.capacity());
@@ -152,7 +152,7 @@ public static Optional rebatchMessage(RawMessage msg,
 
                 if (brokerMeta != null) {
                     CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeDirectBuffer();
-                    compositeByteBuf.addComponents(true, brokerMeta, metadataAndPayload);
+                    compositeByteBuf.addComponents(true, brokerMeta.retain(), metadataAndPayload);
                     metadataAndPayload = compositeByteBuf;
                 }
 

From 0e7056bf91af3c8d10026d2f238b31d56c0c2130 Mon Sep 17 00:00:00 2001
From: GerMoranOverstock <126716444+GerMoranOverstock@users.noreply.github.com>
Date: Wed, 24 May 2023 06:31:36 +0100
Subject: [PATCH 229/280] [improve][doc] Remove javadoc reference to old
 default 30s ack timeout impl (#20377)

---
 .../main/java/org/apache/pulsar/client/api/ConsumerBuilder.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java
index 14a94cb8286dc..870900a48feae 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java
@@ -184,8 +184,6 @@ public interface ConsumerBuilder extends Cloneable {
      * 
    By default, the acknowledgment timeout is disabled (set to `0`, which means infinite).
      * When a consumer with an infinite acknowledgment timeout terminates, any unacknowledged
      * messages that it receives are re-delivered to another consumer.
-     * 
    Since 2.3.0, when a dead letter policy is specified and no ackTimeoutMillis is specified,
-     * the acknowledgment timeout is set to 30 seconds.
      *
      * 
    When enabling acknowledgment timeout, if a message is not acknowledged within the specified timeout,
      * it is re-delivered to the consumer (possibly to a different consumer, in the case of

From 946acc551da55d69a6d26d4696b4954cf89a88d2 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Wed, 24 May 2023 15:06:34 +0800
Subject: [PATCH 230/280] [improve][monitor] Add JVM start time metric (#20381)

---
 .../main/java/org/apache/pulsar/common/stats/JvmMetrics.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java
index 1f15beb8a0b92..8a8da0bb1ac93 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java
@@ -99,6 +99,9 @@ public List generate() {
 
         Runtime r = Runtime.getRuntime();
 
+        RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+
+        m.put("jvm_start_time", runtimeMXBean.getStartTime());
         m.put("jvm_heap_used", r.totalMemory() - r.freeMemory());
         m.put("jvm_max_memory", r.maxMemory());
         m.put("jvm_total_memory", r.totalMemory());

From 05e57dd3a443c5b99c21054c56a1b497455fa867 Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Wed, 24 May 2023 16:41:52 +0800
Subject: [PATCH 231/280] [fix][fn] Fix JavaInstanceStarter inferring type
 class name error (#19896)

---
 .../runtime/JavaInstanceStarter.java          | 19 ++++++++++----
 .../runtime/thread/ThreadRuntime.java         | 19 ++++++++------
 .../runtime/thread/ThreadRuntimeFactory.java  | 26 ++++++++++++++++---
 3 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
index c4f44be3df380..33e837d66e250 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
@@ -49,10 +49,13 @@
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.proto.InstanceControlGrpc;
+import org.apache.pulsar.functions.runtime.thread.ThreadRuntime;
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory;
 import org.apache.pulsar.functions.secretsprovider.ClearTextSecretsProvider;
 import org.apache.pulsar.functions.secretsprovider.SecretsProvider;
 import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager;
+import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManagerImpl;
 
 
 @Slf4j
@@ -192,7 +195,10 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL
             functionDetailsJsonString = functionDetailsJsonString.substring(0, functionDetailsJsonString.length() - 1);
         }
         JsonFormat.parser().merge(functionDetailsJsonString, functionDetailsBuilder);
-        inferringMissingTypeClassName(functionDetailsBuilder, functionInstanceClassLoader);
+        FunctionCacheManager fnCache = new FunctionCacheManagerImpl(rootClassLoader);
+        ClassLoader functionClassLoader = ThreadRuntime.loadJars(jarFile, instanceConfig, functionId,
+                functionDetailsBuilder.getName(), narExtractionDirectory, fnCache);
+        inferringMissingTypeClassName(functionDetailsBuilder, functionClassLoader);
         Function.FunctionDetails functionDetails = functionDetailsBuilder.build();
         instanceConfig.setFunctionDetails(functionDetails);
         instanceConfig.setPort(port);
@@ -237,7 +243,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL
                         .tlsHostnameVerificationEnable(isTrue(tlsHostNameVerificationEnabled))
                         .tlsTrustCertsFilePath(tlsTrustCertFilePath).build(),
                 secretsProvider, collectorRegistry, narExtractionDirectory, rootClassLoader,
-                exposePulsarAdminClientEnabled, webServiceUrl);
+                exposePulsarAdminClientEnabled, webServiceUrl, fnCache);
         runtimeSpawner = new RuntimeSpawner(
                 instanceConfig,
                 jarFile,
@@ -329,7 +335,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func
                     Map userConfigs = new Gson().fromJson(functionDetailsBuilder.getUserConfig(),
                             new TypeToken>() {
                             }.getType());
-                    boolean isWindowConfigPresent = userConfigs.containsKey(WindowConfig.WINDOW_CONFIG_KEY);
+                    boolean isWindowConfigPresent =
+                            userConfigs != null && userConfigs.containsKey(WindowConfig.WINDOW_CONFIG_KEY);
                     String className = functionDetailsBuilder.getClassName();
                     if (isWindowConfigPresent) {
                         WindowConfig windowConfig = new Gson().fromJson(
@@ -360,7 +367,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func
             case SINK:
                 if ((functionDetailsBuilder.hasSink()
                         && functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) {
-                    String typeArg = getSinkType(functionDetailsBuilder.getClassName(), classLoader).getName();
+                    String typeArg =
+                            getSinkType(functionDetailsBuilder.getSink().getClassName(), classLoader).getName();
 
                     Function.SinkSpec.Builder sinkBuilder =
                             Function.SinkSpec.newBuilder(functionDetailsBuilder.getSink());
@@ -378,7 +386,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func
             case SOURCE:
                 if ((functionDetailsBuilder.hasSource()
                         && functionDetailsBuilder.getSource().getTypeClassName().isEmpty())) {
-                    String typeArg = getSourceType(functionDetailsBuilder.getClassName(), classLoader).getName();
+                    String typeArg =
+                            getSourceType(functionDetailsBuilder.getSource().getClassName(), classLoader).getName();
 
                     Function.SourceSpec.Builder sourceBuilder =
                             Function.SourceSpec.newBuilder(functionDetailsBuilder.getSource());
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
index 0aa0cd95aef09..ed128568bcf50 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
@@ -137,14 +137,16 @@ private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig,
                         .getClassLoader();
             }
         }
-        return loadJars(jarFile, instanceConfig, functionId, narExtractionDirectory, fnCache);
+        return loadJars(jarFile, instanceConfig, functionId, instanceConfig.getFunctionDetails().getName(),
+                narExtractionDirectory, fnCache);
     }
 
-    private static ClassLoader loadJars(String jarFile,
-                                 InstanceConfig instanceConfig,
-                                 String functionId,
-                                 String narExtractionDirectory,
-                                 FunctionCacheManager fnCache) throws Exception {
+    public static ClassLoader loadJars(String jarFile,
+                                       InstanceConfig instanceConfig,
+                                       String functionId,
+                                       String functionName,
+                                       String narExtractionDirectory,
+                                       FunctionCacheManager fnCache) throws Exception {
         if (jarFile == null) {
             return Thread.currentThread().getContextClassLoader();
         }
@@ -175,8 +177,9 @@ private static ClassLoader loadJars(String jarFile,
                     Collections.emptyList());
         }
 
-        log.info("Initialize function class loader for function {} at function cache manager, functionClassLoader: {}",
-                instanceConfig.getFunctionDetails().getName(), fnCache.getClassLoader(functionId));
+        log.info(
+                "Initialize function class loader for function {} at function cache manager, functionClassLoader: {}",
+                functionName, fnCache.getClassLoader(functionId));
 
         fnClassLoader = fnCache.getClassLoader(functionId);
         if (null == fnClassLoader) {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
index 7bc055b25d6b9..cb9ad27a2dff8 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
@@ -86,7 +86,21 @@ public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl,
                 stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
                 narExtractionDirectory,
                 rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
-                Optional.empty());
+                Optional.empty(), null);
+    }
+
+    public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl,
+                                String stateStorageImplClass,
+                                String storageServiceUrl,
+                                AuthenticationConfig authConfig, SecretsProvider secretsProvider,
+                                FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory,
+                                ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled,
+                                String pulsarWebServiceUrl, FunctionCacheManager fnCache) throws Exception {
+        initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig,
+                stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
+                narExtractionDirectory,
+                rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
+                Optional.empty(), fnCache);
     }
 
     private void initialize(String threadGroupName, Optional memoryLimit,
@@ -96,7 +110,7 @@ private void initialize(String threadGroupName, Optional connectorsManager,
-                            Optional functionsManager)
+                            Optional functionsManager, FunctionCacheManager fnCache)
             throws PulsarClientException {
 
         if (rootClassLoader == null) {
@@ -106,7 +120,10 @@ private void initialize(String threadGroupName, Optional
Date: Wed, 24 May 2023 12:20:42 +0200
Subject: [PATCH 232/280] [improve][bk] Add integration test with bookie http
 server enabled (#20149)

Signed-off-by: tison 
Co-authored-by: tison 
---
 ...eeperInstallWithHttpServerEnabledTest.java | 84 +++++++++++++++++++
 .../integration/topologies/PulsarCluster.java | 36 +++++---
 .../topologies/PulsarClusterSpec.java         | 10 +++
 .../topologies/PulsarClusterTestBase.java     |  2 +
 4 files changed, 120 insertions(+), 12 deletions(-)
 create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java

diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java
new file mode 100644
index 0000000000000..03d7f974ab39b
--- /dev/null
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests.integration.bookkeeper;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.tests.integration.docker.ContainerExecResult;
+import org.apache.pulsar.tests.integration.topologies.PulsarCluster;
+import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec;
+import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.joining;
+import static org.testng.Assert.assertEquals;
+
+/**
+ * Test bookkeeper setup with http server enabled.
+ */
+@Slf4j
+public class BookkeeperInstallWithHttpServerEnabledTest extends PulsarClusterTestBase {
+
+    @BeforeClass(alwaysRun = true)
+    @Override
+    public final void setupCluster() throws Exception {
+        incrementSetupNumber();
+
+        final String clusterName = Stream.of(this.getClass().getSimpleName(), randomName(5))
+                .filter(s -> !s.isEmpty())
+                .collect(joining("-"));
+        bookkeeperEnvs.put("httpServerEnabled", "true");
+        bookieAdditionalPorts.add(8000);
+        PulsarClusterSpec spec = PulsarClusterSpec.builder()
+                .numBookies(2)
+                .numBrokers(1)
+                .bookkeeperEnvs(bookkeeperEnvs)
+                .bookieAdditionalPorts(bookieAdditionalPorts)
+                .clusterName(clusterName)
+                .build();
+
+        log.info("Setting up cluster {} with {} bookies, {} brokers",
+                spec.clusterName(), spec.numBookies(), spec.numBrokers());
+
+        pulsarCluster = PulsarCluster.forSpec(spec);
+        pulsarCluster.start();
+
+        log.info("Cluster {} is setup", spec.clusterName());
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    public final void tearDownCluster() throws Exception {
+        super.tearDownCluster();
+    }
+
+    @Test
+    public void testBookieHttpServerIsRunning() throws Exception {
+        ContainerExecResult result = pulsarCluster.getAnyBookie().execCmd(
+                PulsarCluster.CURL,
+                "-X",
+                "GET",
+                "http://localhost:8000/heartbeat");
+        assertEquals(result.getExitCode(), 0);
+        assertEquals(result.getStdout(), "OK\n");
+    }
+}
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
index fcc0feec6d44f..bd11b7d387383 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
@@ -157,18 +157,26 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s
 
         // create bookies
         bookieContainers.putAll(
-                runNumContainers("bookie", spec.numBookies(), (name) -> new BKContainer(clusterName, name)
-                        .withNetwork(network)
-                        .withNetworkAliases(appendClusterName(name))
-                        .withEnv("zkServers", appendClusterName(ZKContainer.NAME))
-                        .withEnv("useHostNameAsBookieID", "true")
-                        // Disable fsyncs for tests since they're slow within the containers
-                        .withEnv("journalSyncData", "false")
-                        .withEnv("journalMaxGroupWaitMSec", "0")
-                        .withEnv("clusterName", clusterName)
-                        .withEnv("diskUsageThreshold", "0.99")
-                        .withEnv("nettyMaxFrameSizeBytes", "" + spec.maxMessageSize)
-                )
+                runNumContainers("bookie", spec.numBookies(), (name) -> {
+                    BKContainer bookieContainer = new BKContainer(clusterName, name)
+                            .withNetwork(network)
+                            .withNetworkAliases(appendClusterName(name))
+                            .withEnv("zkServers", appendClusterName(ZKContainer.NAME))
+                            .withEnv("useHostNameAsBookieID", "true")
+                            // Disable fsyncs for tests since they're slow within the containers
+                            .withEnv("journalSyncData", "false")
+                            .withEnv("journalMaxGroupWaitMSec", "0")
+                            .withEnv("clusterName", clusterName)
+                            .withEnv("diskUsageThreshold", "0.99")
+                            .withEnv("nettyMaxFrameSizeBytes", String.valueOf(spec.maxMessageSize));
+                    if (spec.bookkeeperEnvs != null) {
+                        bookieContainer.withEnv(spec.bookkeeperEnvs);
+                    }
+                    if (spec.bookieAdditionalPorts != null) {
+                        spec.bookieAdditionalPorts.forEach(bookieContainer::addExposedPort);
+                    }
+                    return bookieContainer;
+                })
         );
 
         // create brokers
@@ -740,4 +748,8 @@ public void dumpFunctionLogs(String name) {
     private String appendClusterName(String name) {
         return sharedCsContainer ? clusterName + "-" + name : name;
     }
+
+    public BKContainer getAnyBookie() {
+        return getAnyContainer(bookieContainers, "bookie");
+    }
 }
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java
index 385af99a6644b..fa28d20e6b356 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java
@@ -150,6 +150,11 @@ public class PulsarClusterSpec {
      */
     Map brokerEnvs;
 
+    /**
+     * Specify envs for bookkeeper.
+     */
+    Map bookkeeperEnvs;
+
     /**
      * Specify mount files.
      */
@@ -167,4 +172,9 @@ public class PulsarClusterSpec {
      * Additional ports to expose on broker containers.
      */
     List brokerAdditionalPorts;
+
+    /**
+     * Additional ports to expose on bookie containers.
+     */
+    List bookieAdditionalPorts;
 }
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java
index d7a1906ec582e..ae9e44fa98254 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java
@@ -34,8 +34,10 @@
 @Slf4j
 public abstract class PulsarClusterTestBase extends PulsarTestBase {
     protected final Map brokerEnvs = new HashMap<>();
+    protected final Map bookkeeperEnvs = new HashMap<>();
     protected final Map proxyEnvs = new HashMap<>();
     protected final List brokerAdditionalPorts = new LinkedList<>();
+    protected final List bookieAdditionalPorts = new LinkedList<>();
 
     @Override
     protected final void setup() throws Exception {

From 7dcb3eab2916d42bb58fc1639dc58e11b5230997 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= 
Date: Wed, 24 May 2023 21:56:30 +0800
Subject: [PATCH 233/280] [fix][broker] Invalidate metadata children cache
 after key deleted (#20363)

---
 .../org/apache/pulsar/metadata/impl/AbstractMetadataStore.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
index 072d513cca962..6fcf8eb6b49b2 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
@@ -328,6 +328,7 @@ public void accept(Notification n) {
 
         if (type == NotificationType.Created || type == NotificationType.Deleted) {
             existsCache.synchronous().invalidate(path);
+            childrenCache.synchronous().invalidate(path);
             String parent = parent(path);
             if (parent != null) {
                 childrenCache.synchronous().invalidate(parent);
@@ -385,6 +386,7 @@ private CompletableFuture deleteInternal(String path, Optional expec
         // Ensure caches are invalidated before the operation is confirmed
         return storeDelete(path, expectedVersion).thenRun(() -> {
             existsCache.synchronous().invalidate(path);
+            childrenCache.synchronous().invalidate(path);
             String parent = parent(path);
             if (parent != null) {
                 childrenCache.synchronous().invalidate(parent);

From fd36fc1f0d9c12201e5375982ced9ee4b61af944 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 24 May 2023 17:29:05 +0300
Subject: [PATCH 234/280] [improve][ci] Split Pulsar IO unit test job to
 multiple jobs (#20384)

---
 .github/workflows/pulsar-ci.yaml |  4 ++++
 build/run_unit_group.sh          | 12 ++++++++++++
 pom.xml                          | 14 ++++++++++++++
 pulsar-io/pom.xml                | 27 +++++++++++++++++++++------
 4 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index b92599581cd83..2c4abdb704178 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -198,6 +198,10 @@ jobs:
           - name: Pulsar IO
             group: PULSAR_IO
             timeout: 75
+          - name: Pulsar IO - Elastic Search
+            group: PULSAR_IO_ELASTIC
+          - name: Pulsar IO - Kafka Connect Adaptor
+            group: PULSAR_IO_KAFKA_CONNECT
           - name: Pulsar Client
             group: CLIENT
 
diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index ba49820ed1d33..69434b011b37e 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -188,6 +188,18 @@ function test_group_pulsar_io() {
     echo "::endgroup::"
 }
 
+function test_group_pulsar_io_elastic() {
+    echo "::group::Running elastic-search tests"
+    mvn_test --install -Ppulsar-io-elastic-tests,-main
+    echo "::endgroup::"
+}
+
+function test_group_pulsar_io_kafka_connect() {
+    echo "::group::Running Pulsar IO Kafka connect adaptor tests"
+    mvn_test --install -Ppulsar-io-kafka-connect-tests,-main
+    echo "::endgroup::"
+}
+
 function list_test_groups() {
   declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]'
 }
diff --git a/pom.xml b/pom.xml
index 75c25e5478893..37349bbffadba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2438,6 +2438,20 @@ flexible messaging model and an intuitive client API.
       
     
 
+    
+      pulsar-io-elastic-tests
+      
+        pulsar-io
+      
+    
+
+    
+      pulsar-io-kafka-connect-tests
+      
+        pulsar-io
+      
+    
+
     
       pulsar-sql-tests
       
diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml
index 53079cdfbc807..c1a58d059cd48 100644
--- a/pulsar-io/pom.xml
+++ b/pulsar-io/pom.xml
@@ -85,22 +85,16 @@
         batch-discovery-triggerers
         batch-data-generator
         common
-        docs
         aws
         twitter
         cassandra
         aerospike
         http
-        kafka
         rabbitmq
         kinesis
         hdfs3
         jdbc
         data-generator
-        elastic-search
-        kafka-connect-adaptor
-        kafka-connect-adaptor-nar
-        debezium
         hdfs2
         canal
         file
@@ -117,6 +111,27 @@
       
     
 
+    
+      pulsar-io-elastic-tests
+      
+        core
+        common
+        elastic-search
+      
+    
+
+    
+      pulsar-io-kafka-connect-tests
+      
+        core
+        common
+        kafka
+        kafka-connect-adaptor
+        kafka-connect-adaptor-nar
+        debezium
+      
+    
+
     
       core-modules
       

From aff4a1c7edbf019d17795c7a67b1d0f0e701d118 Mon Sep 17 00:00:00 2001
From: vineeth1995 
Date: Wed, 24 May 2023 08:05:58 -0700
Subject: [PATCH 235/280] =?UTF-8?q?[fix][broker]=20Fix=20flaky=20test=20-?=
 =?UTF-8?q?=20testClusterMigrationWithReplica=E2=80=A6=20(#20379)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../apache/pulsar/broker/service/ClusterMigrationTest.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
index df4f66c43d2b4..469e155d409b3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
@@ -458,8 +458,8 @@ public void testClusterMigrationWithReplicationBacklog(boolean persistent, Subsc
         retryStrategically((test) -> !topic1.isReplicationBacklogExist(), 10, 1000);
         assertFalse(topic1.isReplicationBacklogExist());
 
-        // verify that the producer1 is now is now connected to migrated cluster "r2" since backlog is cleared.
-        retryStrategically((test) -> topic2.getProducers().size()==2, 10, 500);
+        producer1.send("test".getBytes());
+        // verify that the producer1 is now connected to migrated cluster "r2" since backlog is cleared.
         assertEquals(topic2.getProducers().size(), 2);
     }
 

From 903425be3afbbcfeb0a7213b0b7d04afe5868151 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 24 May 2023 19:35:15 +0300
Subject: [PATCH 236/280] [fix][test] Fix flaky test
 NonPersistentTopicTest.testMsgDropStat (#20387)

---
 .../client/api/NonPersistentTopicTest.java    | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index 8527406496448..c41ab3e8ccc73 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -64,6 +64,7 @@
 import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.apache.pulsar.zookeeper.ZookeeperServerTest;
+import org.awaitility.Awaitility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
@@ -824,10 +825,12 @@ public void testMsgDropStat() throws Exception {
             stopBroker();
             startBroker();
             Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1")
-                    .receiverQueueSize(1).subscribe();
+                    .receiverQueueSize(1)
+                    .messageListener((c, msg) -> {}).subscribe();
 
             Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2")
-                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
+                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared)
+                    .messageListener((c, msg) -> {}).subscribe();
 
             ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName)
                 .enableBatching(false)
@@ -848,15 +851,19 @@ public void testMsgDropStat() throws Exception {
             }
             latch.await();
 
-            NonPersistentTopic topic = (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
-            pulsar.getBrokerService().updateRates();
-            NonPersistentTopicStats stats = topic.getStats(false, false, false);
-            NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
-            NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1");
-            NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2");
-            assertTrue(npStats.getMsgDropRate() > 0);
-            assertTrue(sub1Stats.getMsgDropRate() > 0);
-            assertTrue(sub2Stats.getMsgDropRate() > 0);
+            NonPersistentTopic topic =
+                    (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
+
+            Awaitility.await().untilAsserted(() -> {
+                pulsar.getBrokerService().updateRates();
+                NonPersistentTopicStats stats = topic.getStats(false, false, false);
+                NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
+                NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1");
+                NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2");
+                assertTrue(npStats.getMsgDropRate() > 0);
+                assertTrue(sub1Stats.getMsgDropRate() > 0);
+                assertTrue(sub2Stats.getMsgDropRate() > 0);
+            });
 
             producer.close();
             consumer.close();

From 545abfc428eab8dfb93314e1051c47d0f026fadf Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Thu, 25 May 2023 00:36:39 +0800
Subject: [PATCH 237/280] [improve][doc] Improve doc for delayed message
 (#20374)

---
 conf/broker.conf                              |  8 +++-----
 conf/standalone.conf                          |  6 ++----
 .../pulsar/broker/ServiceConfiguration.java   | 19 +++++++++----------
 .../data/stats/SubscriptionStatsImpl.java     |  2 +-
 .../policies/data/stats/TopicStatsImpl.java   |  2 +-
 5 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 1183049bf8531..ca118d254f496 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -550,13 +550,11 @@ delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemory
 
 # Control the tick time for when retrying on delayed delivery,
 # affecting the accuracy of the delivery time compared to the scheduled time.
-# Note that this time is used to configure the HashedWheelTimer's tick time for the
-# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory).
+# Note that this time is used to configure the HashedWheelTimer's tick time.
 # Default is 1 second.
 delayedDeliveryTickTimeMillis=1000
 
-# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether
-# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
+# Whether the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
 # time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index
 # for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time
 # has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the
@@ -582,7 +580,7 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
 delayedDeliveryMaxNumBuckets=-1
 
 # Size of the lookahead window to use when detecting if all the messages in the topic
-# have a fixed delay.
+# have a fixed delay for InMemoryDelayedDeliveryTracker (the default DelayedDeliverTracker).
 # Default is 50,000. Setting the lookahead window to 0 will disable the logic to handle
 # fixed delays in messages in a different way.
 delayedDeliveryFixedDelayDetectionLookahead=50000
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 19521bb74696b..8d24d5ad88c7f 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -1237,13 +1237,11 @@ delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemory
 
 # Control the tick time for when retrying on delayed delivery,
 # affecting the accuracy of the delivery time compared to the scheduled time.
-# Note that this time is used to configure the HashedWheelTimer's tick time for the
-# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory).
+# Note that this time is used to configure the HashedWheelTimer's tick time.
 # Default is 1 second.
 delayedDeliveryTickTimeMillis=1000
 
-# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether
-# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
+# Whether the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
 # time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index
 # for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time
 # has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 9966912bc8eae..6fee0e7cd09d2 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -343,17 +343,15 @@ public class ServiceConfiguration implements PulsarConfiguration {
 
     @FieldContext(category = CATEGORY_SERVER, doc = "Control the tick time for when retrying on delayed delivery, "
             + "affecting the accuracy of the delivery time compared to the scheduled time. Default is 1 second. "
-            + "Note that this time is used to configure the HashedWheelTimer's tick time for the "
-            + "InMemoryDelayedDeliveryTrackerFactory.")
+            + "Note that this time is used to configure the HashedWheelTimer's tick time.")
     private long delayedDeliveryTickTimeMillis = 1000;
 
-    @FieldContext(category = CATEGORY_SERVER, doc = "When using the InMemoryDelayedDeliveryTrackerFactory (the default "
-            + "DelayedDeliverTrackerFactory), whether the deliverAt time is strictly followed. When false (default), "
-            + "messages may be sent to consumers before the deliverAt time by as much as the tickTimeMillis. This can "
-            + "reduce the overhead on the broker of maintaining the delayed index for a potentially very short time "
-            + "period. When true, messages will not be sent to consumer until the deliverAt time has passed, and they "
-            + "may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the "
-            + "delayedDeliveryTickTimeMillis.")
+    @FieldContext(category = CATEGORY_SERVER, doc = "Whether the deliverAt time is strictly followed. "
+            + "When false (default), messages may be sent to consumers before the deliverAt time by as much "
+            + "as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index "
+            + "for a potentially very short time period. When true, messages will not be sent to consumer until the "
+            + "deliverAt time has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for "
+            + "the topic plus the delayedDeliveryTickTimeMillis.")
     private boolean isDelayedDeliveryDeliverAtTimeStrict = false;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
@@ -379,7 +377,8 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     private int delayedDeliveryMaxNumBuckets = -1;
 
     @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use "
-            + "when detecting if all the messages in the topic have a fixed delay. "
+            + "when detecting if all the messages in the topic have a fixed delay for "
+            + "InMemoryDelayedDeliveryTracker (the default DelayedDeliverTracker). "
             + "Default is 50,000. Setting the lookahead window to 0 will disable the "
             + "logic to handle fixed delays in messages in a different way.")
     private long delayedDeliveryFixedDelayDetectionLookahead = 50_000;
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
index ea7639a8cd2c6..2bfa54da6a002 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
@@ -131,7 +131,7 @@ public class SubscriptionStatsImpl implements SubscriptionStats {
     /** The serialized size of non-contiguous deleted messages ranges. */
     public int nonContiguousDeletedMessagesRangesSerializedSize;
 
-    /** The size of InMemoryDelayedDeliveryTracer memory usage. */
+    /** The size of DelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
     public Map bucketDelayedIndexStats;
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index c9c4739b904f6..f0141bd58d38f 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -136,7 +136,7 @@ public class TopicStatsImpl implements TopicStats {
     /** The serialized size of non-contiguous deleted messages ranges. */
     public int nonContiguousDeletedMessagesRangesSerializedSize;
 
-    /** The size of InMemoryDelayedDeliveryTracer memory usage. */
+    /** The size of DelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
     /** Map of bucket delayed index statistics. */

From 9918bced4465e0b0746a7959550c90cb76ae945f Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 25 May 2023 03:46:31 -0500
Subject: [PATCH 238/280] [fix][broker] partitioned __change_events topic is
 policy topic (#20392)

---
 .../service/persistent/PersistentTopic.java   |  2 +-
 .../common/naming/SystemTopicNames.java       |  6 +--
 .../common/naming/SystemTopicNamesTest.java   | 47 +++++++++++++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)
 create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 15854f55c5cd1..98e51a2e3ed6e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -819,7 +819,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St
             }
 
             try {
-                if (!topic.endsWith(SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME)
+                if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic)
                         && !checkSubscriptionTypesEnable(subType)) {
                     return FutureUtil.failedFuture(
                             new NotAllowedException("Topic[{" + topic + "}] doesn't support "
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java
index 8fc7d014b5784..716d9bc31facb 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java
@@ -81,7 +81,7 @@ public static boolean isTopicPoliciesSystemTopic(String topic) {
         if (topic == null) {
             return false;
         }
-        return TopicName.get(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME);
+        return TopicName.getPartitionedTopicName(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME);
     }
 
     public static boolean isTransactionInternalName(TopicName topicName) {
@@ -92,7 +92,7 @@ public static boolean isTransactionInternalName(TopicName topicName) {
     }
 
     public static boolean isSystemTopic(TopicName topicName) {
-        TopicName nonePartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName());
-        return isEventSystemTopic(nonePartitionedTopicName) || isTransactionInternalName(nonePartitionedTopicName);
+        TopicName nonPartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName());
+        return isEventSystemTopic(nonPartitionedTopicName) || isTransactionInternalName(nonPartitionedTopicName);
     }
 }
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java
new file mode 100644
index 0000000000000..92d93021973b1
--- /dev/null
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.naming;
+
+import static org.testng.AssertJUnit.assertEquals;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test
+public class SystemTopicNamesTest {
+
+    @DataProvider(name = "topicPoliciesSystemTopicNames")
+    public static Object[][] topicPoliciesSystemTopicNames() {
+        return new Object[][] {
+                {"persistent://public/default/__change_events", true},
+                {"persistent://public/default/__change_events-partition-0", true},
+                {"persistent://random-tenant/random-ns/__change_events", true},
+                {"persistent://random-tenant/random-ns/__change_events-partition-1", true},
+                {"persistent://public/default/not_really__change_events", false},
+                {"persistent://public/default/__change_events-diff-suffix", false},
+                {"persistent://a/b/not_really__change_events", false},
+        };
+    }
+
+    @Test(dataProvider = "topicPoliciesSystemTopicNames")
+    public void testIsTopicPoliciesSystemTopic(String topicName, boolean expectedResult) {
+        assertEquals(expectedResult, SystemTopicNames.isTopicPoliciesSystemTopic(topicName));
+        assertEquals(expectedResult, SystemTopicNames.isSystemTopic(TopicName.get(topicName)));
+        assertEquals(expectedResult, SystemTopicNames.isEventSystemTopic(TopicName.get(topicName)));
+    }
+}

From 25c4b7cee402a1d486d720a71dc2e06aa6d9af64 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 25 May 2023 19:28:18 +0300
Subject: [PATCH 239/280] [fix][test] Fix flaky test
 NonPersistentTopicTest.testMsgDropStat, follow up (#20401)

Fixes #20386

### Motivation

- the previous attempt #20387 to fix the flakiness wasn't effective

### Modifications

- improve the fix and rely on the fact that entryId is -1 when the message is dropped in the broker
  code: https://github.com/apache/pulsar/blob/091ee2504ffbe6ec98e354b76e7f4c045e1914aa/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java#L1699-L1711

### Documentation



- [ ] `doc` 
- [ ] `doc-required` 
- [x] `doc-not-needed` 
- [ ] `doc-complete` 
---
 .../client/api/NonPersistentTopicTest.java    | 30 +++++++++++++------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index c41ab3e8ccc73..63ce0f00dff15 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -38,6 +38,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import lombok.Cleanup;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -50,6 +51,7 @@
 import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.impl.ConsumerImpl;
+import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl;
 import org.apache.pulsar.client.impl.PartitionedProducerImpl;
 import org.apache.pulsar.client.impl.ProducerImpl;
@@ -824,13 +826,14 @@ public void testMsgDropStat() throws Exception {
             conf.setMaxConcurrentNonPersistentMessagePerConnection(1);
             stopBroker();
             startBroker();
+
+            pulsar.getBrokerService().updateRates();
+
             Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1")
-                    .receiverQueueSize(1)
-                    .messageListener((c, msg) -> {}).subscribe();
+                    .receiverQueueSize(1).subscribe();
 
             Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2")
-                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared)
-                    .messageListener((c, msg) -> {}).subscribe();
+                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
 
             ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName)
                 .enableBatching(false)
@@ -839,17 +842,26 @@ public void testMsgDropStat() throws Exception {
             @Cleanup("shutdownNow")
             ExecutorService executor = Executors.newFixedThreadPool(5);
             byte[] msgData = "testData".getBytes();
-            final int totalProduceMessages = 200;
-            CountDownLatch latch = new CountDownLatch(totalProduceMessages);
+            final int totalProduceMessages = 1000;
+            CountDownLatch latch = new CountDownLatch(1);
+            AtomicInteger messagesSent = new AtomicInteger(0);
             for (int i = 0; i < totalProduceMessages; i++) {
                 executor.submit(() -> {
-                    producer.sendAsync(msgData).handle((msg, e) -> {
-                        latch.countDown();
+                    producer.sendAsync(msgData).handle((msgId, e) -> {
+                        int count = messagesSent.incrementAndGet();
+                        // process at least 20% of messages before signalling the latch
+                        // a non-persistent message will return entryId as -1 when it has been dropped
+                        // due to setMaxConcurrentNonPersistentMessagePerConnection limit
+                        // also ensure that it has happened before the latch is signalled
+                        if (count > totalProduceMessages * 0.2 && msgId != null
+                                && ((MessageIdImpl) msgId).getEntryId() == -1) {
+                            latch.countDown();
+                        }
                         return null;
                     });
                 });
             }
-            latch.await();
+            latch.await(5, TimeUnit.SECONDS);
 
             NonPersistentTopic topic =
                     (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();

From f0e97f45e082854015dddea965ab9fc7b4d07e9b Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 25 May 2023 19:29:31 +0300
Subject: [PATCH 240/280] [improve][broker] Use immutable base cursor
 properties (#20400)

---
 .../service/persistent/PersistentSubscription.java       | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index 09dabcd4bfc59..4e74857cef940 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -120,18 +120,15 @@ public class PersistentSubscription extends AbstractSubscription implements Subs
     // Map of properties that is used to mark this subscription as "replicated".
     // Since this is the only field at this point, we can just keep a static
     // instance of the map.
-    private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = new TreeMap<>();
-    private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Collections.emptyMap();
+    private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES =
+            Map.of(REPLICATED_SUBSCRIPTION_PROPERTY, 1L);
+    private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Map.of();
 
     private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache;
     private final PendingAckHandle pendingAckHandle;
     private volatile Map subscriptionProperties;
     private volatile CompletableFuture fenceFuture;
 
-    static {
-        REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L);
-    }
-
     static Map getBaseCursorProperties(boolean isReplicated) {
         return isReplicated ? REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES : NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES;
     }

From 795eb51762b417ea72bf201cdfe5c7585a8c3320 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 26 May 2023 18:25:52 +0300
Subject: [PATCH 241/280] [fix][ci] Update nar maven plugin version to fix
 excessive downloads (#20410)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 37349bbffadba..0fb86ad41a71d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -284,7 +284,7 @@ flexible messaging model and an intuitive client API.
     3.4.1
     3.1.0
     1.1.0
-    1.3.4
+    1.5.0
     3.1.2
     4.0.2
     3.5.3

From a953027aad38c9f54e952133949280ec2f4c04e8 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 26 May 2023 21:21:07 +0300
Subject: [PATCH 242/280] [fix][sec] Upgrade sqlite-jdbc to resolve
 CVE-2023-32697 (#20411)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 0fb86ad41a71d..b45debcc9db25 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,7 +181,7 @@ flexible messaging model and an intuitive client API.
     2.10.10
     2.5.0
     5.1.0
-    3.36.0.3
+    3.42.0.0
     8.0.11
     42.5.1
     0.4.6

From aa3bfcda6935dc2d9c69dce51f96ba80baeae0b7 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Sat, 27 May 2023 18:06:55 +0300
Subject: [PATCH 243/280] [improve][ci] Speed up OWASP dependency check in
 Pulsar CI workflow (#20412)

---
 .github/workflows/pulsar-ci.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 2c4abdb704178..57b9b082da266 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -1273,8 +1273,10 @@ jobs:
           cd $HOME
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
       # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
-      - name: run "clean verify" to trigger dependency check
-        run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
+      - name: trigger dependency check
+        run: |
+          mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
+            -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
 
       - name: Upload report
         uses: actions/upload-artifact@v3

From 1c813fdaeb9f752b84347b5a62e219dd66e79a26 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Mon, 29 May 2023 10:20:08 +0800
Subject: [PATCH 244/280] [fix][broker] Fix ledger cachemiss size metric
 (#20257)

---
 .../bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java       | 4 ++--
 .../bookkeeper/mledger/impl/cache/EntryCacheDisabled.java     | 4 ++--
 .../bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java    | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
index cb3d72cc5972f..e057dee99538e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
@@ -100,8 +100,8 @@ public void recordReadEntriesError() {
         readEntriesOpsFailed.recordEvent();
     }
 
-    public void recordReadEntriesOpsCacheMisses() {
-        readEntriesOpsCacheMisses.recordEvent();
+    public void recordReadEntriesOpsCacheMisses(int count, long totalSize) {
+        readEntriesOpsCacheMisses.recordMultipleEvents(count, totalSize);
     }
 
     public void addAddEntryLatencySample(long latency, TimeUnit unit) {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
index d2add99b701ac..d1050e0062826 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
@@ -93,7 +93,7 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole
                     } finally {
                         ledgerEntries.close();
                     }
-                    ml.getMbean().recordReadEntriesOpsCacheMisses();
+                    ml.getMbean().recordReadEntriesOpsCacheMisses(entries.size(), totalSize);
                     ml.getFactory().getMbean().recordCacheMiss(entries.size(), totalSize);
                     ml.getMbean().addReadEntriesSample(entries.size(), totalSize);
 
@@ -121,7 +121,7 @@ public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks.
                             LedgerEntry ledgerEntry = iterator.next();
                             EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor);
 
-                            ml.getMbean().recordReadEntriesOpsCacheMisses();
+                            ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength());
                             ml.getFactory().getMbean().recordCacheMiss(1, returnEntry.getLength());
                             ml.getMbean().addReadEntriesSample(1, returnEntry.getLength());
                             callback.readEntryComplete(returnEntry, ctx);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
index 7747f9bcd93b6..27aec6f178e39 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
@@ -256,7 +256,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt
                                 LedgerEntry ledgerEntry = iterator.next();
                                 EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor);
 
-                                ml.getMbean().recordReadEntriesOpsCacheMisses();
+                                ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength());
                                 manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength());
                                 ml.getMbean().addReadEntriesSample(1, returnEntry.getLength());
                                 callback.readEntryComplete(returnEntry, ctx);
@@ -450,7 +450,7 @@ CompletableFuture> readFromStorage(ReadHandle lh,
                                     }
                                 }
 
-                                ml.getMbean().recordReadEntriesOpsCacheMisses();
+                                ml.getMbean().recordReadEntriesOpsCacheMisses(entriesToReturn.size(), totalSize);
                                 manager.mlFactoryMBean.recordCacheMiss(entriesToReturn.size(), totalSize);
                                 ml.getMbean().addReadEntriesSample(entriesToReturn.size(), totalSize);
 

From ab810f4f59dd7d6c8b5313ceb334873a7d2cde31 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Mon, 29 May 2023 14:46:38 +0800
Subject: [PATCH 245/280] [fix][broker] If ledger lost, cursor mark delete
 position can not forward (#18620)

Motivation:
Configuration `autoSkipNonRecoverableData` is designed to turn this feature on if we can accept partial data loss. When a ledger is lost, the broker will still work. But now we have this problem: If a ledger is lost, consumer and producer can work, but the cursor mark delete position can not forward.

Modifications:
- When an unrecoverable ledger is found, remove the records in`individualDeletedMessages` and `batchDeletedIndexes`.
- When the managed cursor is recovered, check whether there are invalid records in `individualDeletedMessages` and `batchDeletedIndexes` and print a warning log.
---
 .../bookkeeper/mledger/ManagedCursor.java     |   6 +
 .../bookkeeper/mledger/ManagedLedger.java     |   6 +
 .../mledger/impl/ManagedCursorImpl.java       |  40 +++
 .../mledger/impl/ManagedLedgerImpl.java       |   7 +
 .../bookkeeper/mledger/impl/OpReadEntry.java  |   5 +
 .../LedgerLostAndSkipNonRecoverableTest.java  | 296 ++++++++++++++++++
 6 files changed, 360 insertions(+)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
index 7802ed07781ba..edbfa0b43204e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
@@ -786,6 +786,12 @@ Set asyncReplayEntries(
      */
     long getEstimatedSizeSinceMarkDeletePosition();
 
+    /**
+     * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is
+     * used to delete information about this ledger in the ManagedCursor.
+     */
+    default void skipNonRecoverableLedger(long ledgerId){}
+
     /**
      * Returns cursor throttle mark-delete rate.
      *
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
index 4ca56508891a1..c7dd8ea9129b7 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
@@ -631,6 +631,12 @@ void asyncSetProperties(Map properties, AsyncCallbacks.UpdatePro
      */
     void trimConsumedLedgersInBackground(CompletableFuture promise);
 
+    /**
+     * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is
+     * used to delete information about this ledger in the ManagedCursor.
+     */
+    default void skipNonRecoverableLedger(long ledgerId){}
+
     /**
      * Roll current ledger if it is full.
      */
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index ef607fa7ed7cf..663081c932052 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -2718,6 +2718,46 @@ void setReadPosition(Position newReadPositionInt) {
         }
     }
 
+    /**
+     * Manually acknowledge all entries in the lost ledger.
+     * - Since this is an uncommon event, we focus on maintainability. So we do not modify
+     *   {@link #individualDeletedMessages} and {@link #batchDeletedIndexes}, but call
+     *   {@link #asyncDelete(Position, AsyncCallbacks.DeleteCallback, Object)}.
+     * - This method is valid regardless of the consumer ACK type.
+     * - If there is a consumer ack request after this event, it will also work.
+     */
+    @Override
+    public void skipNonRecoverableLedger(final long ledgerId){
+        LedgerInfo ledgerInfo = ledger.getLedgersInfo().get(ledgerId);
+        if (ledgerInfo == null) {
+            return;
+        }
+        lock.writeLock().lock();
+        log.warn("[{}] [{}] Since the ledger [{}] is lost and the autoSkipNonRecoverableData is true, this ledger will"
+                + " be auto acknowledge in subscription", ledger.getName(), name, ledgerId);
+        try {
+            for (int i = 0; i < ledgerInfo.getEntries(); i++) {
+                if (!individualDeletedMessages.contains(ledgerId, i)) {
+                    asyncDelete(PositionImpl.get(ledgerId, i), new AsyncCallbacks.DeleteCallback() {
+                        @Override
+                        public void deleteComplete(Object ctx) {
+                            // ignore.
+                        }
+
+                        @Override
+                        public void deleteFailed(ManagedLedgerException ex, Object ctx) {
+                            // The method internalMarkDelete already handled the failure operation. We only need to
+                            // make sure the memory state is updated.
+                            // If the broker crashed, the non-recoverable ledger will be detected again.
+                        }
+                    }, null);
+                }
+            }
+        } finally {
+            lock.writeLock().unlock();
+        }
+    }
+
     // //////////////////////////////////////////////////
 
     void startCreatingNewMetadataLedger() {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 15e9d332fa103..9b3d7e46aaa8c 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1741,6 +1741,13 @@ synchronized void ledgerClosed(final LedgerHandle lh) {
         }
     }
 
+    @Override
+    public void skipNonRecoverableLedger(long ledgerId){
+        for (ManagedCursor managedCursor : cursors) {
+            managedCursor.skipNonRecoverableLedger(ledgerId);
+        }
+    }
+
     synchronized void createLedgerAfterClosed() {
         if (isNeededCreateNewLedgerAfterCloseLedger()) {
             log.info("[{}] Creating a new ledger after closed", name);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
index 19211553a5f74..7b59c3903d5bc 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
@@ -116,9 +116,11 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
                     readPosition, exception.getMessage());
             final ManagedLedgerImpl ledger = (ManagedLedgerImpl) cursor.getManagedLedger();
             Position nexReadPosition;
+            Long lostLedger = null;
             if (exception instanceof ManagedLedgerException.LedgerNotExistException) {
                 // try to find and move to next valid ledger
                 nexReadPosition = cursor.getNextLedgerPosition(readPosition.getLedgerId());
+                lostLedger = readPosition.ledgerId;
             } else {
                 // Skip this read operation
                 nexReadPosition = ledger.getValidPositionAfterSkippedEntries(readPosition, count);
@@ -131,6 +133,9 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
                 return;
             }
             updateReadPosition(nexReadPosition);
+            if (lostLedger != null) {
+                cursor.getManagedLedger().skipNonRecoverableLedger(lostLedger);
+            }
             checkReadCompletion();
         } else {
             if (!(exception instanceof TooManyRequestsException)) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
new file mode 100644
index 0000000000000..389af8f2cd9f9
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.BatchMessageIdImpl;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker")
+public class LedgerLostAndSkipNonRecoverableTest extends ProducerConsumerBase {
+
+    private static final String DEFAULT_NAMESPACE = "my-property/my-ns";
+
+    @BeforeClass
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        super.producerBaseSetup();
+    }
+
+    @AfterClass
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    protected void doInitConf() throws Exception {
+        conf.setAutoSkipNonRecoverableData(true);
+    }
+
+    @DataProvider(name = "batchEnabled")
+    public Object[][] batchEnabled(){
+        return new Object[][]{
+                {true},
+                {false}
+        };
+    }
+
+    @Test(timeOut = 30000, dataProvider = "batchEnabled")
+    public void testMarkDeletedPositionCanForwardAfterTopicLedgerLost(boolean enabledBatch) throws Exception {
+        String topicSimpleName = UUID.randomUUID().toString().replaceAll("-", "");
+        String subName = UUID.randomUUID().toString().replaceAll("-", "");
+        String topicName = String.format("persistent://%s/%s", DEFAULT_NAMESPACE, topicSimpleName);
+
+        log.info("create topic and subscription.");
+        Consumer sub = createConsumer(topicName, subName, enabledBatch);
+        sub.redeliverUnacknowledgedMessages();
+        sub.close();
+
+        log.info("send many messages.");
+        int ledgerCount = 3;
+        int messageCountPerLedger = enabledBatch ? 25 : 5;
+        int messageCountPerEntry = enabledBatch ? 5 : 1;
+        List[] sendMessages =
+                sendManyMessages(topicName, ledgerCount, messageCountPerLedger, messageCountPerEntry);
+        int sendMessageCount = Arrays.asList(sendMessages).stream()
+                .flatMap(s -> s.stream()).collect(Collectors.toList()).size();
+        log.info("send {} messages", sendMessageCount);
+
+        log.info("make individual ack.");
+        ConsumerAndReceivedMessages consumerAndReceivedMessages1 =
+                waitConsumeAndAllMessages(topicName, subName, enabledBatch,false);
+        List[] messageIds = consumerAndReceivedMessages1.messageIds;
+        Consumer consumer = consumerAndReceivedMessages1.consumer;
+        MessageIdImpl individualPosition = messageIds[1].get(messageCountPerEntry - 1);
+        MessageIdImpl expectedMarkDeletedPosition =
+                new MessageIdImpl(messageIds[0].get(0).getLedgerId(), messageIds[0].get(0).getEntryId(), -1);
+        MessageIdImpl lastPosition =
+                new MessageIdImpl(messageIds[2].get(4).getLedgerId(), messageIds[2].get(4).getEntryId(), -1);
+        consumer.acknowledge(individualPosition);
+        consumer.acknowledge(expectedMarkDeletedPosition);
+        waitPersistentCursorLedger(topicName, subName, expectedMarkDeletedPosition.getLedgerId(),
+                expectedMarkDeletedPosition.getEntryId());
+        consumer.close();
+
+        log.info("Make lost ledger [{}].", individualPosition.getLedgerId());
+        pulsar.getBrokerService().getTopic(topicName, false).get().get().close(false);
+        pulsarTestContext.getMockBookKeeper().deleteLedger(individualPosition.getLedgerId());
+
+        log.info("send some messages.");
+        sendManyMessages(topicName, 3, messageCountPerEntry);
+
+        log.info("receive all messages then verify mark deleted position");
+        ConsumerAndReceivedMessages consumerAndReceivedMessages2 =
+                waitConsumeAndAllMessages(topicName, subName, enabledBatch, true);
+        waitMarkDeleteLargeAndEquals(topicName, subName, lastPosition.getLedgerId(), lastPosition.getEntryId());
+
+        // cleanup
+        consumerAndReceivedMessages2.consumer.close();
+        admin.topics().delete(topicName);
+    }
+
+    private ManagedCursorImpl getCursor(String topicName, String subName) throws Exception {
+        PersistentSubscription subscription_ =
+                (PersistentSubscription) pulsar.getBrokerService().getTopic(topicName, false)
+                        .get().get().getSubscription(subName);
+        return  (ManagedCursorImpl) subscription_.getCursor();
+    }
+
+    private void waitMarkDeleteLargeAndEquals(String topicName, String subName, final long markDeletedLedgerId,
+                                            final long markDeletedEntryId) throws Exception {
+        Awaitility.await().atMost(Duration.ofSeconds(45)).untilAsserted(() -> {
+            Position persistentMarkDeletedPosition = getCursor(topicName, subName).getMarkDeletedPosition();
+            log.info("markDeletedPosition {}:{}, expected {}:{}", persistentMarkDeletedPosition.getLedgerId(),
+                    persistentMarkDeletedPosition.getEntryId(), markDeletedLedgerId, markDeletedEntryId);
+            Assert.assertTrue(persistentMarkDeletedPosition.getLedgerId() >= markDeletedLedgerId);
+            if (persistentMarkDeletedPosition.getLedgerId() > markDeletedLedgerId){
+                return;
+            }
+            Assert.assertTrue(persistentMarkDeletedPosition.getEntryId() >= markDeletedEntryId);
+        });
+    }
+
+    private void waitPersistentCursorLedger(String topicName, String subName, final long markDeletedLedgerId,
+                                            final long markDeletedEntryId) throws Exception {
+        Awaitility.await().untilAsserted(() -> {
+            Position persistentMarkDeletedPosition = getCursor(topicName, subName).getPersistentMarkDeletedPosition();
+            Assert.assertEquals(persistentMarkDeletedPosition.getLedgerId(), markDeletedLedgerId);
+            Assert.assertEquals(persistentMarkDeletedPosition.getEntryId(), markDeletedEntryId);
+        });
+    }
+
+    private List[] sendManyMessages(String topicName, int ledgerCount, int messageCountPerLedger,
+                                                   int messageCountPerEntry) throws Exception {
+        List[] messageIds = new List[ledgerCount];
+        for (int i = 0; i < ledgerCount; i++){
+            admin.topics().unload(topicName);
+            if (messageCountPerEntry == 1) {
+                messageIds[i] = sendManyMessages(topicName, messageCountPerLedger);
+            } else {
+                messageIds[i] = sendManyBatchedMessages(topicName, messageCountPerEntry,
+                        messageCountPerLedger / messageCountPerEntry);
+            }
+        }
+        return messageIds;
+    }
+
+    private List sendManyMessages(String topicName, int messageCountPerLedger,
+                                                   int messageCountPerEntry) throws Exception {
+        if (messageCountPerEntry == 1) {
+            return sendManyMessages(topicName, messageCountPerLedger);
+        } else {
+            return sendManyBatchedMessages(topicName, messageCountPerEntry,
+                    messageCountPerLedger / messageCountPerEntry);
+        }
+    }
+
+    private List sendManyMessages(String topicName, int msgCount) throws Exception {
+        List messageIdList = new ArrayList<>();
+        final Producer producer = pulsarClient.newProducer(Schema.JSON(String.class))
+                .topic(topicName)
+                .enableBatching(false)
+                .create();
+        long timestamp = System.currentTimeMillis();
+        for (int i = 0; i < msgCount; i++){
+            String messageSuffix = String.format("%s-%s", timestamp, i);
+            MessageIdImpl messageIdSent = (MessageIdImpl) producer.newMessage()
+                    .key(String.format("Key-%s", messageSuffix))
+                    .value(String.format("Msg-%s", messageSuffix))
+                    .send();
+            messageIdList.add(messageIdSent);
+        }
+        producer.close();
+        return messageIdList;
+    }
+
+    private List sendManyBatchedMessages(String topicName, int msgCountPerEntry, int entryCount)
+            throws Exception {
+        Producer producer = pulsarClient.newProducer(Schema.JSON(String.class))
+                .topic(topicName)
+                .enableBatching(true)
+                .batchingMaxPublishDelay(Integer.MAX_VALUE, TimeUnit.SECONDS)
+                .batchingMaxMessages(Integer.MAX_VALUE)
+                .create();
+        List> messageIdFutures = new ArrayList<>();
+        for (int i = 0; i < entryCount; i++){
+            for (int j = 0; j < msgCountPerEntry; j++){
+                CompletableFuture messageIdFuture =
+                        producer.newMessage().value(String.format("entry-seq[%s], batch_index[%s]", i, j)).sendAsync();
+                messageIdFutures.add(messageIdFuture);
+            }
+            producer.flush();
+        }
+        producer.close();
+        FutureUtil.waitForAll(messageIdFutures).get();
+        return messageIdFutures.stream().map(f -> (MessageIdImpl)f.join()).collect(Collectors.toList());
+    }
+
+    private ConsumerAndReceivedMessages waitConsumeAndAllMessages(String topicName, String subName,
+                                                            final boolean enabledBatch,
+                                                            boolean ack) throws Exception {
+        List messageIds = new ArrayList<>();
+        final Consumer consumer = createConsumer(topicName, subName, enabledBatch);
+        while (true){
+            Message message = consumer.receive(5, TimeUnit.SECONDS);
+            if (message != null){
+                messageIds.add((MessageIdImpl) message.getMessageId());
+                if (ack) {
+                    consumer.acknowledge(message);
+                }
+            } else {
+                break;
+            }
+        }
+        log.info("receive {} messages", messageIds.size());
+        return new ConsumerAndReceivedMessages(consumer, sortMessageId(messageIds, enabledBatch));
+    }
+
+    @AllArgsConstructor
+    private static class ConsumerAndReceivedMessages {
+        private Consumer consumer;
+        private List[] messageIds;
+    }
+
+    private List[] sortMessageId(List messageIds, boolean enabledBatch){
+        Map> map = messageIds.stream().collect(Collectors.groupingBy(v -> v.getLedgerId()));
+        TreeMap> sortedMap = new TreeMap<>(map);
+        List[] res = new List[sortedMap.size()];
+        Iterator>> iterator = sortedMap.entrySet().iterator();
+        for (int i = 0; i < sortedMap.size(); i++){
+            res[i] = iterator.next().getValue();
+        }
+        for (List list : res){
+            list.sort((m1, m2) -> {
+                if (enabledBatch){
+                    BatchMessageIdImpl mb1 = (BatchMessageIdImpl) m1;
+                    BatchMessageIdImpl mb2 = (BatchMessageIdImpl) m2;
+                    return (int) (mb1.getLedgerId() * 1000000 + mb1.getEntryId() * 1000 + mb1.getBatchIndex() -
+                            mb2.getLedgerId() * 1000000 + mb2.getEntryId() * 1000 + mb2.getBatchIndex());
+                }
+                return (int) (m1.getLedgerId() * 1000 + m1.getEntryId() -
+                        m2.getLedgerId() * 1000 + m2.getEntryId());
+            });
+        }
+        return res;
+    }
+
+    private Consumer createConsumer(String topicName, String subName, boolean enabledBatch) throws Exception {
+        final Consumer consumer = pulsarClient.newConsumer(Schema.JSON(String.class))
+                .autoScaledReceiverQueueSizeEnabled(false)
+                .subscriptionType(SubscriptionType.Failover)
+                .isAckReceiptEnabled(true)
+                .enableBatchIndexAcknowledgment(enabledBatch)
+                .receiverQueueSize(1000)
+                .topic(topicName)
+                .subscriptionName(subName)
+                .subscribe();
+        return consumer;
+    }
+}

From b5ef09e537ddb24419e9ad05b1dc940ab10ea70a Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Mon, 29 May 2023 18:01:24 +0800
Subject: [PATCH 246/280] [fix][test] Fix SegmentAbortedTxnProcessorTest
 (#20358)

---
 .../SegmentAbortedTxnProcessorTest.java       | 35 ++++++++++++++++---
 .../transaction/TransactionTestBase.java      |  6 ++--
 .../buffer/TransactionBufferCloseTest.java    |  4 ---
 3 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
index cb15ab003f7b7..0600833b1adb6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
@@ -25,6 +25,7 @@
 import java.lang.reflect.Field;
 import java.util.LinkedList;
 import java.util.NavigableMap;
+import java.util.Optional;
 import java.util.Queue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
@@ -38,10 +39,12 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
 import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
+import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
+import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl;
 import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment;
@@ -56,6 +59,7 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.TopicStats;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -71,11 +75,13 @@ public class SegmentAbortedTxnProcessorTest extends TransactionTestBase {
     @Override
     @BeforeClass
     protected void setup() throws Exception {
-        setUpBase(1, 1, PROCESSOR_TOPIC, 0);
+        setUpBase(1, 1, null, 0);
         this.pulsarService = getPulsarServiceList().get(0);
         this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true);
         this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() +
                 SEGMENT_SIZE * 3);
+        admin.topics().createNonPartitionedTopic(PROCESSOR_TOPIC);
+        assertTrue(getSnapshotAbortedTxnProcessor(PROCESSOR_TOPIC) instanceof SnapshotSegmentAbortedTxnProcessorImpl);
     }
 
     @Override
@@ -311,15 +317,18 @@ private void doCompaction(TopicName topic) throws Exception {
      */
     @Test
     public void testSnapshotProcessorUpgrade() throws Exception {
+        String NAMESPACE2 = TENANT + "/ns2";
+        admin.namespaces().createNamespace(NAMESPACE2);
         this.pulsarService = getPulsarServiceList().get(0);
         this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false);
 
         // Create a topic, send 10 messages without using transactions, and send 10 messages using transactions.
         // Abort these transactions and verify the data.
-        final String topicName = "persistent://" + NAMESPACE1 + "/testSnapshotProcessorUpgrade";
+        final String topicName = "persistent://" + NAMESPACE2 + "/testSnapshotProcessorUpgrade";
         Producer producer = pulsarClient.newProducer().topic(topicName).create();
         Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe();
 
+        assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SingleSnapshotAbortedTxnProcessorImpl);
         // Send 10 messages without using transactions
         for (int i = 0; i < 10; i++) {
             producer.send(("test-message-" + i).getBytes());
@@ -352,6 +361,7 @@ public void testSnapshotProcessorUpgrade() throws Exception {
 
         // Unload the topic
         admin.topics().unload(topicName);
+        assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl);
 
         // Sends a new message using a transaction and aborts it.
         Transaction txn = pulsarClient.newTransaction()
@@ -362,7 +372,7 @@ public void testSnapshotProcessorUpgrade() throws Exception {
 
         // Verifies that the topic has exactly one segment.
         Awaitility.await().untilAsserted(() -> {
-            String segmentTopic = "persistent://" + NAMESPACE1 + "/" +
+            String segmentTopic = "persistent://" + NAMESPACE2 + "/" +
                     SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS;
             TopicStats topicStats = admin.topics().getStats(segmentTopic);
             assertEquals(1, topicStats.getMsgInCounter());
@@ -401,7 +411,7 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except
         String topicName = "persistent://" + namespaceName + "/newTopic";
         Producer producer = pulsarClient.newProducer().topic(topicName).create();
         producer.close();
-
+        assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl);
         // Check that the __transaction_buffer_snapshot topic is not created in the same namespace
         String transactionBufferSnapshotTopic = "persistent://" + namespaceName + "/" +
                 SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT;
@@ -415,4 +425,21 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except
         // Destroy the namespace after the test
         admin.namespaces().deleteNamespace(namespaceName, true);
     }
+
+    private AbortedTxnProcessor getSnapshotAbortedTxnProcessor(String topicName) {
+        PersistentTopic persistentTopic = getPersistentTopic(topicName);
+        return WhiteboxImpl.getInternalState(persistentTopic.getTransactionBuffer(), "snapshotAbortedTxnProcessor");
+    }
+
+    private PersistentTopic getPersistentTopic(String topicName) {
+        for (PulsarService pulsar : getPulsarServiceList()) {
+            CompletableFuture> future =
+                    pulsar.getBrokerService().getTopic(topicName, false);
+            if (future == null) {
+                continue;
+            }
+            return (PersistentTopic) future.join().get();
+        }
+        throw new NullPointerException("topic[" + topicName +  "] not found");
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
index f45eda8d21fbe..cd0c089ad41be 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
@@ -114,10 +114,10 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int
                 new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
         admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString());
         createTransactionCoordinatorAssign(numPartitionsOfTC);
+        admin.tenants().createTenant(TENANT,
+                new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
+        admin.namespaces().createNamespace(NAMESPACE1);
         if (topic != null) {
-            admin.tenants().createTenant(TENANT,
-                    new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
-            admin.namespaces().createNamespace(NAMESPACE1);
             if (numPartitions == 0) {
                 admin.topics().createNonPartitionedTopic(topic);
             } else {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
index e92cf29521e34..d1784f6a392bf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.broker.transaction.buffer;
 
-import com.google.common.collect.Sets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -30,7 +29,6 @@
 import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient;
 import org.apache.pulsar.client.impl.PulsarClientImpl;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -49,8 +47,6 @@ protected void setup() throws Exception {
         setUpBase(1, 16, null, 0);
         Awaitility.await().until(() -> ((PulsarClientImpl) pulsarClient)
                 .getTcClient().getState() == TransactionCoordinatorClient.State.READY);
-        admin.tenants().createTenant(TENANT,
-                new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
     }
 
     @AfterMethod(alwaysRun = true)

From 6079a9186ac8d1c96c97828a289d7865c5890ebd Mon Sep 17 00:00:00 2001
From: Zike Yang 
Date: Mon, 29 May 2023 21:03:34 +0800
Subject: [PATCH 247/280] [fix][io] Close the kafka source connector if there
 is uncaught exception (#20424)

Fixes #19880

### Motivation

If any unexpected and uncaught exceptions are thrown in the `runnerThread` of the Kafka source connector, the connector will just log that message and get stuck. The connector won't exit but won't do anything either.

### Modifications

* Close the connector if there is uncaught exception

Signed-off-by: Zike Yang 
---
 .../pulsar/io/kafka/KafkaAbstractSource.java  |  9 +++++++-
 .../kafka/source/KafkaAbstractSourceTest.java | 23 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
index 565c36047474b..8d2cbd8e74e14 100644
--- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
+++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
@@ -189,7 +189,14 @@ public void start() {
             }
         });
         runnerThread.setUncaughtExceptionHandler(
-                (t, e) -> LOG.error("[{}] Error while consuming records", t.getName(), e));
+                (t, e) -> {
+                    LOG.error("[{}] Error while consuming records", t.getName(), e);
+                    try {
+                        this.close();
+                    } catch (InterruptedException ex) {
+                        // The interrupted exception is thrown by the runnerThread itself. Ignore it.
+                    }
+                });
         runnerThread.setName("Kafka Source Thread");
         runnerThread.start();
     }
diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java
index 612cf0bc6d2b1..bc06c3e1935b4 100644
--- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java
+++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java
@@ -20,7 +20,10 @@
 
 
 import com.google.common.collect.ImmutableMap;
+import java.util.Collection;
 import java.util.Collections;
+import java.lang.reflect.Field;
+import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
@@ -28,6 +31,7 @@
 import org.apache.pulsar.io.core.SourceContext;
 import org.apache.pulsar.io.kafka.KafkaAbstractSource;
 import org.apache.pulsar.io.kafka.KafkaSourceConfig;
+import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
@@ -153,6 +157,25 @@ public final void loadFromSaslYamlFileTest() throws IOException {
         assertEquals(config.getSslTruststorePassword(), "cert_pwd");
     }
 
+    @Test
+    public final void closeConnectorWhenUnexpectedExceptionThrownTest() throws Exception {
+        KafkaAbstractSource source = new DummySource();
+        Consumer consumer = mock(Consumer.class);
+        Mockito.doThrow(new RuntimeException("Uncaught exception")).when(consumer)
+                .subscribe(Mockito.any(Collection.class));
+
+        Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer");
+        consumerField.setAccessible(true);
+        consumerField.set(source, consumer);
+
+        source.start();
+
+        Field runningField = KafkaAbstractSource.class.getDeclaredField("running");
+        runningField.setAccessible(true);
+
+        Assert.assertFalse((boolean) runningField.get(source));
+    }
+
     private File getFile(String name) {
         ClassLoader classLoader = getClass().getClassLoader();
         return new File(classLoader.getResource(name).getFile());

From 0bdcaecd38edbf4597261240aa47aa50c879921e Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 29 May 2023 16:33:46 +0300
Subject: [PATCH 248/280] [improve][ci] Improve TestNG test execution to debug
 flaky test failures (#20425)

---
 .../apache/pulsar/tests/FailFastNotifier.java |  3 +--
 .../pulsar/tests/PulsarTestListener.java      | 25 +++++++++++++------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java
index 627a4ec30547b..fe76a79b2c4ce 100644
--- a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java
@@ -124,8 +124,7 @@ public void beforeInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestRes
                         || iTestNGMethod.isAfterTestConfiguration())) {
                     throw new FailFastSkipException("Skipped after failure since testFailFast system property is set.");
                 }
-            }
-            if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) {
+            } else if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) {
                 throw new FailFastSkipException("Skipped after failure since kill switch file exists.");
             }
         }
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java
index b3d70621843ca..2d1f1273272c5 100644
--- a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java
@@ -44,20 +44,29 @@ public void onTestFailure(ITestResult result) {
         if (!(result.getThrowable() instanceof SkipException)) {
             System.out.format("!!!!!!!!! FAILURE-- %s.%s(%s)-------\n", result.getTestClass(),
                     result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
-        }
-        if (result.getThrowable() != null) {
-            result.getThrowable().printStackTrace();
-            if (result.getThrowable() instanceof ThreadTimeoutException) {
-                System.out.println("====== THREAD DUMPS ======");
-                System.out.println(ThreadDumpUtil.buildThreadDiagnosticString());
+            if (result.getThrowable() != null) {
+                result.getThrowable().printStackTrace();
+                if (result.getThrowable() instanceof ThreadTimeoutException) {
+                    System.out.println("====== THREAD DUMPS ======");
+                    System.out.println(ThreadDumpUtil.buildThreadDiagnosticString());
+                }
             }
         }
     }
 
     @Override
     public void onTestSkipped(ITestResult result) {
-        System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(),
-                result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
+        if (!(result.getThrowable() instanceof SkipException)) {
+            System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(),
+                    result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
+            if (result.getThrowable() != null) {
+                result.getThrowable().printStackTrace();
+                if (result.getThrowable() instanceof ThreadTimeoutException) {
+                    System.out.println("====== THREAD DUMPS ======");
+                    System.out.println(ThreadDumpUtil.buildThreadDiagnosticString());
+                }
+            }
+        }
     }
 
     @Override

From fafadee9061f1256cbb73385415bb9b978e74401 Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Mon, 29 May 2023 22:11:28 +0800
Subject: [PATCH 249/280] [improve] [broker] Avoid
 `PersistentSubscription.expireMessages` logic check backlog twice. (#20416)

---
 .../broker/service/persistent/PersistentSubscription.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index 4e74857cef940..dc666f3a18e48 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -1048,8 +1048,9 @@ public List getConsumers() {
 
     @Override
     public boolean expireMessages(int messageTTLInSeconds) {
-        if ((getNumberOfEntriesInBacklog(false) == 0) || (dispatcher != null && dispatcher.isConsumerConnected()
-                && getNumberOfEntriesInBacklog(false) < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK
+        long backlog = getNumberOfEntriesInBacklog(false);
+        if (backlog == 0 || (dispatcher != null && dispatcher.isConsumerConnected()
+                && backlog < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK
                 && !topic.isOldestMessageExpired(cursor, messageTTLInSeconds))) {
             // don't do anything for almost caught-up connected subscriptions
             return false;

From bdd1bf1fcabdfc52527d07a8f8a288930e2cb7dd Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 29 May 2023 20:22:05 +0300
Subject: [PATCH 250/280] [improve][broker] Replace String.intern() with Guava
 Interner (#20432)

---
 .../broker/service/AbstractReplicator.java    |  7 +--
 .../pulsar/broker/service/ServerCnx.java      |  3 +-
 .../pulsar/common/util/StringInterner.java    | 46 +++++++++++++++++++
 3 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
index deab89cda72dc..d38a5da3adba5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.client.impl.PulsarClientImpl;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.common.util.StringInterner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -73,9 +74,9 @@ public AbstractReplicator(String localCluster, String localTopicName, String rem
         this.brokerService = brokerService;
         this.localTopicName = localTopicName;
         this.replicatorPrefix = replicatorPrefix;
-        this.localCluster = localCluster.intern();
+        this.localCluster = StringInterner.intern(localCluster);
         this.remoteTopicName = remoteTopicName;
-        this.remoteCluster = remoteCluster.intern();
+        this.remoteCluster = StringInterner.intern(remoteCluster);
         this.replicationClient = replicationClient;
         this.client = (PulsarClientImpl) brokerService.pulsar().getClient();
         this.producer = null;
@@ -228,7 +229,7 @@ public static String getRemoteCluster(String remoteCursor) {
     }
 
     public static String getReplicatorName(String replicatorPrefix, String cluster) {
-        return (replicatorPrefix + "." + cluster).intern();
+        return StringInterner.intern(replicatorPrefix + "." + cluster);
     }
 
     /**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 888668e15b167..d3c3971dfaced 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -164,6 +164,7 @@
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.common.util.StringInterner;
 import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
 import org.apache.pulsar.common.util.netty.NettyChannelUtil;
 import org.apache.pulsar.common.util.netty.NettyFutureUtil;
@@ -717,7 +718,7 @@ private void completeConnect(int clientProtoVersion, String clientVersion) {
         }
         setRemoteEndpointProtocolVersion(clientProtoVersion);
         if (isNotBlank(clientVersion)) {
-            this.clientVersion = clientVersion.intern();
+            this.clientVersion = StringInterner.intern(clientVersion);
         }
         if (!service.isAuthenticationEnabled()) {
             log.info("[{}] connected with clientVersion={}, clientProtocolVersion={}, proxyVersion={}", remoteAddress,
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java
new file mode 100644
index 0000000000000..3f6b1c453cdbc
--- /dev/null
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.util;
+
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+
+/**
+ * Deduplicates String instances by interning them using Guava's Interner
+ * which is more efficient than String.intern().
+ */
+public class StringInterner {
+    private static final StringInterner INSTANCE = new StringInterner();
+    private final Interner interner;
+
+    public static String intern(String sample) {
+        return INSTANCE.doIntern(sample);
+    }
+
+    private StringInterner() {
+        this.interner = Interners.newWeakInterner();
+    }
+
+    String doIntern(String sample) {
+        if (sample == null) {
+            return null;
+        }
+        return interner.intern(sample);
+    }
+}

From a26cf3eb78858ef5638650b6dbd7dab1f76a8c61 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 01:42:16 +0300
Subject: [PATCH 251/280] [fix][test] Fix flaky test
 NonPersistentTopicTest.testMsgDropStat, 3rd attempt (#20429)

---
 .../pulsar/client/api/NonPersistentTopicTest.java   | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index 63ce0f00dff15..4f64c4271fe89 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -40,6 +40,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import lombok.Cleanup;
+import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.LoadManager;
@@ -821,7 +822,7 @@ public void testMsgDropStat() throws Exception {
 
         int defaultNonPersistentMessageRate = conf.getMaxConcurrentNonPersistentMessagePerConnection();
         try {
-            final String topicName = "non-persistent://my-property/my-ns/stats-topic";
+            final String topicName = BrokerTestUtil.newUniqueName("non-persistent://my-property/my-ns/stats-topic");
             // restart broker with lower publish rate limit
             conf.setMaxConcurrentNonPersistentMessagePerConnection(1);
             stopBroker();
@@ -829,12 +830,15 @@ public void testMsgDropStat() throws Exception {
 
             pulsar.getBrokerService().updateRates();
 
+            @Cleanup
             Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1")
                     .receiverQueueSize(1).subscribe();
 
+            @Cleanup
             Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2")
                     .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
 
+            @Cleanup
             ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName)
                 .enableBatching(false)
                 .messageRoutingMode(MessageRoutingMode.SinglePartition)
@@ -861,12 +865,12 @@ public void testMsgDropStat() throws Exception {
                     });
                 });
             }
-            latch.await(5, TimeUnit.SECONDS);
+            assertTrue(latch.await(5, TimeUnit.SECONDS));
 
             NonPersistentTopic topic =
                     (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
 
-            Awaitility.await().untilAsserted(() -> {
+            Awaitility.await().ignoreExceptions().untilAsserted(() -> {
                 pulsar.getBrokerService().updateRates();
                 NonPersistentTopicStats stats = topic.getStats(false, false, false);
                 NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
@@ -877,9 +881,6 @@ public void testMsgDropStat() throws Exception {
                 assertTrue(sub2Stats.getMsgDropRate() > 0);
             });
 
-            producer.close();
-            consumer.close();
-            consumer2.close();
         } finally {
             conf.setMaxConcurrentNonPersistentMessagePerConnection(defaultNonPersistentMessageRate);
         }

From 7ea8741af265bd8554e7a52a29a4ed59cb749dea Mon Sep 17 00:00:00 2001
From: thetumbled <52550727+thetumbled@users.noreply.github.com>
Date: Tue, 30 May 2023 10:07:09 +0800
Subject: [PATCH 252/280] [Fix][Txn] Unwrap the completion exception. (#20396)

---
 .../pulsar/broker/TransactionMetadataStoreService.java       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
index 3e3b044ec51b8..35aa7cc2fdd26 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
@@ -169,7 +169,8 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc
                                     tcLoadSemaphore.release();
                                 })).exceptionally(e -> {
                             internalPinnedExecutor.execute(() -> {
-                                completableFuture.completeExceptionally(e.getCause());
+                                Throwable realCause = FutureUtil.unwrapCompletionException(e);
+                                completableFuture.completeExceptionally(realCause);
                                 // release before handle request queue,
                                 //in order to client reconnect infinite loop
                                 tcLoadSemaphore.release();
@@ -180,7 +181,7 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc
                                         CompletableFuture future = deque.poll();
                                         if (future != null) {
                                             // this means that this tc client connection connect fail
-                                            future.completeExceptionally(e);
+                                            future.completeExceptionally(realCause);
                                         } else {
                                             break;
                                         }

From e38091044c428af002b16110531497e2abc897d2 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 07:08:41 +0300
Subject: [PATCH 253/280] [improve][misc] Upgrade Netty to 4.1.93.Final
 (#20423)

---
 buildtools/pom.xml                            |  2 +-
 .../server/src/assemble/LICENSE.bin.txt       | 62 +++++++++----------
 .../shell/src/assemble/LICENSE.bin.txt        | 58 ++++++++---------
 pom.xml                                       |  4 +-
 pulsar-sql/presto-distribution/LICENSE        | 60 +++++++++---------
 5 files changed, 93 insertions(+), 93 deletions(-)

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index 9e4ac022024e1..5a391777f2567 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -47,7 +47,7 @@
     4.1
     8.37
     3.1.2
-    4.1.89.Final
+    4.1.93.Final
     4.2.3
     31.0.1-jre
     1.10.12
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 220f0ac0758b8..487e4e96b6a66 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -289,37 +289,37 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.11.jar
     - org.apache.commons-commons-text-1.10.0.jar
  * Netty
-    - io.netty-netty-buffer-4.1.89.Final.jar
-    - io.netty-netty-codec-4.1.89.Final.jar
-    - io.netty-netty-codec-dns-4.1.89.Final.jar
-    - io.netty-netty-codec-http-4.1.89.Final.jar
-    - io.netty-netty-codec-http2-4.1.89.Final.jar
-    - io.netty-netty-codec-socks-4.1.89.Final.jar
-    - io.netty-netty-codec-haproxy-4.1.89.Final.jar
-    - io.netty-netty-common-4.1.89.Final.jar
-    - io.netty-netty-handler-4.1.89.Final.jar
-    - io.netty-netty-handler-proxy-4.1.89.Final.jar
-    - io.netty-netty-resolver-4.1.89.Final.jar
-    - io.netty-netty-resolver-dns-4.1.89.Final.jar
-    - io.netty-netty-resolver-dns-classes-macos-4.1.89.Final.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar
-    - io.netty-netty-transport-4.1.89.Final.jar
-    - io.netty-netty-transport-classes-epoll-4.1.89.Final.jar
-    - io.netty-netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar
-    - io.netty-netty-transport-native-epoll-4.1.89.Final.jar
-    - io.netty-netty-transport-native-unix-common-4.1.89.Final.jar
-    - io.netty-netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar
-    - io.netty-netty-tcnative-classes-2.0.56.Final.jar
-    - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.18.Final.jar
-    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar
-    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar
+    - io.netty-netty-buffer-4.1.93.Final.jar
+    - io.netty-netty-codec-4.1.93.Final.jar
+    - io.netty-netty-codec-dns-4.1.93.Final.jar
+    - io.netty-netty-codec-http-4.1.93.Final.jar
+    - io.netty-netty-codec-http2-4.1.93.Final.jar
+    - io.netty-netty-codec-socks-4.1.93.Final.jar
+    - io.netty-netty-codec-haproxy-4.1.93.Final.jar
+    - io.netty-netty-common-4.1.93.Final.jar
+    - io.netty-netty-handler-4.1.93.Final.jar
+    - io.netty-netty-handler-proxy-4.1.93.Final.jar
+    - io.netty-netty-resolver-4.1.93.Final.jar
+    - io.netty-netty-resolver-dns-4.1.93.Final.jar
+    - io.netty-netty-resolver-dns-classes-macos-4.1.93.Final.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar
+    - io.netty-netty-transport-4.1.93.Final.jar
+    - io.netty-netty-transport-classes-epoll-4.1.93.Final.jar
+    - io.netty-netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-epoll-4.1.93.Final.jar
+    - io.netty-netty-transport-native-unix-common-4.1.93.Final.jar
+    - io.netty-netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
+    - io.netty-netty-tcnative-classes-2.0.61.Final.jar
+    - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.21.Final.jar
+    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar
+    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar
  * Prometheus client
     - io.prometheus.jmx-collector-0.16.1.jar
     - io.prometheus-simpleclient-0.16.0.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 711890809f1bf..c04ac2b7d0363 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -348,35 +348,35 @@ The Apache Software License, Version 2.0
     - commons-text-1.10.0.jar
     - commons-compress-1.21.jar
  * Netty
-    - netty-buffer-4.1.89.Final.jar
-    - netty-codec-4.1.89.Final.jar
-    - netty-codec-dns-4.1.89.Final.jar
-    - netty-codec-http-4.1.89.Final.jar
-    - netty-codec-socks-4.1.89.Final.jar
-    - netty-codec-haproxy-4.1.89.Final.jar
-    - netty-common-4.1.89.Final.jar
-    - netty-handler-4.1.89.Final.jar
-    - netty-handler-proxy-4.1.89.Final.jar
-    - netty-resolver-4.1.89.Final.jar
-    - netty-resolver-dns-4.1.89.Final.jar
-    - netty-transport-4.1.89.Final.jar
-    - netty-transport-classes-epoll-4.1.89.Final.jar
-    - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar
-    - netty-transport-native-unix-common-4.1.89.Final.jar
-    - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar
-    - netty-tcnative-classes-2.0.56.Final.jar
-    - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar
-    - netty-resolver-dns-classes-macos-4.1.89.Final.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar
+    - netty-buffer-4.1.93.Final.jar
+    - netty-codec-4.1.93.Final.jar
+    - netty-codec-dns-4.1.93.Final.jar
+    - netty-codec-http-4.1.93.Final.jar
+    - netty-codec-socks-4.1.93.Final.jar
+    - netty-codec-haproxy-4.1.93.Final.jar
+    - netty-common-4.1.93.Final.jar
+    - netty-handler-4.1.93.Final.jar
+    - netty-handler-proxy-4.1.93.Final.jar
+    - netty-resolver-4.1.93.Final.jar
+    - netty-resolver-dns-4.1.93.Final.jar
+    - netty-transport-4.1.93.Final.jar
+    - netty-transport-classes-epoll-4.1.93.Final.jar
+    - netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
+    - netty-transport-native-unix-common-4.1.93.Final.jar
+    - netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
+    - netty-tcnative-classes-2.0.61.Final.jar
+    - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar
+    - netty-resolver-dns-classes-macos-4.1.93.Final.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar
  * Prometheus client
     - simpleclient-0.16.0.jar
     - simpleclient_log4j2-0.16.0.jar
diff --git a/pom.xml b/pom.xml
index b45debcc9db25..c1a126b1161c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,8 +140,8 @@ flexible messaging model and an intuitive client API.
     1.1.8.4 
     4.1.12.1 
     5.1.0
-    4.1.89.Final
-    0.0.18.Final
+    4.1.93.Final
+    0.0.21.Final
     9.4.48.v20220622
     2.5.2
     2.34
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index a85d1bc363c1b..2a13985ac4ed5 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -231,37 +231,37 @@ The Apache Software License, Version 2.0
     - commons-compress-1.21.jar
     - commons-lang3-3.11.jar
  * Netty
-    - netty-buffer-4.1.89.Final.jar
-    - netty-codec-4.1.89.Final.jar
-    - netty-codec-dns-4.1.89.Final.jar
-    - netty-codec-http-4.1.89.Final.jar
-    - netty-codec-haproxy-4.1.89.Final.jar
-    - netty-codec-socks-4.1.89.Final.jar
-    - netty-handler-proxy-4.1.89.Final.jar
-    - netty-common-4.1.89.Final.jar
-    - netty-handler-4.1.89.Final.jar
+    - netty-buffer-4.1.93.Final.jar
+    - netty-codec-4.1.93.Final.jar
+    - netty-codec-dns-4.1.93.Final.jar
+    - netty-codec-http-4.1.93.Final.jar
+    - netty-codec-haproxy-4.1.93.Final.jar
+    - netty-codec-socks-4.1.93.Final.jar
+    - netty-handler-proxy-4.1.93.Final.jar
+    - netty-common-4.1.93.Final.jar
+    - netty-handler-4.1.93.Final.jar
     - netty-reactive-streams-2.0.6.jar
-    - netty-resolver-4.1.89.Final.jar
-    - netty-resolver-dns-4.1.89.Final.jar
-    - netty-resolver-dns-classes-macos-4.1.89.Final.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar
-    - netty-tcnative-classes-2.0.56.Final.jar
-    - netty-transport-4.1.89.Final.jar
-    - netty-transport-classes-epoll-4.1.89.Final.jar
-    - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar
-    - netty-transport-native-unix-common-4.1.89.Final.jar
-    - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar
-    - netty-codec-http2-4.1.89.Final.jar
-    - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar
+    - netty-resolver-4.1.93.Final.jar
+    - netty-resolver-dns-4.1.93.Final.jar
+    - netty-resolver-dns-classes-macos-4.1.93.Final.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
+    - netty-tcnative-classes-2.0.61.Final.jar
+    - netty-transport-4.1.93.Final.jar
+    - netty-transport-classes-epoll-4.1.93.Final.jar
+    - netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
+    - netty-transport-native-unix-common-4.1.93.Final.jar
+    - netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar
+    - netty-codec-http2-4.1.93.Final.jar
+    - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar
  * GRPC
     - grpc-api-1.45.1.jar
     - grpc-context-1.45.1.jar

From e0b953175aa39bbea2da967c753e696f45dd69e9 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 11:37:13 +0300
Subject: [PATCH 254/280] [fix][test] Improve deleteNamespaceWithRetry (#20431)

---
 .../broker/auth/MockedPulsarServiceBaseTest.java   | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
index b688d5fbf24d8..c32d3fc3b0b27 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.pulsar.broker.auth;
 
-import static org.apache.pulsar.broker.BrokerTestUtil.*;
+import static org.apache.pulsar.broker.BrokerTestUtil.spyWithoutRecordingInvocations;
 import com.google.common.collect.Sets;
 import java.lang.reflect.Field;
 import java.net.InetSocketAddress;
@@ -55,11 +55,11 @@
 import org.apache.pulsar.tests.TestRetrySupport;
 import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.zookeeper.MockZooKeeper;
+import org.awaitility.Awaitility;
 import org.mockito.Mockito;
 import org.mockito.internal.util.MockUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.DataProvider;
 
 /**
@@ -615,12 +615,18 @@ public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmi
      */
     public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin admin,
                                                 Collection pulsars) throws Exception {
-        Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> {
+        Awaitility.await()
+                .pollDelay(500, TimeUnit.MILLISECONDS)
+                .until(() -> {
             try {
                 // Maybe fail by race-condition with create topics, just retry.
                 admin.namespaces().deleteNamespace(ns, force);
                 return true;
-            } catch (Exception ex) {
+            } catch (PulsarAdminException.NotFoundException ex) {
+                // namespace was already deleted, ignore exception
+                return true;
+            } catch (Exception e) {
+                log.warn("Failed to delete namespace {} (force={})", ns, force, e);
                 return false;
             }
         });

From 639c460a585c2f6cbb6133849348c8b92d2f6fbd Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Tue, 30 May 2023 17:57:08 +0800
Subject: [PATCH 255/280] [fix][broker] Skip loading broker interceptor when
 disableBrokerInterceptors is true (#20422)

Signed-off-by: Zixuan Liu 
---
 .../pulsar/broker/ServiceConfiguration.java   |  2 +-
 .../broker/intercept/BrokerInterceptors.java  |  5 ++++
 .../intercept/BrokerInterceptorTest.java      | 25 +++++++++++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 6fee0e7cd09d2..5dc15bd6a4dcf 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1347,7 +1347,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
 
     @FieldContext(
         category = CATEGORY_SERVER,
-        doc = "Enable or disable the broker interceptor, which is only used for testing for now"
+        doc = "Enable or disable the broker interceptor"
     )
     private boolean disableBrokerInterceptors = true;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java
index cef3f0eb609a1..4ffd8732db94e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java
@@ -59,6 +59,11 @@ public BrokerInterceptors(Map intercep
      * @return the collection of broker event interceptor
      */
     public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException {
+        if (conf.isDisableBrokerInterceptors()) {
+            log.info("Skip loading the broker interceptors when disableBrokerInterceptors is true");
+            return null;
+        }
+
         BrokerInterceptorDefinitions definitions =
                 BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(),
                         conf.getNarExtractionDirectory());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java
index d1cf91635f992..c612104f8bf1b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java
@@ -20,9 +20,12 @@
 
 import static org.mockito.ArgumentMatchers.same;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -36,6 +39,7 @@
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Response;
+import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
@@ -307,4 +311,25 @@ public void requestInterceptorFailedTest() {
         }
     }
 
+    @Test
+    public void testLoadWhenDisableBrokerInterceptorsIsTrue() throws IOException {
+        ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class);
+        serviceConfiguration.setDisableBrokerInterceptors(true);
+        BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration);
+        assertNull(brokerInterceptor);
+
+        verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors();
+        verify(serviceConfiguration, times(0)).getBrokerInterceptorsDirectory();
+    }
+
+    @Test
+    public void testLoadWhenDisableBrokerInterceptorsIsFalse() throws IOException {
+        ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class);
+        serviceConfiguration.setDisableBrokerInterceptors(false);
+        BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration);
+        assertNull(brokerInterceptor);
+
+        verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors();
+        verify(serviceConfiguration, times(1)).getBrokerInterceptorsDirectory();
+    }
 }

From 5f2c29db4c78f062cedba4014eb459885ec3f1b7 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 13:03:19 +0300
Subject: [PATCH 256/280] [fix][test] Replace usage of
 org.testcontainers.shaded.* classes in tests (#20437)

---
 .../src/main/resources/pulsar/suppressions.xml      |  2 +-
 .../broker/LedgerLostAndSkipNonRecoverableTest.java |  2 +-
 .../bucket/BucketDelayedDeliveryTrackerTest.java    |  6 +++---
 .../AntiAffinityNamespaceGroupExtensionTest.java    |  2 +-
 .../loadbalance/extensions/BrokerRegistryTest.java  |  3 +--
 .../extensions/ExtensibleLoadManagerImplTest.java   | 13 ++++++-------
 .../reporter/BrokerLoadDataReporterTest.java        |  2 +-
 .../reporter/TopBundleLoadDataReporterTest.java     |  2 +-
 .../transaction/SegmentAbortedTxnProcessorTest.java |  5 ++---
 .../apache/pulsar/client/impl/NegativeAcksTest.java |  3 +--
 10 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml
index 7c78988db3e90..57a01c60f6a27 100644
--- a/buildtools/src/main/resources/pulsar/suppressions.xml
+++ b/buildtools/src/main/resources/pulsar/suppressions.xml
@@ -38,7 +38,7 @@
     
 
     
-    
+    
 
     
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
index 389af8f2cd9f9..dbaaee8f48cd5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
@@ -44,7 +44,7 @@
 import org.apache.pulsar.client.impl.BatchMessageIdImpl;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.common.util.FutureUtil;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 39b3992fbd195..1e3e72aa0ec44 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -46,14 +46,14 @@
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest;
 import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage;
 import org.apache.pulsar.broker.delayed.MockManagedCursor;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
+import org.awaitility.Awaitility;
 import org.roaringbitmap.RoaringBitmap;
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
-import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.DataProvider;
@@ -439,7 +439,7 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
 
         tracker.close();
     }
-    
+
     @Test(dataProvider = "delayedTracker")
     public void testClear(BucketDelayedDeliveryTracker tracker)
             throws ExecutionException, InterruptedException, TimeoutException {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
index 3469dbe5a5499..014e6fa3c7a04 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import lombok.Cleanup;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.loadbalance.AntiAffinityNamespaceGroupTest;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
@@ -42,7 +43,6 @@
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.common.naming.ServiceUnitId;
-import org.testcontainers.shaded.org.apache.commons.lang3.reflect.FieldUtils;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
index 26986a494f0be..fca41837b9df4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
@@ -25,7 +25,6 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import java.time.Duration;
 import java.util.HashSet;
 import java.util.List;
@@ -54,7 +53,7 @@
 import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.awaitility.Awaitility;
-import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
+import org.awaitility.reflect.WhiteboxImpl;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 498f48d16d2cd..e8a4682e528d5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -48,10 +48,14 @@
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import com.google.common.collect.Sets;
+import java.net.URL;
 import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -59,11 +63,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
-import java.net.URL;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -101,7 +100,7 @@
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
index 93ab35981e1c4..9b0530349d036 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
@@ -45,9 +45,9 @@
 import org.apache.pulsar.client.util.ExecutorProvider;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
+import org.awaitility.Awaitility;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
index be8c6af2b0404..344387b293004 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
@@ -48,7 +48,7 @@
 import org.apache.pulsar.broker.service.PulsarStats;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.awaitility.Awaitility;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
index 0600833b1adb6..32de5944daddf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
@@ -21,7 +21,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import java.lang.reflect.Field;
 import java.util.LinkedList;
 import java.util.NavigableMap;
@@ -58,8 +57,8 @@
 import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.TopicStats;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
-import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
+import org.awaitility.Awaitility;
+import org.awaitility.reflect.WhiteboxImpl;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java
index b4d01e263bc7a..0ae36b4ca9045 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java
@@ -21,7 +21,6 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
-
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
@@ -39,7 +38,7 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.client.api.TopicMessageId;
 import org.awaitility.Awaitility;
-import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
+import org.awaitility.reflect.WhiteboxImpl;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

From 35d961242089a0448cb425da637a2afbdaca0005 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Tue, 30 May 2023 18:07:31 +0800
Subject: [PATCH 257/280] [fix] [meta]Switch to the metadata store thread after
 zk operation (#20303)

---
 .../metadata/impl/AbstractMetadataStore.java  | 13 ++++
 .../pulsar/metadata/impl/ZKMetadataStore.java | 46 ++++++------
 .../pulsar/metadata/MetadataStoreTest.java    | 70 +++++++++++++++++++
 3 files changed, 106 insertions(+), 23 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
index 6fcf8eb6b49b2..4cadf2397a7fa 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
@@ -41,6 +41,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -534,6 +535,18 @@ public void execute(Runnable task, CompletableFuture future) {
         }
     }
 
+    /**
+     * Run the task in the executor thread and fail the future if the executor is shutting down.
+     */
+    @VisibleForTesting
+    public void execute(Runnable task, Supplier>> futures) {
+        try {
+            executor.execute(task);
+        } catch (final Throwable t) {
+            futures.get().forEach(f -> f.completeExceptionally(t));
+        }
+    }
+
     protected static String parent(String path) {
         int idx = path.lastIndexOf('/');
         if (idx <= 0) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
index a6d8eb8344c96..079ae3e2ae5c3 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
@@ -204,29 +204,29 @@ protected void batchOperation(List ops) {
                 }
 
                 // Trigger all the futures in the batch
-                for (int i = 0; i < ops.size(); i++) {
-                    OpResult opr = results.get(i);
-                    MetadataOp op = ops.get(i);
-
-                    switch (op.getType()) {
-                        case PUT:
-                            handlePutResult(op.asPut(), opr);
-                            break;
-                        case DELETE:
-                            handleDeleteResult(op.asDelete(), opr);
-                            break;
-                        case GET:
-                            handleGetResult(op.asGet(), opr);
-                            break;
-                        case GET_CHILDREN:
-                            handleGetChildrenResult(op.asGetChildren(), opr);
-                            break;
-
-                        default:
-                            op.getFuture().completeExceptionally(new MetadataStoreException(
-                                    "Operation type not supported in multi: " + op.getType()));
-                    }
-                }
+                execute(() -> {
+                    for (int i = 0; i < ops.size(); i++) {
+                        OpResult opr = results.get(i);
+                        MetadataOp op = ops.get(i);
+                            switch (op.getType()) {
+                                case PUT:
+                                    handlePutResult(op.asPut(), opr);
+                                    break;
+                                case DELETE:
+                                    handleDeleteResult(op.asDelete(), opr);
+                                    break;
+                                case GET:
+                                    handleGetResult(op.asGet(), opr);
+                                    break;
+                                case GET_CHILDREN:
+                                    handleGetChildrenResult(op.asGetChildren(), opr);
+                                    break;
+                                default:
+                                    op.getFuture().completeExceptionally(new MetadataStoreException(
+                                            "Operation type not supported in multi: " + op.getType()));
+                            }
+                        }
+                }, () -> ops.stream().map(MetadataOp::getFuture).collect(Collectors.toList()));
             }, null);
         } catch (Throwable t) {
             ops.forEach(o -> o.getFuture().completeExceptionally(new MetadataStoreException(t)));
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
index 949b4a9b2bacb..c87e9bda436bf 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -50,8 +51,10 @@
 import org.apache.pulsar.metadata.api.Notification;
 import org.apache.pulsar.metadata.api.NotificationType;
 import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
 import org.assertj.core.util.Lists;
 import org.awaitility.Awaitility;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @Slf4j
@@ -425,6 +428,73 @@ public void testDeleteUnusedDirectories(String provider, Supplier urlSup
         assertFalse(store.exists(prefix).join());
     }
 
+    @DataProvider(name = "conditionOfSwitchThread")
+    public Object[][] conditionOfSwitchThread(){
+        return new Object[][]{
+            {false, false},
+            {false, true},
+            {true, false},
+            {true, true}
+        };
+    }
+
+    @Test(dataProvider = "conditionOfSwitchThread")
+    public void testThreadSwitchOfZkMetadataStore(boolean hasSynchronizer, boolean enabledBatch) throws Exception {
+        final String prefix = newKey();
+        final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", "");
+        MetadataStoreConfig.MetadataStoreConfigBuilder builder =
+                MetadataStoreConfig.builder().metadataStoreName(metadataStoreName);
+        builder.fsyncEnable(false);
+        builder.batchingEnabled(enabledBatch);
+        if (!hasSynchronizer) {
+            builder.synchronizer(null);
+        }
+        MetadataStoreConfig config = builder.build();
+        @Cleanup
+        ZKMetadataStore store = (ZKMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config);
+
+        final Runnable verify = () -> {
+            String currentThreadName = Thread.currentThread().getName();
+            String errorMessage = String.format("Expect to switch to thread %s, but currently it is thread %s",
+                    metadataStoreName, currentThreadName);
+            assertTrue(Thread.currentThread().getName().startsWith(metadataStoreName), errorMessage);
+        };
+
+        // put with node which has parent(but the parent node is not exists).
+        store.put(prefix + "/a1/b1/c1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // put.
+        store.put(prefix + "/b1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // get.
+        store.get(prefix + "/b1").thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // get the node which is not exists.
+        store.get(prefix + "/non").thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // delete.
+        store.delete(prefix + "/b1", Optional.empty()).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // delete the node which is not exists.
+        store.delete(prefix + "/non", Optional.empty()).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).exceptionally(ex -> {
+            verify.run();
+            return null;
+        }).join();
+    }
+
     @Test(dataProvider = "impl")
     public void testPersistent(String provider, Supplier urlSupplier) throws Exception {
         String metadataUrl = urlSupplier.get();

From 113d70d9299d5bc893cee9b524c7b1623dee5892 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= 
Date: Tue, 30 May 2023 18:46:56 +0800
Subject: [PATCH 258/280] [improve][test] Add unit test for metadata cache
 #20363 (#20436)

- Add `MetadataStoreTest#testExistsDistributed` for distributed metaStore implementations only
- Add `MetadataStoreTest#testGetChildrenDistributed` for distributed metaStore implementations only
---
 .../metadata/BaseMetadataStoreTest.java       |  8 +++
 .../pulsar/metadata/MetadataStoreTest.java    | 52 +++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
index ec6e6e03eae71..411ee038c48b0 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
@@ -82,6 +82,14 @@ public Object[][] implementations() {
         };
     }
 
+    @DataProvider(name = "distributedImpl")
+    public Object[][] distributedImplementations() {
+        return new Object[][]{
+                {"ZooKeeper", stringSupplier(() -> zks.getConnectionString())},
+                {"Etcd", stringSupplier(() -> "etcd:" + getEtcdClusterConnectString())},
+        };
+    }
+
     private synchronized String getEtcdClusterConnectString() {
         if (etcdCluster == null) {
             etcdCluster = EtcdClusterExtension.builder().withClusterName("test").withNodes(1).withSsl(false).build()
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
index c87e9bda436bf..246661edc43ee 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
@@ -663,4 +663,56 @@ public void testClosedMetadataStore(String provider, Supplier urlSupplie
             assertTrue(e.getCause() instanceof MetadataStoreException.AlreadyClosedException);
         }
     }
+
+    @Test(dataProvider = "distributedImpl")
+    public void testGetChildrenDistributed(String provider, Supplier urlSupplier) throws Exception {
+        @Cleanup
+        MetadataStore store1 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+        @Cleanup
+        MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+        String parent = newKey();
+        byte[] value = "value1".getBytes(StandardCharsets.UTF_8);
+        store1.put(parent, value, Optional.empty()).get();
+        store1.put(parent + "/a", value, Optional.empty()).get();
+        assertEquals(store1.getChildren(parent).get(), List.of("a"));
+        store1.delete(parent + "/a", Optional.empty()).get();
+        assertEquals(store1.getChildren(parent).get(), Collections.emptyList());
+        store1.delete(parent, Optional.empty()).get();
+        assertEquals(store1.getChildren(parent).get(), Collections.emptyList());
+        store2.put(parent + "/b", value, Optional.empty()).get();
+        // There is a chance watcher event is not triggered before the store1.getChildren() call.
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> assertEquals(store1.getChildren(parent).get(), List.of("b")));
+        store2.put(parent + "/c", value, Optional.empty()).get();
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> assertEquals(store1.getChildren(parent).get(), List.of("b", "c")));
+    }
+
+    @Test(dataProvider = "distributedImpl")
+    public void testExistsDistributed(String provider, Supplier urlSupplier) throws Exception {
+        @Cleanup
+        MetadataStore store1 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+        @Cleanup
+        MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+        String parent = newKey();
+        byte[] value = "value1".getBytes(StandardCharsets.UTF_8);
+        assertFalse(store1.exists(parent).get());
+        store1.put(parent, value, Optional.empty()).get();
+        assertTrue(store1.exists(parent).get());
+        assertFalse(store1.exists(parent + "/a").get());
+        store2.put(parent + "/a", value, Optional.empty()).get();
+        assertTrue(store1.exists(parent + "/a").get());
+        // There is a chance watcher event is not triggered before the store1.exists() call.
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> assertFalse(store1.exists(parent + "/b").get()));
+    }
 }

From fb05c4559c83023a00ceb097c994a088a050c03d Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Tue, 30 May 2023 15:00:41 +0300
Subject: [PATCH 259/280] [improve][misc] PIP-265: Delete wiki/proposals and
 define new process in new folder (#20418)

---
 pip/README.md                  |  97 ++++++++++++++++++++
 pip/TEMPLATE.md                | 163 +++++++++++++++++++++++++++++++++
 wiki/proposals/PIP-template.md |  91 ------------------
 wiki/proposals/PIP.md          | 120 ------------------------
 4 files changed, 260 insertions(+), 211 deletions(-)
 create mode 100644 pip/README.md
 create mode 100644 pip/TEMPLATE.md
 delete mode 100644 wiki/proposals/PIP-template.md
 delete mode 100644 wiki/proposals/PIP.md

diff --git a/pip/README.md b/pip/README.md
new file mode 100644
index 0000000000000..3ed9a1d34cd1d
--- /dev/null
+++ b/pip/README.md
@@ -0,0 +1,97 @@
+# Pulsar Improvement Proposal (PIP)
+
+## What is a PIP?
+
+The PIP is a "Pulsar Improvement Proposal" and it's the mechanism used to propose changes to the Apache Pulsar codebases.
+
+The changes might be in terms of new features, large code refactoring, changes to APIs.
+
+In practical terms, the PIP defines a process in which developers can submit a design doc, receive feedback and get the "go ahead" to execute.
+
+### What is the goal of a PIP?
+
+There are several goals for the PIP process:
+
+1. Ensure community technical discussion of major changes to the Apache Pulsar codebase.
+
+2. Provide clear and thorough design documentation of the proposed changes. Make sure every Pulsar developer will have enough context to effectively perform a code review of the Pull Requests.
+
+3. Use the PIP document to serve as the baseline on which to create the documentation for the new feature.
+
+4. Have greater scrutiny to changes are affecting the public APIs (as defined below) to reduce chances of introducing breaking changes or APIs that are not expressing an ideal semantic.
+
+It is not a goal for PIP to add undue process or slow-down the development.
+
+### When is a PIP required?
+
+* Any new feature for Pulsar brokers or client
+* Any change to the public APIs (Client APIs, REST APIs, Plugin APIs)
+* Any change to the wire protocol APIs
+* Any change to the API of Pulsar CLI tools (eg: new options)
+* Any change to the semantic of existing functionality, even when current behavior is incorrect.
+* Any large code change that will touch multiple components
+* Any changes to the metrics (metrics endpoint, topic stats, topics internal stats, broker stats, etc.)
+* Any change to the configuration
+
+### When is a PIP *not* required?
+
+* Bug-fixes
+* Simple enhancements that won't affect the APIs or the semantic
+* Small documentation changes
+* Small website changes
+* Build scripts changes (except: a complete rewrite)
+
+### Who can create a PIP?
+
+Any person willing to contribute to the Apache Pulsar project is welcome to create a PIP.
+
+## How does the PIP process work?
+
+A PIP proposal can be in these states:
+1. **DRAFT**: (Optional) This might be used for contributors to collaborate and to seek feedback on an incomplete version of the proposal.
+
+2. **DISCUSSION**: The proposal has been submitted to the community for feedback and approval.
+
+3. **ACCEPTED**: The proposal has been accepted by the Pulsar project.
+
+4. **REJECTED**: The proposal has not been accepted by the Pulsar project.
+
+5. **IMPLEMENTED**: The implementation of the proposed changes have been completed and everything has been merged.
+
+6. **RELEASED**: The proposed changes have been included in an official
+   Apache Pulsar release.
+
+
+The process works in the following way:
+
+1. Fork https://github.com/apache/pulsar repository (Using the fork button on GitHub).
+2. Clone the repository, and on it, copy the file `pip/TEMPLATE.md` and name it `pip-xxx.md`. The number `xxx` should be the next sequential number after the last contributed PIP. You view the list of contributed PIPs (at any status) as a list of Pull Requests having a title starting with `[pip][design] PIP-`. Use the link [here](https://github.com/apache/pulsar/pulls?q=is%3Apr+title%3A%22%5Bpip%5D%5Bdesign%5D+PIP-%22) as shortcut.
+3. Write the proposal following the section outlined by the template and the explanation for each section in the comment it contains (you can delete the comment once done).
+   * If you need diagrams, avoid attaching large files. You can use [MermaidJS](https://mermaid.js.org/) as simple language to describe many types of diagrams. 
+4. Create GitHub Pull request (PR). The PR title should be `[pip][design] PIP-xxx: {title}`, where the `xxx` match the number given in previous step (file-name). Replace `{title}` with a short title to your proposal.
+5. The author(s) will email the dev@pulsar.apache.org mailing list to kick off a discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion will happen in broader context either on the mailing list or as general comments on the PR. Many of the discussion items will be on particular aspect of the proposal, hence they should be as comments in the PR to specific lines in the proposal file.
+6. Update file with a link to the discussion on the mailing. You can obtain it from [Apache Pony Mail](https://lists.apache.org/list.html?dev@pulsar.apache.org).
+7. Based on the discussion and feedback, some changes might be applied by authors to the text of the proposal. They will be applied as extra commits, making it easier to track the changes.
+8. Once some consensus is reached, there will be a vote to formally approve the proposal. The vote will be held on the dev@pulsar.apache.org mailing list, by
+   sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}`.
+   Make sure to update the PIP with a link to the vote. You can obtain it from [Apache Pony Mail](https://lists.apache.org/list.html?dev@pulsar.apache.org). 
+   Everyone is welcome to vote on the proposal, though only the vote of the PMC members will be considered binding.
+   It is required to have a lazy majority of at least 3 binding +1s votes.
+   The vote should stay open for at least 48 hours.
+9. When the vote is closed, if the outcome is positive, ask a PMC member (using voting thread on mailing list) to merge the PR.
+10. If the outcome is negative, please close the PR (with a small comment that the close is a result of a vote).
+
+All the future implementation Pull Requests that will be created, should always reference the PIP-XXX in the commit log message and the PR title.
+It is advised to create a master GitHub issue to formulate the execution plan and track its progress.
+
+## List of PIPs
+
+### Historical PIPs
+You can the view list of PIPs previously managed by GitHub wiki or GitHub issues [here](https://github.com/apache/pulsar/wiki#pulsar-improvement-proposals)
+
+### List of PIPs
+1. You can view all PIPs (besides the historical ones) as the list of Pull Requests having title starting with `[pip][design] PIP-`. Here is the [link](https://github.com/apache/pulsar/pulls?q=is%3Apr+title%3A%22%5Bpip%5D%5Bdesign%5D+PIP-%22) for it. 
+   - Merged PR means the PIP was accepted.
+   - Closed PR means the PIP was rejected.
+   - Open PR means the PIP was submitted and is in the process of discussion.
+2. You can also take a look at the file in the `pip` folder. Each one is an approved PIP.
\ No newline at end of file
diff --git a/pip/TEMPLATE.md b/pip/TEMPLATE.md
new file mode 100644
index 0000000000000..6f907eef7e8e9
--- /dev/null
+++ b/pip/TEMPLATE.md
@@ -0,0 +1,163 @@
+
+
+# Background knowledge
+
+
+
+# Motivation
+
+
+
+# Goals
+
+## In Scope
+
+
+
+## Out of Scope
+
+
+
+
+# High Level Design
+
+
+
+# Detailed Design
+
+## Design & Implementation Details
+
+
+
+## Public-facing Changes
+
+
+
+### Public API
+
+
+### Binary protocol
+
+### Configuration
+
+### CLI
+
+### Metrics
+
+
+
+
+# Monitoring
+
+
+
+# Security Considerations
+
+
+# Backward & Forward Compatability
+
+## Revert
+
+
+
+## Upgrade
+
+
+
+# Alternatives
+
+
+
+# General Notes
+
+# Links
+
+
+* Mailing List discussion thread:
+* Mailing List voting thread:
diff --git a/wiki/proposals/PIP-template.md b/wiki/proposals/PIP-template.md
deleted file mode 100644
index 752878a56b519..0000000000000
--- a/wiki/proposals/PIP-template.md
+++ /dev/null
@@ -1,91 +0,0 @@
-# Pulsar Improvement Proposal Template
-
-* **Status**: "one of ['Under Discussion', 'Accepted', 'Adopted', 'Rejected']"
-* **Author**: (Names)
-* **Pull Request**: (Link to the main pull request to resolve this PIP)
-* **Mailing List discussion**: (Link to the mailing list discussion)
-* **Release**: (Which release include this PIP)
-
-### Motivation
-
-_Describe the problems you are trying to solve_
-
-### Public Interfaces
-
-_Briefly list any new interfaces that will be introduced as part of this proposal or any existing interfaces that will be removed or changed. The purpose of this section is to concisely call out the public contract that will come along with this feature._
-
-A public interface is any change to the following:
-
-- Data format, Metadata format
-- The wire protocol and API behavior
-- Any class in the public packages
-- Monitoring
-- Command-line tools and arguments
-- Configuration settings
-- Anything else that will likely break existing users in some way when they upgrade
-
-### Proposed Changes
-
-_Describe the new thing you want to do in appropriate detail. This may be fairly extensive and have large subsections of its own. Or it may be a few sentences. Use judgment based on the scope of the change._
-
-### Compatibility, Deprecation, and Migration Plan
-
-- What impact (if any) will there be on existing users?
-- If we are changing behavior how will we phase out the older behavior?
-- If we need special migration tools, describe them here.
-- When will we remove the existing behavior?
-
-### Test Plan
-
-_Describe in few sentences how the BP will be tested. We are mostly interested in system tests (since unit-tests are specific to implementation details). How will we know that the implementation works as expected? How will we know nothing broke?_
-
-### Rejected Alternatives
-
-_If there are alternative ways of accomplishing the same thing, what were they? The purpose of this section is to motivate why the design is the way it is and not some other way._
-
----
-
-```
-* **Status**: "one of ['Under Discussion', 'Accepted', 'Adopted', 'Rejected']"
-* **Author**: (Names)
-* **Pull Request**: (Link to the main pull request to resolve this PIP)
-* **Mailing List discussion**: (Link to the mailing list discussion)
-* **Release**: (Which release include this PIP)
-
-### Motivation
-
-_Describe the problems you are trying to solve_
-
-### Public Interfaces
-
-_Briefly list any new interfaces that will be introduced as part of this proposal or any existing interfaces that will be removed or changed. The purpose of this section is to concisely call out the public contract that will come along with this feature._
-
-A public interface is any change to the following:
-
-- Data format, Metadata format
-- The wire protocol and API behavior
-- Any class in the public packages
-- Monitoring
-- Command-line tools and arguments
-- Configuration settings
-- Anything else that will likely break existing users in some way when they upgrade
-
-### Proposed Changes
-
-_Describe the new thing you want to do in appropriate detail. This may be fairly extensive and have large subsections of its own. Or it may be a few sentences. Use judgment based on the scope of the change._
-
-### Compatibility, Deprecation, and Migration Plan
-
-- What impact (if any) will there be on existing users? 
-- If we are changing behavior how will we phase out the older behavior? 
-- If we need special migration tools, describe them here.
-- When will we remove the existing behavior?
-
-### Test Plan
-
-_Describe in few sentences how the BP will be tested. We are mostly interested in system tests (since unit-tests are specific to implementation details). How will we know that the implementation works as expected? How will we know nothing broke?_
-
-### Rejected Alternatives
-
-_If there are alternative ways of accomplishing the same thing, what were they? The purpose of this section is to motivate why the design is the way it is and not some other way._
-```
\ No newline at end of file
diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md
deleted file mode 100644
index fc68905726eb8..0000000000000
--- a/wiki/proposals/PIP.md
+++ /dev/null
@@ -1,120 +0,0 @@
-# Pulsar Improvement Proposal (PIP)
-
-## What is a PIP?
-
-The PIP is a "Pulsar Improvement Proposal" and it's the mechanism used to
-propose changes to the Apache Pulsar codebases.
-
-The changes might be in terms of new features, large code refactoring, changes
-to APIs.
-
-In practical terms, the PIP defines a process in which developers can submit
-a design doc, receive feedback and get the "go ahead" to execute.
-
-## What is the goal of a PIP?
-
-There are several goals for the PIP process:
-
-1. Ensure community technical discussion of major changes to the Apache Pulsar
-   codebase
-
-2. Provide clear and thorough design documentation of the proposed changes.
-   Make sure every Pulsar developer will have enough context to effectively
-   perform a code review of the Pull Requests.
-
-3. Use the PIP document to serve as the starting base on which to create the
-   documentation for the new feature.
-
-4. Have greater scrutiny to changes are affecting the public APIs to reduce
-   chances of introducing breaking changes or APIs that are not expressing
-   an ideal semantic.
-
-
-It is not a goal for PIP to add undue process or slow-down the development.
-
-## When is a PIP required?
-
-* Any new feature for Pulsar brokers or client
-* Any change to the public APIs (Client APIs, REST APIs, Plugin APIs)
-* Any change to the wire protocol APIs
-* Any change to the API of Pulsar CLI tools (eg: new options)
-* Any change to the semantic of existing functionality, even when current
-  behavior is incorrect.
-* Any large code change that will touch multiple components
-* Any changes to the metrics (metrics endpoint, topic stats, topics internal stats, broker stats, etc.)
-
-## When is a PIP *not* required?
-
-* Bug-fixes
-* Simple enhancements that won't affect the APIs or the semantic
-* Documentation changes
-* Website changes
-* Build scripts changes (except: a complete rewrite)
-
-## Who can create a PIP?
-
-Any person willing to contribute to the Apache Pulsar project is welcome to
-create a PIP.
-
-## How does the PIP process work?
-
-A PIP proposal can be in these states:
-1. **DRAFT**: (Optional) This might be used for contributors to collaborate and
-   to seek feedback on an incomplete version of the proposal.
-
-2. **DISCUSSION**: The proposal has been submitted to the community for
-   feedback and approval.
-
-3. **ACCEPTED**: The proposal has been accepted by the Pulsar project.
-
-4. **REJECTED**: The proposal has not been accepted by the Pulsar project.
-
-5. **IMPLEMENTED**: The implementation of the proposed changes have been
-   completed and everything has been merged.
-
-5. **RELEASED**: The proposed changes have been included in an official
-   Apache Pulsar release.
-
-The process works in the following way:
-
-1. The author(s) of the proposal will create a GitHub issue ticket choosing the
-   template for PIP proposals. The issue title should be "PIP-xxx: title", where
-   the "xxx" number should be chosen to be the next number from the existing PIP 
-   issues, listed [here](https://github.com/apache/pulsar/issues?q=is%3Aissue+label%3APIP+)
-2. The author(s) will send a note to the dev@pulsar.apache.org mailing list
-   to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion
-   need to happen in the mailing list. Please avoid discussing it using
-   GitHub comments in the PIP GitHub issue, as it creates two tracks 
-   of feedback.
-3. Based on the discussion and feedback, some changes might be applied by
-   authors to the text of the proposal.
-4. Once some consensus is reached, there will be a vote to formally approve
-   the proposal.
-   The vote will be held on the dev@pulsar.apache.org mailing list, by
-   sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}".
-   Everyone is welcome to vote on the proposal, though only the the vote of the PMC 
-   members will be considered binding.
-   It is required to have a lazy majority of at least 3 binding +1s votes.
-   The vote should stay open for at least 48 hours.
-5. When the vote is closed, if the outcome is positive, the state of the
-   proposal is updated, and the Pull Requests associated with this proposal can
-   start to get merged into the master branch.
-
-All the Pull Requests that are created, should always reference the
-PIP-XXX in the
-commit log message and the PR title.
-
-## Labels of a PIP
-
-In addition to its current state, the GitHub issue for the PIP will also be
-tagged with other labels.
-
-Some examples:
-* Execution status: In progress, Completed, Need Help, ...
-* Targeted Pulsar release: 2.9, 2.10, ...
-
-
-## Template for a PIP design doc
-
-Read [the template file](/.github/ISSUE_TEMPLATE/pip.md).
-

From 70ea994316e5111bfdf995e8164f7f3a7ba0bd2b Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Tue, 30 May 2023 16:48:36 +0300
Subject: [PATCH 260/280] [improve][misc]  Deprecate PIP issue type (#20439)

---
 .github/ISSUE_TEMPLATE/pip.md    | 165 +------------------------------
 .github/PULL_REQUEST_TEMPLATE.md |   2 +-
 2 files changed, 5 insertions(+), 162 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/pip.md b/.github/ISSUE_TEMPLATE/pip.md
index 520826f8b1ceb..4274f2f637936 100644
--- a/.github/ISSUE_TEMPLATE/pip.md
+++ b/.github/ISSUE_TEMPLATE/pip.md
@@ -1,167 +1,10 @@
 ---
 name: PIP
-about: Submit a Pulsar Improvement Proposal (PIP)
+about: [DEPRECATED. see 'pip' folder] Submit a Pulsar Improvement Proposal (PIP)
+description: DEPRECATED. Please read `pip/README.md`
 title: 'PIP-XYZ: '
 labels: PIP
 ---
 
-
-
-# Background knowledge
-
-
-
-# Motivation
-
-
-
-# Goals
-
-## In Scope
-
-
-
-## Out of Scope
-
-
-
-
-# High Level Design
-
-
-
-# Detailed Design
-
-## Design & Implementation Details
-
-
-
-## Public-facing Changes
-
-
-
-### Public API 
-
-
-### Binary protocol
-
-### Configuration
-
-### CLI
-
-### Metrics
-
-
-
-
-# Monitoring
-
-
-
-# Security Considerations
-
-
-# Backward & Forward Compatability
-
-## Revert
-
-
-
-## Upgrade
-
-
-
-# Alternatives
-
-
-
-# General Notes
-
-# Links
-
-
-* Mailing List discussion thread:
-* Mailing List voting thread:
+We have stopped using GitHub issues to hold the PIP content.
+Please read [here](https://github.com/apache/pulsar/blob/master/pip/README.md) how to submit a PIP
\ No newline at end of file
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 01ac26570b2d1..fdb8459024b1f 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -24,7 +24,7 @@ Master Issue: #xyz
 
 PIP: #xyz 
 
-
+
 
 ### Motivation
 

From 5796842dd2a1dbec11101d14a742c729ef22ed5a Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Tue, 30 May 2023 17:05:16 +0300
Subject: [PATCH 261/280] [fix][misc] Fix invalid PIP GitHub issue template
 (#20440)

---
 .github/ISSUE_TEMPLATE/pip.md | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/pip.md b/.github/ISSUE_TEMPLATE/pip.md
index 4274f2f637936..e0bc586669493 100644
--- a/.github/ISSUE_TEMPLATE/pip.md
+++ b/.github/ISSUE_TEMPLATE/pip.md
@@ -1,10 +1,9 @@
 ---
 name: PIP
-about: [DEPRECATED. see 'pip' folder] Submit a Pulsar Improvement Proposal (PIP)
-description: DEPRECATED. Please read `pip/README.md`
-title: 'PIP-XYZ: '
+about: '[DEPRECATED. see pip folder] Submit a Pulsar Improvement Proposal (PIP)'
+title: 'DEPRECATED - Read https://github.com/apache/pulsar/blob/master/pip/README.md'
 labels: PIP
 ---
 
 We have stopped using GitHub issues to hold the PIP content.
-Please read [here](https://github.com/apache/pulsar/blob/master/pip/README.md) how to submit a PIP
\ No newline at end of file
+Please read [here](https://github.com/apache/pulsar/blob/master/pip/README.md) how to submit a PIP

From ad75cc87e045ffca56a44f69e189f2e080373cdb Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Wed, 31 May 2023 09:45:53 +0800
Subject: [PATCH 262/280] [improve][meta] Support set metadata size threshold
 for compression. (#19561)

---
 .../mledger/ManagedLedgerFactoryConfig.java   | 22 +++++
 .../mledger/MetadataCompressionConfig.java    | 56 +++++++++++++
 .../impl/ManagedLedgerFactoryImpl.java        |  5 +-
 .../mledger/impl/MetaStoreImpl.java           | 81 +++++++++----------
 .../impl/ManagedCursorInfoMetadataTest.java   | 57 +++++++++++--
 .../impl/ManagedLedgerInfoMetadataTest.java   | 63 +++++++++++++--
 .../pulsar/broker/ServiceConfiguration.java   | 13 +++
 .../broker/ManagedLedgerClientFactory.java    |  4 +
 8 files changed, 244 insertions(+), 57 deletions(-)
 create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
index 5aa4e8374d73a..8a4b4d4013f8f 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
@@ -92,8 +92,30 @@ public class ManagedLedgerFactoryConfig {
      */
     private String managedLedgerInfoCompressionType = MLDataFormats.CompressionType.NONE.name();
 
+    /**
+     * ManagedLedgerInfo compression threshold. If the origin metadata size below configuration.
+     * compression will not apply.
+     */
+    private long managedLedgerInfoCompressionThresholdInBytes = 0;
+
     /**
      * ManagedCursorInfo compression type. If the compression type is null or invalid, don't compress data.
      */
     private String managedCursorInfoCompressionType = MLDataFormats.CompressionType.NONE.name();
+
+    /**
+     * ManagedCursorInfo compression threshold. If the origin metadata size below configuration.
+     * compression will not apply.
+     */
+    private long managedCursorInfoCompressionThresholdInBytes = 0;
+
+    public MetadataCompressionConfig getCompressionConfigForManagedLedgerInfo() {
+        return new MetadataCompressionConfig(managedLedgerInfoCompressionType,
+                managedLedgerInfoCompressionThresholdInBytes);
+    }
+
+    public MetadataCompressionConfig getCompressionConfigForManagedCursorInfo() {
+        return new MetadataCompressionConfig(managedCursorInfoCompressionType,
+                managedCursorInfoCompressionThresholdInBytes);
+    }
 }
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java
new file mode 100644
index 0000000000000..601c270ab7680
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.ToString;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.commons.lang.StringUtils;
+
+@Data
+@AllArgsConstructor
+@ToString
+public class MetadataCompressionConfig {
+    MLDataFormats.CompressionType compressionType;
+    long compressSizeThresholdInBytes;
+
+    public MetadataCompressionConfig(String compressionType) throws IllegalArgumentException {
+        this(compressionType, 0);
+    }
+
+    public MetadataCompressionConfig(String compressionType, long compressThreshold) throws IllegalArgumentException {
+        this.compressionType = parseCompressionType(compressionType);
+        this.compressSizeThresholdInBytes = compressThreshold;
+    }
+
+    public static MetadataCompressionConfig noCompression =
+            new MetadataCompressionConfig(MLDataFormats.CompressionType.NONE, 0);
+
+    private MLDataFormats.CompressionType parseCompressionType(String value) throws IllegalArgumentException {
+        if (StringUtils.isEmpty(value)) {
+            return MLDataFormats.CompressionType.NONE;
+        }
+
+        MLDataFormats.CompressionType compressionType;
+        compressionType = MLDataFormats.CompressionType.valueOf(value);
+
+        return compressionType;
+    }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index 9f3fe9bb0c4a7..f076f68299dd0 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -193,8 +193,9 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
         this.bookkeeperFactory = bookKeeperGroupFactory;
         this.isBookkeeperManaged = isBookkeeperManaged;
         this.metadataStore = metadataStore;
-        this.store = new MetaStoreImpl(metadataStore, scheduledExecutor, config.getManagedLedgerInfoCompressionType(),
-                config.getManagedCursorInfoCompressionType());
+        this.store = new MetaStoreImpl(metadataStore, scheduledExecutor,
+                config.getCompressionConfigForManagedLedgerInfo(),
+                config.getCompressionConfigForManagedCursorInfo());
         this.config = config;
         this.mbean = new ManagedLedgerFactoryMBeanImpl(this);
         this.entryCacheManager = new RangeEntryCacheManagerImpl(this);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index 1bc2d2b04bed0..d9269ec83b179 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -37,11 +37,11 @@
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
 import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException;
+import org.apache.bookkeeper.mledger.MetadataCompressionConfig;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo;
-import org.apache.commons.lang.StringUtils;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.compression.CompressionCodec;
 import org.apache.pulsar.common.compression.CompressionCodecProvider;
@@ -62,50 +62,35 @@ public class MetaStoreImpl implements MetaStore, Consumer {
     private final OrderedExecutor executor;
 
     private static final int MAGIC_MANAGED_INFO_METADATA = 0x4778; // 0100 0111 0111 1000
-    private final CompressionType ledgerInfoCompressionType;
-    private final CompressionType cursorInfoCompressionType;
+    private final MetadataCompressionConfig ledgerInfoCompressionConfig;
+    private final MetadataCompressionConfig cursorInfoCompressionConfig;
 
     private final Map> managedLedgerInfoUpdateCallbackMap;
 
     public MetaStoreImpl(MetadataStore store, OrderedExecutor executor) {
         this.store = store;
         this.executor = executor;
-        this.ledgerInfoCompressionType = CompressionType.NONE;
-        this.cursorInfoCompressionType = CompressionType.NONE;
+        this.ledgerInfoCompressionConfig = MetadataCompressionConfig.noCompression;
+        this.cursorInfoCompressionConfig = MetadataCompressionConfig.noCompression;
         managedLedgerInfoUpdateCallbackMap = new ConcurrentHashMap<>();
         if (store != null) {
             store.registerListener(this);
         }
     }
 
-    public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String ledgerInfoCompressionType,
-                         String cursorInfoCompressionType) {
+    public MetaStoreImpl(MetadataStore store, OrderedExecutor executor,
+                         MetadataCompressionConfig ledgerInfoCompressionConfig,
+                         MetadataCompressionConfig cursorInfoCompressionConfig) {
         this.store = store;
         this.executor = executor;
-        this.ledgerInfoCompressionType = parseCompressionType(ledgerInfoCompressionType);
-        this.cursorInfoCompressionType = parseCompressionType(cursorInfoCompressionType);
+        this.ledgerInfoCompressionConfig = ledgerInfoCompressionConfig;
+        this.cursorInfoCompressionConfig = cursorInfoCompressionConfig;
         managedLedgerInfoUpdateCallbackMap = new ConcurrentHashMap<>();
         if (store != null) {
             store.registerListener(this);
         }
     }
 
-    private CompressionType parseCompressionType(String value) {
-        if (StringUtils.isEmpty(value)) {
-            return CompressionType.NONE;
-        }
-
-        CompressionType compressionType;
-        try {
-            compressionType = CompressionType.valueOf(value);
-        } catch (Exception e) {
-            log.error("Failed to get compression type {} error msg: {}.", value, e.getMessage());
-            throw e;
-        }
-
-        return compressionType;
-    }
-
     @Override
     public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map properties,
             MetaStoreCallback callback) {
@@ -421,29 +406,43 @@ private static MetaStoreException getException(Throwable t) {
     }
 
     public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) {
-        if (ledgerInfoCompressionType.equals(CompressionType.NONE)) {
+        CompressionType compressionType = ledgerInfoCompressionConfig.getCompressionType();
+        if (compressionType.equals(CompressionType.NONE)) {
             return managedLedgerInfo.toByteArray();
         }
-        MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata
-                .newBuilder()
-                .setCompressionType(ledgerInfoCompressionType)
-                .setUncompressedSize(managedLedgerInfo.getSerializedSize())
-                .build();
-        return compressManagedInfo(managedLedgerInfo.toByteArray(), mlInfoMetadata.toByteArray(),
-                mlInfoMetadata.getSerializedSize(), ledgerInfoCompressionType);
+
+        int uncompressedSize = managedLedgerInfo.getSerializedSize();
+        if (uncompressedSize > ledgerInfoCompressionConfig.getCompressSizeThresholdInBytes()) {
+            MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata
+                    .newBuilder()
+                    .setCompressionType(compressionType)
+                    .setUncompressedSize(uncompressedSize)
+                    .build();
+            return compressManagedInfo(managedLedgerInfo.toByteArray(), mlInfoMetadata.toByteArray(),
+                    mlInfoMetadata.getSerializedSize(), compressionType);
+        }
+
+        return managedLedgerInfo.toByteArray();
     }
 
     public byte[] compressCursorInfo(ManagedCursorInfo managedCursorInfo) {
-        if (cursorInfoCompressionType.equals(CompressionType.NONE)) {
+        CompressionType compressionType = cursorInfoCompressionConfig.getCompressionType();
+        if (compressionType.equals(CompressionType.NONE)) {
             return managedCursorInfo.toByteArray();
         }
-        MLDataFormats.ManagedCursorInfoMetadata metadata = MLDataFormats.ManagedCursorInfoMetadata
-                .newBuilder()
-                .setCompressionType(cursorInfoCompressionType)
-                .setUncompressedSize(managedCursorInfo.getSerializedSize())
-                .build();
-        return compressManagedInfo(managedCursorInfo.toByteArray(), metadata.toByteArray(),
-                metadata.getSerializedSize(), cursorInfoCompressionType);
+
+        int uncompressedSize = managedCursorInfo.getSerializedSize();
+        if (uncompressedSize > cursorInfoCompressionConfig.getCompressSizeThresholdInBytes()) {
+            MLDataFormats.ManagedCursorInfoMetadata metadata = MLDataFormats.ManagedCursorInfoMetadata
+                    .newBuilder()
+                    .setCompressionType(compressionType)
+                    .setUncompressedSize(uncompressedSize)
+                    .build();
+            return compressManagedInfo(managedCursorInfo.toByteArray(), metadata.toByteArray(),
+                    metadata.getSerializedSize(), compressionType);
+        }
+
+        return managedCursorInfo.toByteArray();
     }
 
     public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws InvalidProtocolBufferException {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java
index 08d8fd939a01d..70ba4b543ec09 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java
@@ -19,11 +19,13 @@
 package org.apache.bookkeeper.mledger.impl;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.expectThrows;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.MetadataCompressionConfig;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
 import org.apache.pulsar.common.api.proto.CompressionType;
 import org.testng.annotations.DataProvider;
@@ -49,16 +51,14 @@ private Object[][] compressionTypeProvider() {
         };
     }
 
-    @Test(dataProvider = "compressionTypeProvider")
-    public void testEncodeAndDecode(String compressionType) throws IOException {
-        long ledgerId = 10000;
+    private MLDataFormats.ManagedCursorInfo.Builder generateManagedCursorInfo(long ledgerId, int positionNumber) {
         MLDataFormats.ManagedCursorInfo.Builder builder = MLDataFormats.ManagedCursorInfo.newBuilder();
 
         builder.setCursorsLedgerId(ledgerId);
         builder.setMarkDeleteLedgerId(ledgerId);
 
         List batchedEntryDeletionIndexInfos = new ArrayList<>();
-        for (int i = 0; i < 1000; i++) {
+        for (int i = 0; i < positionNumber; i++) {
             MLDataFormats.NestedPositionInfo nestedPositionInfo = MLDataFormats.NestedPositionInfo.newBuilder()
                     .setEntryId(i).setLedgerId(i).build();
             MLDataFormats.BatchedEntryDeletionIndexInfo batchedEntryDeletionIndexInfo = MLDataFormats
@@ -67,17 +67,24 @@ public void testEncodeAndDecode(String compressionType) throws IOException {
         }
         builder.addAllBatchedEntryDeletionIndexInfo(batchedEntryDeletionIndexInfos);
 
+        return builder;
+    }
+
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testEncodeAndDecode(String compressionType) throws IOException {
+        long ledgerId = 10000;
+        MLDataFormats.ManagedCursorInfo.Builder builder = generateManagedCursorInfo(ledgerId, 1000);
         MetaStoreImpl metaStore;
         if (INVALID_TYPE.equals(compressionType)) {
             IllegalArgumentException compressionTypeEx = expectThrows(IllegalArgumentException.class, () -> {
-                new MetaStoreImpl(null, null, null, compressionType);
+                new MetaStoreImpl(null, null, null, new MetadataCompressionConfig(compressionType));
             });
             assertEquals(compressionTypeEx.getMessage(),
                     "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType."
                             + compressionType);
             return;
         } else {
-            metaStore = new MetaStoreImpl(null, null, null, compressionType);
+            metaStore = new MetaStoreImpl(null, null, null, new MetadataCompressionConfig(compressionType));
         }
 
         MLDataFormats.ManagedCursorInfo managedCursorInfo = builder.build();
@@ -93,4 +100,42 @@ public void testEncodeAndDecode(String compressionType) throws IOException {
         MLDataFormats.ManagedCursorInfo info2 = metaStore.parseManagedCursorInfo(managedCursorInfo.toByteArray());
         assertEquals(info1, info2);
     }
+
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testCompressionThreshold(String compressionType) throws IOException {
+        int compressThreshold = 512;
+
+        long ledgerId = 10000;
+        // should not compress
+        MLDataFormats.ManagedCursorInfo smallInfo = generateManagedCursorInfo(ledgerId, 1).build();
+        assertTrue(smallInfo.getSerializedSize() < compressThreshold);
+
+        // should compress
+        MLDataFormats.ManagedCursorInfo bigInfo = generateManagedCursorInfo(ledgerId, 1000).build();
+        assertTrue(bigInfo.getSerializedSize() > compressThreshold);
+
+        MetaStoreImpl metaStore;
+        if (INVALID_TYPE.equals(compressionType)) {
+            IllegalArgumentException compressionTypeEx = expectThrows(IllegalArgumentException.class, () -> {
+                new MetaStoreImpl(null, null, null,
+                        new MetadataCompressionConfig(compressionType, compressThreshold));
+            });
+            assertEquals(compressionTypeEx.getMessage(),
+                    "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType."
+                            + compressionType);
+            return;
+        } else {
+            metaStore = new MetaStoreImpl(null, null, null,
+                    new MetadataCompressionConfig(compressionType, compressThreshold));
+        }
+
+        byte[] compressionBytes = metaStore.compressCursorInfo(smallInfo);
+        // not compressed
+        assertEquals(compressionBytes.length, smallInfo.getSerializedSize());
+
+
+        byte[] compressionBigBytes = metaStore.compressCursorInfo(bigInfo);
+        // compressed
+        assertTrue(compressionBigBytes.length != smallInfo.getSerializedSize());
+    }
 }
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java
index 7ddf6541c9a39..6e1f447225e53 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.MetadataCompressionConfig;
 import org.apache.bookkeeper.mledger.offload.OffloadUtils;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
 import org.apache.commons.lang3.RandomUtils;
@@ -33,6 +34,8 @@
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 
 /**
  * ManagedLedgerInfo metadata test.
@@ -53,11 +56,9 @@ private Object[][] compressionTypeProvider() {
         };
     }
 
-    @Test(dataProvider = "compressionTypeProvider")
-    public void testEncodeAndDecode(String compressionType) throws IOException {
-        long ledgerId = 10000;
+    private MLDataFormats.ManagedLedgerInfo.Builder generateManagedLedgerInfo(long ledgerId, int ledgerInfoNumber) {
         List ledgerInfoList = new ArrayList<>();
-        for (int i = 0; i < 100; i++) {
+        for (int i = 0; i < ledgerInfoNumber; i++) {
             MLDataFormats.ManagedLedgerInfo.LedgerInfo.Builder builder = MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder();
             builder.setLedgerId(ledgerId);
             builder.setEntries(RandomUtils.nextInt());
@@ -84,13 +85,18 @@ public void testEncodeAndDecode(String compressionType) throws IOException {
             ledgerId ++;
         }
 
-        MLDataFormats.ManagedLedgerInfo managedLedgerInfo = MLDataFormats.ManagedLedgerInfo.newBuilder()
-                .addAllLedgerInfo(ledgerInfoList)
-                .build();
+        return MLDataFormats.ManagedLedgerInfo.newBuilder()
+                .addAllLedgerInfo(ledgerInfoList);
+    }
+
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testEncodeAndDecode(String compressionType) throws IOException {
+        long ledgerId = 10000;
+        MLDataFormats.ManagedLedgerInfo managedLedgerInfo = generateManagedLedgerInfo(ledgerId,100).build();
 
         MetaStoreImpl metaStore;
         try {
-            metaStore = new MetaStoreImpl(null, null, compressionType, null);
+            metaStore = new MetaStoreImpl(null, null, new MetadataCompressionConfig(compressionType), null);
             if ("INVALID_TYPE".equals(compressionType)) {
                 Assert.fail("The managedLedgerInfo compression type is invalid, should fail.");
             }
@@ -126,4 +132,45 @@ public void testParseEmptyData() throws InvalidProtocolBufferException {
         Assert.assertEquals(managedLedgerInfo.toString(), "");
     }
 
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testCompressionThreshold(String compressionType) {
+        long ledgerId = 10000;
+        int compressThreshold = 512;
+
+        // should not compress
+        MLDataFormats.ManagedLedgerInfo smallInfo = generateManagedLedgerInfo(ledgerId, 0).build();
+        assertTrue(smallInfo.getSerializedSize() < compressThreshold);
+
+        // should compress
+        MLDataFormats.ManagedLedgerInfo bigInfo = generateManagedLedgerInfo(ledgerId, 1000).build();
+        assertTrue(bigInfo.getSerializedSize() > compressThreshold);
+
+        MLDataFormats.ManagedLedgerInfo managedLedgerInfo = generateManagedLedgerInfo(ledgerId,100).build();
+
+        MetaStoreImpl metaStore;
+        try {
+            MetadataCompressionConfig metadataCompressionConfig =
+                    new MetadataCompressionConfig(compressionType, compressThreshold);
+            metaStore = new MetaStoreImpl(null, null, metadataCompressionConfig, null);
+            if ("INVALID_TYPE".equals(compressionType)) {
+                Assert.fail("The managedLedgerInfo compression type is invalid, should fail.");
+            }
+        } catch (Exception e) {
+            if ("INVALID_TYPE".equals(compressionType)) {
+                Assert.assertEquals(e.getClass(), IllegalArgumentException.class);
+                Assert.assertEquals(
+                        "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType."
+                                + compressionType, e.getMessage());
+                return;
+            } else {
+                throw e;
+            }
+        }
+
+        byte[] compressionBytes = metaStore.compressLedgerInfo(smallInfo);
+        assertEquals(compressionBytes.length, smallInfo.getSerializedSize());
+
+        byte[] compressionBytesBig = metaStore.compressLedgerInfo(bigInfo);
+        assertTrue(compressionBytesBig.length !=smallInfo.getSerializedSize());
+    }
 }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 5dc15bd6a4dcf..c9bb4b86ba780 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2135,12 +2135,25 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
                     + "If value is invalid or NONE, then save the ManagedLedgerInfo bytes data directly.")
     private String managedLedgerInfoCompressionType = "NONE";
 
+    @FieldContext(category = CATEGORY_STORAGE_ML,
+            doc = "ManagedLedgerInfo compression size threshold (bytes), "
+                    + "only compress metadata when origin size more then this value.\n"
+                    + "0 means compression will always apply.\n")
+    private long managedLedgerInfoCompressionThresholdInBytes = 16 * 1024;
+
 
     @FieldContext(category = CATEGORY_STORAGE_ML,
             doc = "ManagedCursorInfo compression type, option values (NONE, LZ4, ZLIB, ZSTD, SNAPPY). \n"
                     + "If value is NONE, then save the ManagedCursorInfo bytes data directly.")
     private String managedCursorInfoCompressionType = "NONE";
 
+
+    @FieldContext(category = CATEGORY_STORAGE_ML,
+            doc = "ManagedCursorInfo compression size threshold (bytes), "
+                    + "only compress metadata when origin size more then this value.\n"
+                    + "0 means compression will always apply.\n")
+    private long managedCursorInfoCompressionThresholdInBytes = 16 * 1024;
+
     @FieldContext(
             dynamic = true,
             category = CATEGORY_STORAGE_ML,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
index b16b9a7dd4833..d86649abd3c93 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
@@ -70,8 +70,12 @@ public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadata
         managedLedgerFactoryConfig.setTraceTaskExecution(conf.isManagedLedgerTraceTaskExecution());
         managedLedgerFactoryConfig.setCursorPositionFlushSeconds(conf.getManagedLedgerCursorPositionFlushSeconds());
         managedLedgerFactoryConfig.setManagedLedgerInfoCompressionType(conf.getManagedLedgerInfoCompressionType());
+        managedLedgerFactoryConfig.setManagedLedgerInfoCompressionThresholdInBytes(
+                conf.getManagedLedgerInfoCompressionThresholdInBytes());
         managedLedgerFactoryConfig.setStatsPeriodSeconds(conf.getManagedLedgerStatsPeriodSeconds());
         managedLedgerFactoryConfig.setManagedCursorInfoCompressionType(conf.getManagedCursorInfoCompressionType());
+        managedLedgerFactoryConfig.setManagedCursorInfoCompressionThresholdInBytes(
+                conf.getManagedCursorInfoCompressionThresholdInBytes());
 
         Configuration configuration = new ClientConfiguration();
         if (conf.isBookkeeperClientExposeStatsToPrometheus()) {

From da2a1482a7dbd57bfc7c130315db249fe53e0260 Mon Sep 17 00:00:00 2001
From: tison 
Date: Wed, 31 May 2023 09:49:18 +0800
Subject: [PATCH 263/280] [fix][cli] Fulfill add-opens to function-localrunner
 also (#20417)

Signed-off-by: tison 
---
 bin/bookkeeper             |  2 +-
 bin/function-localrunner   | 14 ++++++++------
 bin/pulsar                 |  2 +-
 bin/pulsar-admin-common.sh |  2 +-
 bin/pulsar-perf            |  2 +-
 5 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/bin/bookkeeper b/bin/bookkeeper
index fb516a98acdc2..0cc07dd49aba5 100755
--- a/bin/bookkeeper
+++ b/bin/bookkeeper
@@ -168,7 +168,7 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $BOOKIE_LOG_CONF`"
 # Allow Netty to use reflection access
 OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
 
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then
diff --git a/bin/function-localrunner b/bin/function-localrunner
index 45a37cb306794..2e0aa0f6dffe2 100755
--- a/bin/function-localrunner
+++ b/bin/function-localrunner
@@ -40,13 +40,15 @@ PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
 PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}
 
 # Garbage collection log.
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
-# java version has space, use [[ -n $PARAM ]] to judge if variable exists
-if [[ -n "$IS_JAVA_8" ]]; then
-  PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"}
-else
-# After jdk 9, gc log param should config like this. Ignoring version less than jdk 8
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
+if [[ -z "$IS_JAVA_8" ]]; then
+  # >= JDK 9
   PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xlog:gc:logs/pulsar_gc_%p.log:time,uptime:filecount=10,filesize=20M"}
+  # '--add-opens' option is not supported in JDK 1.8
+  OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED"
+else
+  # == JDK 1.8
+  PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"}
 fi
 
 # Extra options to be passed to the jvm
diff --git a/bin/pulsar b/bin/pulsar
index e3b22caced52e..20ed1f7f22b0f 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -291,7 +291,7 @@ OPTS="$OPTS -Dzookeeper.clientTcpKeepAlive=true"
 
 # Allow Netty to use reflection access
 OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then
diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh
index 8223ac5b3bf24..8aa21c00f634d 100755
--- a/bin/pulsar-admin-common.sh
+++ b/bin/pulsar-admin-common.sh
@@ -91,7 +91,7 @@ PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
 OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
 OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
 
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then
diff --git a/bin/pulsar-perf b/bin/pulsar-perf
index 47c02bc3d67d5..bdc1dc1ed8b8c 100755
--- a/bin/pulsar-perf
+++ b/bin/pulsar-perf
@@ -134,7 +134,7 @@ PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
 OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true"
 
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then

From f92011762deea79040fb006fab0d0f297fae9e1f Mon Sep 17 00:00:00 2001
From: ken <1647023764@qq.com>
Date: Wed, 31 May 2023 10:10:00 +0800
Subject: [PATCH 264/280] [improve][broker] Change limitStatsLogging config
 default value to true (#20409)

Co-authored-by: fanjianye 
---
 conf/broker.conf                                   |  4 ++--
 conf/standalone.conf                               |  4 ++--
 deployment/terraform-ansible/templates/broker.conf |  4 ++--
 .../apache/pulsar/broker/ServiceConfiguration.java |  2 +-
 .../broker/BookKeeperClientFactoryImplTest.java    | 14 +++++++-------
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index ca118d254f496..22ca71864e9cd 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1078,8 +1078,8 @@ bookkeeperExplicitLacIntervalInMills=0
 # bookkeeperClientExposeStatsToPrometheus=false
 
 # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
-# to limit per_channel_bookie_client metrics. default is false
-# bookkeeperClientLimitStatsLogging=false
+# to limit per_channel_bookie_client metrics. default is true
+# bookkeeperClientLimitStatsLogging=true
 
 ### --- Managed Ledger --- ###
 
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 8d24d5ad88c7f..46e6aed76e42a 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -697,8 +697,8 @@ bookkeeperUseV2WireProtocol=true
 # bookkeeperClientExposeStatsToPrometheus=false
 
 # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
-# to limit per_channel_bookie_client metrics. default is false
-# bookkeeperClientLimitStatsLogging=false
+# to limit per_channel_bookie_client metrics. default is true
+# bookkeeperClientLimitStatsLogging=true
 
 ### --- Managed Ledger --- ###
 
diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
index f42d4c807d5d9..37e512fb35cc6 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -745,8 +745,8 @@ bookkeeperExplicitLacIntervalInMills=0
 # bookkeeperClientExposeStatsToPrometheus=false
 
 # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
-# to limit per_channel_bookie_client metrics. default is false
-# bookkeeperClientLimitStatsLogging=false
+# to limit per_channel_bookie_client metrics. default is true
+# bookkeeperClientLimitStatsLogging=true
 
 ### --- Managed Ledger --- ###
 
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index c9bb4b86ba780..a709e49e3a980 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1820,7 +1820,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
             category = CATEGORY_STORAGE_BK,
             doc = "whether limit per_channel_bookie_client metrics of bookkeeper client stats"
     )
-    private boolean bookkeeperClientLimitStatsLogging = false;
+    private boolean bookkeeperClientLimitStatsLogging = true;
 
     @FieldContext(
             category = CATEGORY_STORAGE_BK,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java
index a02689dc9763a..0dea84e727a88 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java
@@ -303,22 +303,22 @@ public void testBookKeeperIoThreadsConfiguration() throws Exception {
     public void testBookKeeperLimitStatsLoggingConfiguration() throws Exception {
         BookKeeperClientFactoryImpl factory = new BookKeeperClientFactoryImpl();
         ServiceConfiguration conf = new ServiceConfiguration();
-        assertFalse(
-                factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf).getLimitStatsLogging());
+        assertTrue(factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf)
+                .getLimitStatsLogging());
         EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class);
         BookKeeper.Builder builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class),
                 factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf));
         ClientConfiguration clientConfiguration =
                 (ClientConfiguration) FieldUtils.readField(builder, "conf", true);
-        assertFalse(clientConfiguration.getLimitStatsLogging());
+        assertTrue(clientConfiguration.getLimitStatsLogging());
 
-        conf.setBookkeeperClientLimitStatsLogging(true);
-        assertTrue(factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf)
-                .getLimitStatsLogging());
+        conf.setBookkeeperClientLimitStatsLogging(false);
+        assertFalse(
+                factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf).getLimitStatsLogging());
         builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class),
                 factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf));
         clientConfiguration =
                 (ClientConfiguration) FieldUtils.readField(builder, "conf", true);
-        assertTrue(clientConfiguration.getLimitStatsLogging());
+        assertFalse(clientConfiguration.getLimitStatsLogging());
     }
 }

From 563a573168ba51b60f060a993416d7ea642cc27b Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Wed, 31 May 2023 10:18:03 +0800
Subject: [PATCH 265/280] [fix][fn] Fix function update error (#19895)

---
 .../worker/rest/api/FunctionsImpl.java        |  5 +++-
 .../functions/worker/rest/api/SinksImpl.java  |  5 +++-
 .../worker/rest/api/SourcesImpl.java          |  5 +++-
 .../functions/PulsarFunctionsTest.java        | 29 +++++++++++++++++++
 4 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index c7967600da5f6..715d660ddff0d 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -55,6 +55,7 @@
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.utils.ComponentTypeUtils;
 import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
 import org.apache.pulsar.functions.utils.functions.FunctionArchive;
 import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
 import org.apache.pulsar.functions.worker.FunctionsManager;
@@ -374,8 +375,10 @@ public void updateFunction(final String tenant,
 
             Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder;
             if (isNotBlank(functionPkgUrl) || uploadedInputStream != null) {
+                Function.FunctionMetaData metaData = functionMetaDataBuilder.build();
+                metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData);
                 try {
-                    packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(),
+                    packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData,
                             functionPkgUrl, fileDetail, componentPackageFile);
                 } catch (Exception e) {
                     log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType),
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
index 98450c4a0b5d0..5370fe93a7a30 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
@@ -54,6 +54,7 @@
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.utils.ComponentTypeUtils;
+import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
 import org.apache.pulsar.functions.utils.SinkConfigUtils;
 import org.apache.pulsar.functions.utils.io.Connector;
 import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
@@ -378,8 +379,10 @@ public void updateSink(final String tenant,
 
             Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder;
             if (isNotBlank(sinkPkgUrl) || uploadedInputStream != null) {
+                Function.FunctionMetaData metaData = functionMetaDataBuilder.build();
+                metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData);
                 try {
-                    packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(),
+                    packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData,
                             sinkPkgUrl, fileDetail, componentPackageFile);
                 } catch (Exception e) {
                     log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType),
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
index 876c7e7572e78..2f491424d658a 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
@@ -53,6 +53,7 @@
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.utils.ComponentTypeUtils;
+import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
 import org.apache.pulsar.functions.utils.SourceConfigUtils;
 import org.apache.pulsar.functions.utils.io.Connector;
 import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
@@ -372,8 +373,10 @@ public void updateSource(final String tenant,
 
             Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder;
             if (isNotBlank(sourcePkgUrl) || uploadedInputStream != null) {
+                Function.FunctionMetaData metaData = functionMetaDataBuilder.build();
+                metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData);
                 try {
-                    packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(),
+                    packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData,
                             sourcePkgUrl, fileDetail, componentPackageFile);
                 } catch (Exception e) {
                     log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType),
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
index 1e54764ad5d2b..e6ba6acff83c0 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
@@ -729,6 +729,19 @@ protected void testExclamationFunction(Runtime runtime,
         //get function status
         getFunctionStatus(functionName, 0, true, 2);
 
+        // update code file
+        switch (runtime) {
+            case JAVA:
+                updateFunctionCodeFile(functionName, Runtime.JAVA, "test");
+                break;
+            case PYTHON:
+                updateFunctionCodeFile(functionName, Runtime.PYTHON, EXCLAMATION_PYTHON_FILE);
+                break;
+            case GO:
+                updateFunctionCodeFile(functionName, Runtime.GO, EXCLAMATION_GO_FILE);
+                break;
+        }
+
         // delete function
         deleteFunction(functionName);
 
@@ -894,6 +907,22 @@ private void updateFunctionParallelism(String functionName, int parallelism) thr
         assertTrue(result.getStdout().contains("Updated successfully"));
     }
 
+    private void updateFunctionCodeFile(String functionName, Runtime runtime, String codeFile) throws Exception {
+
+        CommandGenerator generator = new CommandGenerator();
+        generator.setFunctionName(functionName);
+        generator.setRuntime(runtime);
+        String command = generator.generateUpdateFunctionCommand(codeFile);
+
+        log.info("---------- Function command: {}", command);
+        String[] commands = {
+                "sh", "-c", command
+        };
+        ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(
+                commands);
+        assertTrue(result.getStdout().contains("Updated successfully"));
+    }
+
     protected  void submitFunction(Runtime runtime,
                                       String inputTopicName,
                                       String outputTopicName,

From fb1b46e5e993d77c583f715d2bea2eadbb052a81 Mon Sep 17 00:00:00 2001
From: Andy Walker 
Date: Tue, 30 May 2023 23:03:36 -0400
Subject: [PATCH 266/280] [fix][fn] Go functions must retrieve consumers by
 non-particioned topic ID (#20413)

Co-authored-by: Andy Walker 
---
 pulsar-function-go/pf/instance.go | 18 ++++++++++++++++--
 pulsar-function-go/pf/util.go     | 11 +++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 138489444d160..6f73f3e631212 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -404,11 +404,25 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) {
 // ackInputMessage doesn't produce any result, or the user doesn't want the result.
 func (gi *goInstance) ackInputMessage(inputMessage pulsar.Message) {
 	log.Debugf("ack input message topic name is: %s", inputMessage.Topic())
-	gi.consumers[inputMessage.Topic()].Ack(inputMessage)
+	gi.respondMessage(inputMessage, true)
 }
 
 func (gi *goInstance) nackInputMessage(inputMessage pulsar.Message) {
-	gi.consumers[inputMessage.Topic()].Nack(inputMessage)
+	gi.respondMessage(inputMessage, false)
+}
+
+func (gi *goInstance) respondMessage(inputMessage pulsar.Message, ack bool) {
+	topicName, err := ParseTopicName(inputMessage.Topic())
+	if err != nil {
+		log.Errorf("unable respond to message ID %s - invalid topic: %v", messageIDStr(inputMessage), err)
+		return
+	}
+	// consumers are indexed by topic name only (no partition)
+	if ack {
+		gi.consumers[topicName.NameWithoutPartition()].Ack(inputMessage)
+		return
+	}
+	gi.consumers[topicName.NameWithoutPartition()].Nack(inputMessage)
 }
 
 func getIdleTimeout(timeoutMilliSecond time.Duration) time.Duration {
diff --git a/pulsar-function-go/pf/util.go b/pulsar-function-go/pf/util.go
index d5b32da841121..1d1aa1cab939f 100644
--- a/pulsar-function-go/pf/util.go
+++ b/pulsar-function-go/pf/util.go
@@ -21,6 +21,8 @@ package pf
 
 import (
 	"fmt"
+
+	"github.com/apache/pulsar-client-go/pulsar"
 )
 
 func getProperties(fullyQualifiedName string, instanceID int) map[string]string {
@@ -39,3 +41,12 @@ func getDefaultSubscriptionName(tenant, namespace, name string) string {
 func getFullyQualifiedInstanceID(tenant, namespace, name string, instanceID int) string {
 	return fmt.Sprintf("%s/%s/%s:%d", tenant, namespace, name, instanceID)
 }
+
+func messageIDStr(msg pulsar.Message) string {
+	// :::
+	return fmt.Sprintf("%d:%d:%d:%d",
+		msg.ID().LedgerID(),
+		msg.ID().EntryID(),
+		msg.ID().PartitionIdx(),
+		msg.ID().BatchIdx())
+}

From 5c74d207426e26ec4feea279ed2ae1b1c2909cba Mon Sep 17 00:00:00 2001
From: thetumbled <52550727+thetumbled@users.noreply.github.com>
Date: Wed, 31 May 2023 14:34:32 +0800
Subject: [PATCH 267/280] [fix] [broker] do not filter system topic while
 shedding. (#18949)

### Motivation

Currently, topics/bundles in `pulsar/system` will be filter while doing shedding, which is introduced by mistake by pr https://github.com/apache/pulsar/pull/15252.
But we need to unload topics/bundles in `pulsar/system` for load balancing.

### Modifications

do not filter topics/bundles in `pulsar/system`.
---
 .../apache/pulsar/broker/loadbalance/LoadData.java    |  2 +-
 .../pulsar/broker/namespace/NamespaceService.java     | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
index 87f630f1a09fb..a632a47f05116 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
@@ -64,7 +64,7 @@ public Map getBundleData() {
 
     public Map getBundleDataForLoadShedding() {
         return bundleData.entrySet().stream()
-                .filter(e -> !NamespaceService.isSystemServiceNamespace(
+                .filter(e -> !NamespaceService.filterNamespaceForShedding(
                         NamespaceBundle.getBundleNamespace(e.getKey())))
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 9d8d9e3890a19..e2d4ef5153769 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1607,6 +1607,17 @@ public static boolean isSystemServiceNamespace(String namespace) {
                 || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches();
     }
 
+    /**
+     * used for filtering bundles in special namespace.
+     * @param namespace
+     * @return True if namespace is HEARTBEAT_NAMESPACE or SLA_NAMESPACE
+     */
+    public static boolean filterNamespaceForShedding(String namespace) {
+        return SLA_NAMESPACE_PATTERN.matcher(namespace).matches()
+                || HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches()
+                || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches();
+    }
+
     public static boolean isHeartbeatNamespace(ServiceUnitId ns) {
         String namespace = ns.getNamespaceObject().toString();
         return HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches()

From 8e0ebba345f5f02bf2434fec9a75268d92b1b60d Mon Sep 17 00:00:00 2001
From: YingQun Zhong 
Date: Wed, 31 May 2023 15:58:51 +0800
Subject: [PATCH 268/280] [fix][offload] fix offload metrics error (#20366)

---
 .../pulsar/common/naming/TopicName.java       | 36 +++++++++++++++++++
 .../pulsar/common/naming/TopicNameTest.java   | 34 ++++++++++++++++++
 .../impl/FileStoreBackedReadHandleImpl.java   | 11 +++---
 .../FileSystemManagedLedgerOffloader.java     | 22 ++++++++----
 .../FileSystemManagedLedgerOffloaderTest.java | 24 +++++++------
 .../impl/BlobStoreBackedInputStreamImpl.java  |  9 +++--
 .../impl/BlobStoreBackedReadHandleImpl.java   |  4 ++-
 .../impl/BlobStoreBackedReadHandleImplV2.java |  4 ++-
 .../impl/BlobStoreManagedLedgerOffloader.java | 12 ++++---
 .../BlockAwareSegmentInputStreamImpl.java     |  7 ++--
 .../BlobStoreManagedLedgerOffloaderTest.java  |  9 ++---
 11 files changed, 135 insertions(+), 37 deletions(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
index 79ef64c1ae459..eebca0e0d7214 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
@@ -339,6 +339,42 @@ public String getPersistenceNamingEncoding() {
         }
     }
 
+    /**
+     * get topic full name from managedLedgerName.
+     *
+     * @return the topic full name, format -> domain://tenant/namespace/topic
+     */
+    public static String fromPersistenceNamingEncoding(String mlName) {
+        // The managedLedgerName convention is: tenant/namespace/domain/topic
+        // We want to transform to topic full name in the order: domain://tenant/namespace/topic
+        if (mlName == null || mlName.length() == 0) {
+            return mlName;
+        }
+        List parts = Splitter.on("/").splitToList(mlName);
+        String tenant;
+        String cluster;
+        String namespacePortion;
+        String domain;
+        String localName;
+        if (parts.size() == 4) {
+            tenant = parts.get(0);
+            cluster = null;
+            namespacePortion = parts.get(1);
+            domain = parts.get(2);
+            localName = parts.get(3);
+            return String.format("%s://%s/%s/%s", domain, tenant, namespacePortion, localName);
+        } else if (parts.size() == 5) {
+            tenant = parts.get(0);
+            cluster = parts.get(1);
+            namespacePortion = parts.get(2);
+            domain = parts.get(3);
+            localName = parts.get(4);
+            return String.format("%s://%s/%s/%s/%s", domain, tenant, cluster, namespacePortion, localName);
+        } else {
+            throw new IllegalArgumentException("Invalid managedLedger name: " + mlName);
+        }
+    }
+
     /**
      * Get a string suitable for completeTopicName lookup.
      *
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
index 8e32fbe3d33c0..835045f9167dd 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
@@ -236,6 +236,40 @@ public void testDecodeEncode() throws Exception {
         assertEquals(name.getPersistenceNamingEncoding(), "prop/colo/ns/persistent/" + encodedName);
     }
 
+    @Test
+    public void testFromPersistenceNamingEncoding() {
+        // case1: V2
+        String mlName1 = "public_tenant/default_namespace/persistent/test_topic";
+        String expectedTopicName1 = "persistent://public_tenant/default_namespace/test_topic";
+
+        TopicName name1 = TopicName.get(expectedTopicName1);
+        assertEquals(name1.getPersistenceNamingEncoding(), mlName1);
+        assertEquals(TopicName.fromPersistenceNamingEncoding(mlName1), expectedTopicName1);
+
+        // case2: V1
+        String mlName2 = "public_tenant/my_cluster/default_namespace/persistent/test_topic";
+        String expectedTopicName2 = "persistent://public_tenant/my_cluster/default_namespace/test_topic";
+
+        TopicName name2 = TopicName.get(expectedTopicName2);
+        assertEquals(name2.getPersistenceNamingEncoding(), mlName2);
+        assertEquals(TopicName.fromPersistenceNamingEncoding(mlName2), expectedTopicName2);
+
+        // case3: null
+        String mlName3 = "";
+        String expectedTopicName3 = "";
+        assertEquals(expectedTopicName3, TopicName.fromPersistenceNamingEncoding(mlName3));
+
+        // case4: Invalid name
+        try {
+            String mlName4 = "public_tenant/my_cluster/default_namespace/persistent/test_topic/sub_topic";
+            TopicName.fromPersistenceNamingEncoding(mlName4);
+            fail("Should have raised exception");
+        } catch (IllegalArgumentException e) {
+            // Exception is expected.
+        }
+    }
+
+
     @SuppressWarnings("deprecation")
     @Test
     public void testTopicNameWithoutCluster() throws Exception {
diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java
index bdeb88e9ac93c..506fbb8de68bf 100644
--- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java
+++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.MapFile;
+import org.apache.pulsar.common.naming.TopicName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,6 +52,7 @@ public class FileStoreBackedReadHandleImpl implements ReadHandle {
     private final LedgerMetadata ledgerMetadata;
     private final LedgerOffloaderStats offloaderStats;
     private final String managedLedgerName;
+    private final String topicName;
 
     private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader reader, long ledgerId,
                                           LedgerOffloaderStats offloaderStats,
@@ -60,13 +62,14 @@ private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader r
         this.reader = reader;
         this.offloaderStats = offloaderStats;
         this.managedLedgerName = managedLedgerName;
+        this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         LongWritable key = new LongWritable();
         BytesWritable value = new BytesWritable();
         try {
             key.set(FileSystemManagedLedgerOffloader.METADATA_KEY_INDEX);
             long startReadIndexTime = System.nanoTime();
             reader.get(key, value);
-            offloaderStats.recordReadOffloadIndexLatency(managedLedgerName,
+            offloaderStats.recordReadOffloadIndexLatency(topicName,
                     System.nanoTime() - startReadIndexTime, TimeUnit.NANOSECONDS);
             this.ledgerMetadata = parseLedgerMetadata(ledgerId, value.copyBytes());
         } catch (IOException e) {
@@ -125,7 +128,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr
                 while (entriesToRead > 0) {
                     long startReadTime = System.nanoTime();
                     reader.next(key, value);
-                    this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName,
+                    this.offloaderStats.recordReadOffloadDataLatency(topicName,
                             System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS);
                     int length = value.getLength();
                     long entryId = key.get();
@@ -135,7 +138,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr
                         buf.writeBytes(value.copyBytes());
                         entriesToRead--;
                         nextExpectedId++;
-                        this.offloaderStats.recordReadOffloadBytes(managedLedgerName, length);
+                        this.offloaderStats.recordReadOffloadBytes(topicName, length);
                     } else if (entryId > lastEntry) {
                         log.info("Expected to read {}, but read {}, which is greater than last entry {}",
                                 nextExpectedId, entryId, lastEntry);
@@ -144,7 +147,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr
             }
                 promise.complete(LedgerEntriesImpl.create(entries));
             } catch (Throwable t) {
-                this.offloaderStats.recordReadOffloadError(managedLedgerName);
+                this.offloaderStats.recordReadOffloadError(topicName);
                 promise.completeExceptionally(t);
                 entries.forEach(LedgerEntry::close);
             }
diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
index d5e09ba725421..25b63374946c8 100644
--- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
+++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.MapFile;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -197,9 +198,10 @@ public void run() {
                 return;
             }
             long ledgerId = readHandle.getId();
-            final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME);
-            String storagePath = getStoragePath(storageBasePath, topicName);
+            final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME);
+            String storagePath = getStoragePath(storageBasePath, managedLedgerName);
             String dataFilePath = getDataFilePath(storagePath, ledgerId, uuid);
+            final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
             LongWritable key = new LongWritable();
             BytesWritable value = new BytesWritable();
             try {
@@ -241,7 +243,7 @@ public void run() {
                 promise.complete(null);
             } catch (Exception e) {
                 log.error("Exception when get CompletableFuture : ManagerLedgerName: {}, "
-                        + "LedgerId: {}, UUID: {} ", topicName, ledgerId, uuid, e);
+                        + "LedgerId: {}, UUID: {} ", managedLedgerName, ledgerId, uuid, e);
                 if (e instanceof InterruptedException) {
                     Thread.currentThread().interrupt();
                 }
@@ -306,22 +308,27 @@ public static FileSystemWriter create(LedgerEntries ledgerEntriesOnce,
         @Override
         public void run() {
             String managedLedgerName = ledgerReader.extraMetadata.get(MANAGED_LEDGER_NAME);
+            String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
             if (ledgerReader.fileSystemWriteException == null) {
                 Iterator iterator = ledgerEntriesOnce.iterator();
                 while (iterator.hasNext()) {
                     LedgerEntry entry = iterator.next();
                     long entryId = entry.getEntryId();
                     key.set(entryId);
+                    byte[] currentEntryBytes;
+                    int currentEntrySize;
                     try {
-                        value.set(entry.getEntryBytes(), 0, entry.getEntryBytes().length);
+                        currentEntryBytes = entry.getEntryBytes();
+                        currentEntrySize = currentEntryBytes.length;
+                        value.set(currentEntryBytes, 0, currentEntrySize);
                         dataWriter.append(key, value);
                     } catch (IOException e) {
                         ledgerReader.fileSystemWriteException = e;
-                        ledgerReader.offloaderStats.recordWriteToStorageError(managedLedgerName);
+                        ledgerReader.offloaderStats.recordWriteToStorageError(topicName);
                         break;
                     }
                     haveOffloadEntryNumber.incrementAndGet();
-                    ledgerReader.offloaderStats.recordOffloadBytes(managedLedgerName, entry.getLength());
+                    ledgerReader.offloaderStats.recordOffloadBytes(topicName, currentEntrySize);
                 }
             }
             countDownLatch.countDown();
@@ -367,6 +374,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map promise = new CompletableFuture<>();
         try {
             fileSystem.delete(new Path(dataFilePath), true);
@@ -376,7 +384,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map
-                this.offloaderStats.recordDeleteOffloadOps(ledgerName, t == null));
+                this.offloaderStats.recordDeleteOffloadOps(topicName, t == null));
     }
 
     @Override
diff --git a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java
index 1aebab571c971..b9de5d1a49e9a 100644
--- a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java
+++ b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.pulsar.common.naming.TopicName;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 import java.net.URI;
@@ -46,8 +47,9 @@
 
 public class FileSystemManagedLedgerOffloaderTest extends FileStoreTestBase {
     private final PulsarMockBookKeeper bk;
-    private String topic = "public/default/testOffload";
-    private String storagePath = createStoragePath(topic);
+    private String managedLedgerName = "public/default/persistent/testOffload";
+    private String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
+    private String storagePath = createStoragePath(managedLedgerName);
     private LedgerHandle lh;
     private ReadHandle toWrite;
     private final int numberOfEntries = 601;
@@ -56,7 +58,7 @@ public class FileSystemManagedLedgerOffloaderTest extends FileStoreTestBase {
     public FileSystemManagedLedgerOffloaderTest() throws Exception {
         this.bk = new PulsarMockBookKeeper(scheduler);
         this.toWrite = buildReadHandle();
-        map.put("ManagedLedgerName", topic);
+        map.put("ManagedLedgerName", managedLedgerName);
     }
 
     private ReadHandle buildReadHandle() throws Exception {
@@ -125,10 +127,10 @@ public void testOffloadAndReadMetrics() throws Exception {
         offloader.offload(toWrite, uuid, map).get();
 
         LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats;
-        assertTrue(offloaderStats.getOffloadError(topic) == 0);
-        assertTrue(offloaderStats.getOffloadBytes(topic) > 0);
-        assertTrue(offloaderStats.getReadLedgerLatency(topic).count > 0);
-        assertTrue(offloaderStats.getWriteStorageError(topic) == 0);
+        assertTrue(offloaderStats.getOffloadError(topicName) == 0);
+        assertTrue(offloaderStats.getOffloadBytes(topicName) > 0);
+        assertTrue(offloaderStats.getReadLedgerLatency(topicName).count > 0);
+        assertTrue(offloaderStats.getWriteStorageError(topicName) == 0);
 
         ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get();
         LedgerEntries toTestEntries = toTest.read(0, numberOfEntries - 1);
@@ -137,10 +139,10 @@ public void testOffloadAndReadMetrics() throws Exception {
             LedgerEntry toTestEntry = toTestIter.next();
         }
 
-        assertTrue(offloaderStats.getReadOffloadError(topic) == 0);
-        assertTrue(offloaderStats.getReadOffloadBytes(topic) > 0);
-        assertTrue(offloaderStats.getReadOffloadDataLatency(topic).count > 0);
-        assertTrue(offloaderStats.getReadOffloadIndexLatency(topic).count > 0);
+        assertTrue(offloaderStats.getReadOffloadError(topicName) == 0);
+        assertTrue(offloaderStats.getReadOffloadBytes(topicName) > 0);
+        assertTrue(offloaderStats.getReadOffloadDataLatency(topicName).count > 0);
+        assertTrue(offloaderStats.getReadOffloadIndexLatency(topicName).count > 0);
     }
 
     @Test
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java
index aa27df46c5e65..0dea46726f50a 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java
@@ -26,6 +26,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.BackedInputStream;
 import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.options.GetOptions;
@@ -44,6 +45,7 @@ public class BlobStoreBackedInputStreamImpl extends BackedInputStream {
     private final int bufferSize;
     private LedgerOffloaderStats offloaderStats;
     private String managedLedgerName;
+    private String topicName;
 
     private long cursor;
     private long bufferOffsetStart;
@@ -71,6 +73,7 @@ public BlobStoreBackedInputStreamImpl(BlobStore blobStore, String bucket, String
         this(blobStore, bucket, key, versionCheck, objectLen, bufferSize);
         this.offloaderStats = offloaderStats;
         this.managedLedgerName = managedLedgerName;
+        this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
     }
 
     /**
@@ -110,13 +113,13 @@ private boolean refillBufferIfNeeded() throws IOException {
                 // because JClouds streams the content
                 // and actually the HTTP call finishes when the stream is fully read
                 if (this.offloaderStats != null) {
-                    this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName,
+                    this.offloaderStats.recordReadOffloadDataLatency(topicName,
                             System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS);
-                    this.offloaderStats.recordReadOffloadBytes(managedLedgerName, endRange - startRange + 1);
+                    this.offloaderStats.recordReadOffloadBytes(topicName, endRange - startRange + 1);
                 }
             } catch (Throwable e) {
                 if (null != this.offloaderStats) {
-                    this.offloaderStats.recordReadOffloadError(this.managedLedgerName);
+                    this.offloaderStats.recordReadOffloadError(this.topicName);
                 }
                 throw new IOException("Error reading from BlobStore", e);
             }
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java
index cdabe5ece0ba2..5a571bb208e34 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java
@@ -45,6 +45,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockBuilder;
 import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.slf4j.Logger;
@@ -261,6 +262,7 @@ public static ReadHandle open(ScheduledExecutorService executor,
         int retryCount = 3;
         OffloadIndexBlock index = null;
         IOException lastException = null;
+        String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         // The following retry is used to avoid to some network issue cause read index file failure.
         // If it can not recovery in the retry, we will throw the exception and the dispatcher will schedule to
         // next read.
@@ -269,7 +271,7 @@ public static ReadHandle open(ScheduledExecutorService executor,
         while (retryCount-- > 0) {
             long readIndexStartTime = System.nanoTime();
             Blob blob = blobStore.getBlob(bucket, indexKey);
-            offloaderStats.recordReadOffloadIndexLatency(managedLedgerName,
+            offloaderStats.recordReadOffloadIndexLatency(topicName,
                     System.nanoTime() - readIndexStartTime, TimeUnit.NANOSECONDS);
             versionCheck.check(indexKey, blob);
             OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create();
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java
index 2e3d0b08970ca..e40a0a3834c85 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java
@@ -46,6 +46,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockV2Builder;
 import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.slf4j.Logger;
@@ -297,13 +298,14 @@ public static ReadHandle open(ScheduledExecutorService executor,
             throws IOException {
         List inputStreams = new LinkedList<>();
         List indice = new LinkedList<>();
+        String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         for (int i = 0; i < indexKeys.size(); i++) {
             String indexKey = indexKeys.get(i);
             String key = keys.get(i);
             log.debug("open bucket: {} index key: {}", bucket, indexKey);
             long startTime = System.nanoTime();
             Blob blob = blobStore.getBlob(bucket, indexKey);
-            offloaderStats.recordReadOffloadIndexLatency(managedLedgerName,
+            offloaderStats.recordReadOffloadIndexLatency(topicName,
                     System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
             log.debug("indexKey blob: {} {}", indexKey, blob);
             versionCheck.check(indexKey, blob);
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
index 6d69b5edbc3fb..7a15c414aa8c4 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
@@ -63,6 +63,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.BlobStoreLocation;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
@@ -176,7 +177,8 @@ public Map getOffloadDriverMetadata() {
     public CompletableFuture offload(ReadHandle readHandle,
                                            UUID uuid,
                                            Map extraMetadata) {
-        final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME);
+        final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME);
+        final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         final BlobStore writeBlobStore = blobStores.get(config.getBlobStoreLocation());
         log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata,
                 config.getBlobStoreLocation(), writeBlobStore);
@@ -226,7 +228,7 @@ public CompletableFuture offload(ReadHandle readHandle,
                         .calculateBlockSize(config.getMaxBlockSizeInBytes(), readHandle, startEntry, entryBytesWritten);
 
                     try (BlockAwareSegmentInputStream blockStream = new BlockAwareSegmentInputStreamImpl(
-                            readHandle, startEntry, blockSize, this.offloaderStats, topicName)) {
+                            readHandle, startEntry, blockSize, this.offloaderStats, managedLedgerName)) {
 
                         Payload partPayload = Payloads.newInputStreamPayload(blockStream);
                         partPayload.getContentMetadata().setContentLength((long) blockSize);
@@ -611,7 +613,8 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid,
 
         return promise.whenComplete((__, t) -> {
             if (null != this.ml) {
-                this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null);
+                this.offloaderStats.recordDeleteOffloadOps(
+                  TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null);
             }
         });
     }
@@ -636,7 +639,8 @@ public CompletableFuture deleteOffloaded(UUID uid, Map off
         });
 
         return promise.whenComplete((__, t) ->
-                this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null));
+                this.offloaderStats.recordDeleteOffloadOps(
+                  TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null));
     }
 
     @Override
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java
index d07fbdb92477b..06d7f2129ba31 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java
@@ -36,6 +36,7 @@
 import org.apache.bookkeeper.mledger.LedgerOffloaderStats;
 import org.apache.bookkeeper.mledger.offload.jcloud.BlockAwareSegmentInputStream;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -73,6 +74,7 @@ public class BlockAwareSegmentInputStreamImpl extends BlockAwareSegmentInputStre
     // Keep a list of all entries ByteBuf, each ByteBuf contains 2 buf: entry header and entry content.
     private List entriesByteBuf = null;
     private LedgerOffloaderStats offloaderStats;
+    private String managedLedgerName;
     private String topicName;
     private int currentOffset = 0;
     private final AtomicBoolean close = new AtomicBoolean(false);
@@ -91,7 +93,8 @@ public BlockAwareSegmentInputStreamImpl(ReadHandle ledger, long startEntryId, in
                                             LedgerOffloaderStats offloaderStats, String ledgerName) {
         this(ledger, startEntryId, blockSize);
         this.offloaderStats = offloaderStats;
-        this.topicName = ledgerName;
+        this.managedLedgerName = ledgerName;
+        this.topicName = TopicName.fromPersistenceNamingEncoding(ledgerName);
     }
 
     private ByteBuf readEntries(int len) throws IOException {
@@ -183,7 +186,7 @@ private List readNextEntriesFromLedger(long start, long maxNumberEntrie
                 log.debug("read ledger entries. start: {}, end: {} cost {}", start, end,
                         TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - startTime));
             }
-            if (offloaderStats != null && topicName != null) {
+            if (offloaderStats != null && managedLedgerName != null) {
                 offloaderStats.recordReadLedgerLatency(topicName, System.nanoTime() - startTime,
                         TimeUnit.NANOSECONDS);
             }
diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java
index ef0bea29e35c3..ac87a8e424038 100644
--- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java
+++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java
@@ -49,6 +49,7 @@
 import org.apache.bookkeeper.mledger.OffloadedLedgerMetadata;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.options.CopyOptions;
 import org.mockito.Mockito;
@@ -172,10 +173,10 @@ public void testOffloadAndReadMetrics() throws Exception {
         LedgerOffloader offloader = getOffloader();
 
         UUID uuid = UUID.randomUUID();
-
-        String topic = "test";
+        String managedLegerName = "public/default/persistent/testOffload";
+        String topic = TopicName.fromPersistenceNamingEncoding(managedLegerName);
         Map extraMap = new HashMap<>();
-        extraMap.put("ManagedLedgerName", topic);
+        extraMap.put("ManagedLedgerName", managedLegerName);
         offloader.offload(toWrite, uuid, extraMap).get();
 
         LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats;
@@ -187,7 +188,7 @@ public void testOffloadAndReadMetrics() throws Exception {
 
         Map map = new HashMap<>();
         map.putAll(offloader.getOffloadDriverMetadata());
-        map.put("ManagedLedgerName", topic);
+        map.put("ManagedLedgerName", managedLegerName);
         ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get();
         LedgerEntries toTestEntries = toTest.read(0, toTest.getLastAddConfirmed());
         Iterator toTestIter = toTestEntries.iterator();

From 9f93af32d7844003d03cf754ec8cfc50b7177220 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 31 May 2023 16:11:30 +0800
Subject: [PATCH 269/280] [improve][broker] Do not expose
 bucketDelayedIndexStats (#20383)

---
 .../common/policies/data/stats/SubscriptionStatsImpl.java       | 2 ++
 .../pulsar/common/policies/data/stats/TopicStatsImpl.java       | 1 +
 2 files changed, 3 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
index 2bfa54da6a002..cfc6cab9e1110 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.common.policies.data.stats;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -134,6 +135,7 @@ public class SubscriptionStatsImpl implements SubscriptionStats {
     /** The size of DelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
+    @JsonIgnore
     public Map bucketDelayedIndexStats;
 
     /** SubscriptionProperties (key/value strings) associated with this subscribe. */
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index f0141bd58d38f..7a48df89b8bcb 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -140,6 +140,7 @@ public class TopicStatsImpl implements TopicStats {
     public long delayedMessageIndexSizeInBytes;
 
     /** Map of bucket delayed index statistics. */
+    @JsonIgnore
     public Map bucketDelayedIndexStats;
 
     /** The compaction stats. */

From b3360745dc0db8c8a5ec60fb28cdb9e2718b29c6 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 31 May 2023 16:23:24 +0300
Subject: [PATCH 270/280] [improve][broker] Remove uneffective solution for
 reducing GC pressure (#20428)

---
 .../pulsar/broker/service/Consumer.java       |  6 +-
 .../pulsar/broker/service/Producer.java       |  6 +-
 .../pulsar/broker/service/ServerCnx.java      | 14 ++++
 .../pulsar/broker/service/TransportCnx.java   |  2 +
 .../data/stats/ConsumerStatsImpl.java         | 57 +++------------
 .../data/stats/PublisherStatsImpl.java        | 73 +++----------------
 6 files changed, 39 insertions(+), 119 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index a3f9da41e6b35..275d685280865 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -193,11 +193,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo
         this.metadata = metadata != null ? metadata : Collections.emptyMap();
 
         stats = new ConsumerStatsImpl();
-        if (cnx.hasHAProxyMessage()) {
-            stats.setAddress(cnx.getHAProxyMessage().sourceAddress() + ":" + cnx.getHAProxyMessage().sourcePort());
-        } else {
-            stats.setAddress(cnx.clientAddress().toString());
-        }
+        stats.setAddress(cnx.clientSourceAddressAndPort());
         stats.consumerName = consumerName;
         stats.setConnectedSince(DateFormatter.now());
         stats.setClientVersion(cnx.getClientVersion());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
index 53b79f06e8e24..f7d2bb2dd2797 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
@@ -127,11 +127,7 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN
         this.metadata = metadata != null ? metadata : Collections.emptyMap();
 
         this.stats = isNonPersistentTopic ? new NonPersistentPublisherStatsImpl() : new PublisherStatsImpl();
-        if (cnx.hasHAProxyMessage()) {
-            stats.setAddress(cnx.getHAProxyMessage().sourceAddress() + ":" + cnx.getHAProxyMessage().sourcePort());
-        } else {
-            stats.setAddress(cnx.clientAddress().toString());
-        }
+        stats.setAddress(cnx.clientSourceAddressAndPort());
         stats.setConnectedSince(DateFormatter.now());
         stats.setClientVersion(cnx.getClientVersion());
         stats.setProducerName(producerName);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index d3c3971dfaced..98c0e5b497998 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -215,6 +215,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
     private final String replicatorPrefix;
     private String clientVersion = null;
     private String proxyVersion = null;
+    private String clientSourceAddressAndPort;
     private int nonPersistentPendingMessages = 0;
     private final int maxNonPersistentPendingMessages;
     private String originalPrincipal = null;
@@ -3375,6 +3376,19 @@ public String clientSourceAddress() {
         }
     }
 
+    @Override
+    public String clientSourceAddressAndPort() {
+        if (clientSourceAddressAndPort == null) {
+            if (hasHAProxyMessage()) {
+                clientSourceAddressAndPort =
+                        getHAProxyMessage().sourceAddress() + ":" + getHAProxyMessage().sourcePort();
+            } else {
+                clientSourceAddressAndPort = clientAddress().toString();
+            }
+        }
+        return clientSourceAddressAndPort;
+    }
+
     CompletableFuture connectionCheckInProgress;
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
index d267160652ae4..94f934fec681e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
@@ -31,6 +31,8 @@ public interface TransportCnx {
 
     SocketAddress clientAddress();
 
+    String clientSourceAddressAndPort();
+
     BrokerService getBrokerService();
 
     PulsarCommandSender getCommandSender();
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java
index ddae2e7135695..548abdc9ada33 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.common.policies.data.stats;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -79,22 +78,11 @@ public class ConsumerStatsImpl implements ConsumerStats {
     public String readPositionWhenJoining;
 
     /** Address of this consumer. */
-    @JsonIgnore
-    private int addressOffset = -1;
-    @JsonIgnore
-    private int addressLength;
-
+    private String address;
     /** Timestamp of connection. */
-    @JsonIgnore
-    private int connectedSinceOffset = -1;
-    @JsonIgnore
-    private int connectedSinceLength;
-
+    private String connectedSince;
     /** Client library version. */
-    @JsonIgnore
-    private int clientVersionOffset = -1;
-    @JsonIgnore
-    private int clientVersionLength;
+    private String clientVersion;
 
     // ignore this json field to skip from stats in future release. replaced with readable #getLastAckedTime().
     @Deprecated
@@ -111,13 +99,6 @@ public class ConsumerStatsImpl implements ConsumerStats {
     /** Metadata (key/value strings) associated with this consumer. */
     public Map metadata;
 
-    /**
-     * In order to prevent multiple string object allocation under stats: create a string-buffer
-     * that stores data for all string place-holders.
-     */
-    @JsonIgnore
-    private StringBuilder stringBuffer = new StringBuilder();
-
     public ConsumerStatsImpl add(ConsumerStatsImpl stats) {
         Objects.requireNonNull(stats);
         this.msgRateOut += stats.msgRateOut;
@@ -134,47 +115,27 @@ public ConsumerStatsImpl add(ConsumerStatsImpl stats) {
     }
 
     public String getAddress() {
-        return addressOffset == -1 ? null : stringBuffer.substring(addressOffset, addressOffset + addressLength);
+        return address;
     }
 
     public void setAddress(String address) {
-        if (address == null) {
-            this.addressOffset = -1;
-            return;
-        }
-        this.addressOffset = this.stringBuffer.length();
-        this.addressLength = address.length();
-        this.stringBuffer.append(address);
+        this.address = address;
     }
 
     public String getConnectedSince() {
-        return connectedSinceOffset == -1 ? null
-                : stringBuffer.substring(connectedSinceOffset, connectedSinceOffset + connectedSinceLength);
+        return connectedSince;
     }
 
     public void setConnectedSince(String connectedSince) {
-        if (connectedSince == null) {
-            this.connectedSinceOffset = -1;
-            return;
-        }
-        this.connectedSinceOffset = this.stringBuffer.length();
-        this.connectedSinceLength = connectedSince.length();
-        this.stringBuffer.append(connectedSince);
+        this.connectedSince = connectedSince;
     }
 
     public String getClientVersion() {
-        return clientVersionOffset == -1 ? null
-                : stringBuffer.substring(clientVersionOffset, clientVersionOffset + clientVersionLength);
+        return clientVersion;
     }
 
     public void setClientVersion(String clientVersion) {
-        if (clientVersion == null) {
-            this.clientVersionOffset = -1;
-            return;
-        }
-        this.clientVersionOffset = this.stringBuffer.length();
-        this.clientVersionLength = clientVersion.length();
-        this.stringBuffer.append(clientVersion);
+        this.clientVersion = clientVersion;
     }
 
     public String getReadPositionWhenJoining() {
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java
index 41407a37e7ca0..304361bb2daec 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java
@@ -53,35 +53,13 @@ public class PublisherStatsImpl implements PublisherStats {
     public boolean supportsPartialProducer;
 
     /** Producer name. */
-    @JsonIgnore
-    private int producerNameOffset = -1;
-    @JsonIgnore
-    private int producerNameLength;
-
+    private String producerName;
     /** Address of this publisher. */
-    @JsonIgnore
-    private int addressOffset = -1;
-    @JsonIgnore
-    private int addressLength;
-
+    private String address;
     /** Timestamp of connection. */
-    @JsonIgnore
-    private int connectedSinceOffset = -1;
-    @JsonIgnore
-    private int connectedSinceLength;
-
+    private String connectedSince;
     /** Client library version. */
-    @JsonIgnore
-    private int clientVersionOffset = -1;
-    @JsonIgnore
-    private int clientVersionLength;
-
-    /**
-     * In order to prevent multiple string objects under stats: create a string-buffer that stores data for all string
-     * place-holders.
-     */
-    @JsonIgnore
-    private StringBuilder stringBuffer = new StringBuilder();
+    private String clientVersion;
 
     /** Metadata (key/value strings) associated with this publisher. */
     public Map metadata;
@@ -99,61 +77,34 @@ public PublisherStatsImpl add(PublisherStatsImpl stats) {
     }
 
     public String getProducerName() {
-        return producerNameOffset == -1 ? null
-                : stringBuffer.substring(producerNameOffset, producerNameOffset + producerNameLength);
+        return producerName;
     }
 
     public void setProducerName(String producerName) {
-        if (producerName == null) {
-            this.producerNameOffset = -1;
-            return;
-        }
-        this.producerNameOffset = this.stringBuffer.length();
-        this.producerNameLength = producerName.length();
-        this.stringBuffer.append(producerName);
+        this.producerName = producerName;
     }
 
     public String getAddress() {
-        return addressOffset == -1 ? null : stringBuffer.substring(addressOffset, addressOffset + addressLength);
+        return address;
     }
 
     public void setAddress(String address) {
-        if (address == null) {
-            this.addressOffset = -1;
-            return;
-        }
-        this.addressOffset = this.stringBuffer.length();
-        this.addressLength = address.length();
-        this.stringBuffer.append(address);
+        this.address = address;
     }
 
     public String getConnectedSince() {
-        return connectedSinceOffset == -1 ? null
-                : stringBuffer.substring(connectedSinceOffset, connectedSinceOffset + connectedSinceLength);
+        return connectedSince;
     }
 
     public void setConnectedSince(String connectedSince) {
-        if (connectedSince == null) {
-            this.connectedSinceOffset = -1;
-            return;
-        }
-        this.connectedSinceOffset = this.stringBuffer.length();
-        this.connectedSinceLength = connectedSince.length();
-        this.stringBuffer.append(connectedSince);
+        this.connectedSince = connectedSince;
     }
 
     public String getClientVersion() {
-        return clientVersionOffset == -1 ? null
-                : stringBuffer.substring(clientVersionOffset, clientVersionOffset + clientVersionLength);
+        return clientVersion;
     }
 
     public void setClientVersion(String clientVersion) {
-        if (clientVersion == null) {
-            this.clientVersionOffset = -1;
-            return;
-        }
-        this.clientVersionOffset = this.stringBuffer.length();
-        this.clientVersionLength = clientVersion.length();
-        this.stringBuffer.append(clientVersion);
+        this.clientVersion = clientVersion;
     }
 }

From 242758d5770de46e506855ff881472cbc274cedb Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 31 May 2023 22:57:16 +0300
Subject: [PATCH 271/280] [fix][test] Fix flaky PersistentSubscriptionTest
 (#20434)

---
 .../PersistentSubscriptionTest.java           | 106 +++++++++---------
 .../NonStartableTestPulsarService.java        |  17 +++
 2 files changed, 72 insertions(+), 51 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
index 401f52daa6291..87408598889e7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
@@ -92,7 +92,12 @@ public class PersistentSubscriptionTest {
     public void setup() throws Exception {
         pulsarTestContext = PulsarTestContext.builderForNonStartableContext()
                 .spyByDefault()
-                .configCustomizer(config -> config.setTransactionCoordinatorEnabled(true))
+                .configCustomizer(config -> {
+                    config.setTransactionCoordinatorEnabled(true);
+                    config.setTransactionPendingAckStoreProviderClassName(
+                            CustomTransactionPendingAckStoreProvider.class.getName());
+                    config.setTransactionBufferProviderClassName(InMemTransactionBufferProvider.class.getName());
+                })
                 .useTestPulsarResources()
                 .build();
 
@@ -100,56 +105,6 @@ public void setup() throws Exception {
         doReturn(Optional.of(new Policies())).when(namespaceResources)
                 .getPoliciesIfCached(any());
 
-        doReturn(new InMemTransactionBufferProvider()).when(pulsarTestContext.getPulsarService())
-                .getTransactionBufferProvider();
-        doReturn(new TransactionPendingAckStoreProvider() {
-            @Override
-            public CompletableFuture newPendingAckStore(PersistentSubscription subscription) {
-                return CompletableFuture.completedFuture(new PendingAckStore() {
-                    @Override
-                    public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService) {
-                        try {
-                            Field field = PendingAckHandleState.class.getDeclaredField("state");
-                            field.setAccessible(true);
-                            field.set(pendingAckHandle, PendingAckHandleState.State.Ready);
-                        } catch (NoSuchFieldException | IllegalAccessException e) {
-                            fail();
-                        }
-                    }
-
-                    @Override
-                    public CompletableFuture closeAsync() {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-                });
-            }
-
-            @Override
-            public CompletableFuture checkInitializedBefore(PersistentSubscription subscription) {
-                return CompletableFuture.completedFuture(true);
-            }
-        }).when(pulsarTestContext.getPulsarService()).getTransactionPendingAckStoreProvider();
-
         ledgerMock = mock(ManagedLedgerImpl.class);
         cursorMock = mock(ManagedCursorImpl.class);
         managedLedgerConfigMock = mock(ManagedLedgerConfig.class);
@@ -279,4 +234,53 @@ public void testAcknowledgeUpdateCursorLastActive() throws Exception {
         // `acknowledgeMessage` should update cursor last active
         assertTrue(persistentSubscription.cursor.getLastActive() > beforeAcknowledgeTimestamp);
     }
+
+    public static class CustomTransactionPendingAckStoreProvider implements TransactionPendingAckStoreProvider {
+        @Override
+        public CompletableFuture newPendingAckStore(PersistentSubscription subscription) {
+            return CompletableFuture.completedFuture(new PendingAckStore() {
+                @Override
+                public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService) {
+                    try {
+                        Field field = PendingAckHandleState.class.getDeclaredField("state");
+                        field.setAccessible(true);
+                        field.set(pendingAckHandle, PendingAckHandleState.State.Ready);
+                    } catch (NoSuchFieldException | IllegalAccessException e) {
+                        fail();
+                    }
+                }
+
+                @Override
+                public CompletableFuture closeAsync() {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendIndividualAck(TxnID txnID,
+                                                                   List> positions) {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) {
+                    return CompletableFuture.completedFuture(null);
+                }
+            });
+        }
+
+        @Override
+        public CompletableFuture checkInitializedBefore(PersistentSubscription subscription) {
+            return CompletableFuture.completedFuture(true);
+        }
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java
index 4b7762a2acfdf..13c4d7d72af2c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java
@@ -21,6 +21,7 @@
 import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs;
 import static org.mockito.Mockito.mock;
 import io.netty.channel.EventLoopGroup;
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -39,6 +40,8 @@
 import org.apache.pulsar.broker.service.schema.DefaultSchemaRegistryService;
 import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
 import org.apache.pulsar.broker.storage.ManagedLedgerStorage;
+import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider;
+import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.impl.PulsarClientImpl;
@@ -89,6 +92,20 @@ public NonStartableTestPulsarService(SpyConfig spyConfig, ServiceConfiguration c
         } catch (PulsarServerException e) {
             throw new RuntimeException(e);
         }
+        if (config.isTransactionCoordinatorEnabled()) {
+            try {
+                setTransactionBufferProvider(TransactionBufferProvider
+                        .newProvider(config.getTransactionBufferProviderClassName()));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            try {
+                setTransactionPendingAckStoreProvider(TransactionPendingAckStoreProvider
+                        .newProvider(config.getTransactionPendingAckStoreProviderClassName()));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
     }
 
     @Override

From c35b820bb323c8e52bd9cd8ccd29565c23764117 Mon Sep 17 00:00:00 2001
From: crossoverJie 
Date: Thu, 1 Jun 2023 11:48:11 +0800
Subject: [PATCH 272/280] [fix][broker] Fix skip message API when hole messages
 exists (#20326)

---
 .../mledger/impl/ManagedCursorImpl.java       |  1 -
 .../pulsar/broker/admin/AdminApiTest.java     | 49 +++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 663081c932052..1ce0403a54762 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -1779,7 +1779,6 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) {
                 } finally {
                     if (r.lowerEndpoint() instanceof PositionImplRecyclable) {
                         ((PositionImplRecyclable) r.lowerEndpoint()).recycle();
-                        ((PositionImplRecyclable) r.upperEndpoint()).recycle();
                     }
                 }
             }, recyclePositionRangeConverter);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index 855343e18a24b..8561ea68c16e4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -55,11 +55,13 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.ws.rs.client.InvocationCallback;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response.Status;
 import lombok.Builder;
 import lombok.Cleanup;
+import lombok.SneakyThrows;
 import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.ManagedLedgerInfo;
@@ -90,6 +92,7 @@
 import org.apache.pulsar.client.api.ConsumerCryptoFailureAction;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageListener;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
@@ -993,6 +996,52 @@ public void persistentTopics(String topicName) throws Exception {
         assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>());
     }
 
+    @Test(dataProvider = "topicName")
+    public void testSkipHoleMessages(String topicName) throws Exception {
+        final String subName = topicName;
+        assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>());
+
+        final String persistentTopicName = "persistent://prop-xyz/ns1/" + topicName;
+        // Force to create a topic
+        publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 0);
+        assertEquals(admin.topics().getList("prop-xyz/ns1"),
+                List.of("persistent://prop-xyz/ns1/" + topicName));
+
+        // create consumer and subscription
+        @Cleanup
+        PulsarClient client = PulsarClient.builder()
+                .serviceUrl(pulsar.getWebServiceAddress())
+                .statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        AtomicInteger total = new AtomicInteger();
+        Consumer consumer = client.newConsumer().topic(persistentTopicName)
+                .messageListener(new MessageListener() {
+                    @SneakyThrows
+                    @Override
+                    public void received(Consumer consumer, Message msg) {
+                        if (total.get() %2 !=0){
+                            // artificially created 50 hollow messages
+                            consumer.acknowledge(msg);
+                        }
+                        total.incrementAndGet();
+                    }
+                })
+                .subscriptionName(subName)
+                .subscriptionType(SubscriptionType.Exclusive).subscribe();
+
+        assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of(subName));
+
+        publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 100);
+        TimeUnit.SECONDS.sleep(2);
+        TopicStats topicStats = admin.topics().getStats(persistentTopicName);
+        long msgBacklog = topicStats.getSubscriptions().get(subName).getMsgBacklog();
+        log.info("back={}",msgBacklog);
+        int skipNumber = 20;
+        admin.topics().skipMessages(persistentTopicName, subName, skipNumber);
+        topicStats = admin.topics().getStats(persistentTopicName);
+        assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), msgBacklog - skipNumber);
+    }
+
     @Test(dataProvider = "topicNamesForAllTypes")
     public void partitionedTopics(String topicType, String topicName) throws Exception {
         final String namespace = "prop-xyz/ns1";

From 5e6e6cebcdbeec32ed49729f658f2d5cd0d98347 Mon Sep 17 00:00:00 2001
From: hleecs 
Date: Thu, 1 Jun 2023 12:36:46 +0800
Subject: [PATCH 273/280] [feat][broker]PIP-255 Part-1: Add listener interface
 for namespace service (#20406)

---
 .../NamespaceBundleSplitListener.java         | 29 ++++++++++++++++
 .../broker/namespace/NamespaceService.java    | 27 +++++++++++++++
 .../namespace/NamespaceCreateBundlesTest.java | 33 +++++++++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java
new file mode 100644
index 0000000000000..a3312f5689e38
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.namespace;
+
+import java.util.function.Predicate;
+import org.apache.pulsar.common.naming.NamespaceBundle;
+
+/**
+ * Listener for NamespaceBundle split.
+ */
+public interface NamespaceBundleSplitListener extends Predicate {
+    void onSplit(NamespaceBundle bundle);
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index e2d4ef5153769..cf969460c3345 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -139,6 +139,10 @@ public class NamespaceService implements AutoCloseable {
     private final ConcurrentOpenHashMap namespaceClients;
 
     private final List bundleOwnershipListeners;
+
+    private final List bundleSplitListeners;
+
+
     private final RedirectManager redirectManager;
 
 
@@ -167,6 +171,7 @@ public NamespaceService(PulsarService pulsar) {
         this.namespaceClients =
                 ConcurrentOpenHashMap.newBuilder().build();
         this.bundleOwnershipListeners = new CopyOnWriteArrayList<>();
+        this.bundleSplitListeners = new CopyOnWriteArrayList<>();
         this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class);
         this.redirectManager = new RedirectManager(pulsar);
     }
@@ -975,6 +980,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle,
                                 // affect the split operation which is already safely completed
                                 r.forEach(this::unloadNamespaceBundle);
                             }
+                            onNamespaceBundleSplit(bundle);
                         })
                         .exceptionally(e -> {
                             String msg1 = format(
@@ -1230,6 +1236,18 @@ protected void onNamespaceBundleUnload(NamespaceBundle bundle) {
         }
     }
 
+    protected void onNamespaceBundleSplit(NamespaceBundle bundle) {
+        for (NamespaceBundleSplitListener bundleSplitListener : bundleSplitListeners) {
+            try {
+                if (bundleSplitListener.test(bundle)) {
+                    bundleSplitListener.onSplit(bundle);
+                }
+            } catch (Throwable t) {
+                LOG.error("Call bundle {} split listener {} error", bundle, bundleSplitListener, t);
+            }
+        }
+    }
+
     public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener... listeners) {
         Objects.requireNonNull(listeners);
         for (NamespaceBundleOwnershipListener listener : listeners) {
@@ -1240,6 +1258,15 @@ public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener
         getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners));
     }
 
+    public void addNamespaceBundleSplitListener(NamespaceBundleSplitListener... listeners) {
+        Objects.requireNonNull(listeners);
+        for (NamespaceBundleSplitListener listener : listeners) {
+            if (listener != null) {
+                bundleSplitListeners.add(listener);
+            }
+        }
+    }
+
     private void notifyNamespaceBundleOwnershipListener(NamespaceBundle bundle,
                     NamespaceBundleOwnershipListener... listeners) {
         if (listeners != null) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java
index 43d37466918ce..73cfaf1b0d96b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java
@@ -20,15 +20,21 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import lombok.Cleanup;
 
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.pulsar.broker.service.BrokerTestBase;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerBuilder;
+import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.policies.data.BookieAffinityGroupData;
 import org.apache.pulsar.common.policies.data.Policies;
+import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -81,4 +87,31 @@ public void testSplitBundleUpdatesLocalPoliciesWithoutOverwriting() throws Excep
         assertNotNull(admin.namespaces().getBookieAffinityGroup(namespaceName));
         producer.close();
     }
+
+    @Test
+    public void testBundleSplitListener() throws Exception {
+        String namespaceName = "prop/" + UUID.randomUUID().toString();
+        String topicName = "persistent://" + namespaceName + "/my-topic5";
+        admin.namespaces().createNamespace(namespaceName);
+        @Cleanup
+        Producer producer = pulsarClient.newProducer().topic(topicName).sendTimeout(1,
+            TimeUnit.SECONDS).create();
+        producer.send(new byte[1]);
+        String bundleRange = admin.lookups().getBundleRange(topicName);
+        AtomicBoolean isTriggered = new AtomicBoolean(false);
+        pulsar.getNamespaceService().addNamespaceBundleSplitListener(new NamespaceBundleSplitListener() {
+            @Override
+            public void onSplit(NamespaceBundle bundle) {
+                assertEquals(bundleRange, bundle.getBundleRange());
+                isTriggered.set(true);
+            }
+
+            @Override
+            public boolean test(NamespaceBundle namespaceBundle) {
+                return true;
+            }
+        });
+        admin.namespaces().splitNamespaceBundle(namespaceName, bundleRange, false, null);
+        Awaitility.await().untilAsserted(() -> assertTrue(isTriggered.get()));
+    }
 }

From 195345683b45a34dde48b5e6c7d88706b17e91bc Mon Sep 17 00:00:00 2001
From: daojun 
Date: Sun, 15 Jan 2023 18:53:08 +0800
Subject: [PATCH 274/280] Add `topic_load_failed` metric

---
 .../apache/pulsar/broker/service/BrokerService.java    |  9 +++++++++
 .../org/apache/pulsar/broker/service/PulsarStats.java  |  8 ++++++++
 .../pulsar/broker/stats/BrokerOperabilityMetrics.java  | 10 +++++++++-
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 663d013dc7439..e334988de1d3c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -1226,6 +1226,10 @@ public void deleteTopicAuthenticationWithRetry(String topic, CompletableFuture> createNonPersistentTopic(String topic) {
         CompletableFuture> topicFuture = new CompletableFuture<>();
+        topicFuture.exceptionally(t -> {
+            pulsarStats.recordTopicLoadFailed(topic);
+            return Optional.empty();
+        });
         if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) {
             if (log.isDebugEnabled()) {
                 log.debug("Broker is unable to load non-persistent topic {}", topic);
@@ -1618,6 +1622,11 @@ private void createPersistentTopic(final String topic, boolean createIfMissing,
         TopicName topicName = TopicName.get(topic);
         final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
 
+        topicFuture.exceptionally(t -> {
+            pulsarStats.recordTopicLoadFailed(topic);
+            return Optional.empty();
+        });
+
         if (isTransactionInternalName(topicName)) {
             String msg = String.format("Can not create transaction system topic %s", topic);
             log.warn(msg);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
index e959e9bbda2bb..d73a0cca7d553 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
@@ -265,6 +265,14 @@ public void recordTopicLoadTimeValue(String topic, long topicLoadLatencyMs) {
         }
     }
 
+    public void recordTopicLoadFailed(String topic) {
+        try {
+            brokerOperabilityMetrics.recordTopicLoadFailed();
+        } catch (Exception ex) {
+            log.warn("Exception while recording topic failed for topic {}, {}", topic, ex.getMessage());
+        }
+    }
+
     public void recordConnectionCreate() {
         brokerOperabilityMetrics.recordConnectionCreate();
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
index 909133338719f..71521b42282ac 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.stats;
 
+import io.prometheus.client.Counter;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -38,6 +39,7 @@ public class BrokerOperabilityMetrics {
     private final LongAdder connectionCreateFailCount;
     private final LongAdder connectionTotalClosedCount;
     private final LongAdder connectionActive;
+    private final Counter topicLoadFailed = Counter.build("topic_load_failed", "-").register();
 
     public BrokerOperabilityMetrics(String localCluster, String brokerName) {
         this.metricsList = new ArrayList<>();
@@ -84,7 +86,9 @@ Map getDimensionMap(String metricsName) {
     }
 
     Metrics getTopicLoadMetrics() {
-        return getDimensionMetrics("topic_load_times", "topic_load", topicLoadStats);
+        Metrics metrics = getDimensionMetrics("topic_load_times", "topic_load", topicLoadStats);
+        metrics.put("brk_topic_load_failed_count", topicLoadFailed.get());
+        return metrics;
     }
 
     Metrics getDimensionMetrics(String metricsName, String dimensionName, DimensionStats stats) {
@@ -112,6 +116,10 @@ public void recordTopicLoadTimeValue(long topicLoadLatencyMs) {
         topicLoadStats.recordDimensionTimeValue(topicLoadLatencyMs, TimeUnit.MILLISECONDS);
     }
 
+    public void recordTopicLoadFailed() {
+        this.topicLoadFailed.inc();
+    }
+
     public void recordConnectionCreate() {
         this.connectionTotalCreatedCount.increment();
         this.connectionActive.increment();

From b17a9511ade458d7cf23ebbf3731dd3bc142b62b Mon Sep 17 00:00:00 2001
From: daojun 
Date: Sun, 15 Jan 2023 19:47:06 +0800
Subject: [PATCH 275/280] review fix

---
 .../java/org/apache/pulsar/broker/service/BrokerService.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index e334988de1d3c..35d14ee6db5c7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -1228,7 +1228,7 @@ private CompletableFuture> createNonPersistentTopic(String topic
         CompletableFuture> topicFuture = new CompletableFuture<>();
         topicFuture.exceptionally(t -> {
             pulsarStats.recordTopicLoadFailed(topic);
-            return Optional.empty();
+            return null;
         });
         if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) {
             if (log.isDebugEnabled()) {
@@ -1624,7 +1624,7 @@ private void createPersistentTopic(final String topic, boolean createIfMissing,
 
         topicFuture.exceptionally(t -> {
             pulsarStats.recordTopicLoadFailed(topic);
-            return Optional.empty();
+            return null;
         });
 
         if (isTransactionInternalName(topicName)) {

From 28461b70c8a52123d7a7dabb1f7479a68f656efb Mon Sep 17 00:00:00 2001
From: daojun 
Date: Tue, 17 Jan 2023 02:01:26 +0800
Subject: [PATCH 276/280] review fix & add tests

---
 .../pulsar/broker/service/BrokerService.java  |  4 +-
 .../pulsar/broker/service/PulsarStats.java    |  8 +-
 .../broker/service/BrokerServiceTest.java     | 96 ++++++++++++++++---
 .../broker/stats/PrometheusMetricsTest.java   |  4 +
 4 files changed, 92 insertions(+), 20 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 35d14ee6db5c7..3dc7718c32a74 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -1227,7 +1227,7 @@ public void deleteTopicAuthenticationWithRetry(String topic, CompletableFuture> createNonPersistentTopic(String topic) {
         CompletableFuture> topicFuture = new CompletableFuture<>();
         topicFuture.exceptionally(t -> {
-            pulsarStats.recordTopicLoadFailed(topic);
+            pulsarStats.recordTopicLoadFailed();
             return null;
         });
         if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) {
@@ -1623,7 +1623,7 @@ private void createPersistentTopic(final String topic, boolean createIfMissing,
         final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
 
         topicFuture.exceptionally(t -> {
-            pulsarStats.recordTopicLoadFailed(topic);
+            pulsarStats.recordTopicLoadFailed();
             return null;
         });
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
index d73a0cca7d553..db14892d26663 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
@@ -265,12 +265,8 @@ public void recordTopicLoadTimeValue(String topic, long topicLoadLatencyMs) {
         }
     }
 
-    public void recordTopicLoadFailed(String topic) {
-        try {
-            brokerOperabilityMetrics.recordTopicLoadFailed();
-        } catch (Exception ex) {
-            log.warn("Exception while recording topic failed for topic {}, {}", topic, ex.getMessage());
-        }
+    public void recordTopicLoadFailed() {
+        brokerOperabilityMetrics.recordTopicLoadFailed();
     }
 
     public void recordConnectionCreate() {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index e9b7ddb991e57..b1f785aef8ea1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -33,6 +33,7 @@
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.EventLoopGroup;
 import io.netty.util.concurrent.DefaultThreadFactory;
@@ -42,14 +43,7 @@
 import java.io.InputStreamReader;
 import java.lang.reflect.Field;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
@@ -100,13 +94,13 @@
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.BundlesData;
-import org.apache.pulsar.common.policies.data.LocalPolicies;
-import org.apache.pulsar.common.policies.data.SubscriptionStats;
-import org.apache.pulsar.common.policies.data.TopicStats;
+import org.apache.pulsar.common.policies.data.*;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.netty.EventLoopUtil;
 import org.awaitility.Awaitility;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -1513,4 +1507,82 @@ public void testDynamicConfigurationsForceDeleteTenantAllowed() throws Exception
             assertTrue(conf.isForceDeleteTenantAllowed());
         });
     }
+
+
+    @Test
+    public void testBrokerStatsTopicLoadFailed() throws Exception {
+        admin.namespaces().createNamespace("prop/ns-test");
+
+        String persistentTopic = "persistent://prop/ns-test/topic1_" + UUID.randomUUID();
+        String nonPersistentTopic = "non-persistent://prop/ns-test/topic2_" + UUID.randomUUID();
+
+        BrokerService brokerService = pulsar.getBrokerService();
+        brokerService = Mockito.spy(brokerService);
+        // mock create persistent topic failed
+        Mockito
+                .doAnswer(invocation -> {
+                    CompletableFuture f = new CompletableFuture<>();
+                    f.completeExceptionally(new RuntimeException("This is an exception"));
+                    return f;
+                })
+                .when(brokerService).getManagedLedgerConfig(Mockito.eq(TopicName.get(persistentTopic)));
+
+        // mock create non-persistent topic failed
+        Mockito
+                .doAnswer(inv -> {
+                    CompletableFuture f = new CompletableFuture<>();
+                    f.completeExceptionally(new RuntimeException("This is an exception"));
+                    return f;
+                })
+                .when(brokerService).checkTopicNsOwnership(Mockito.eq(nonPersistentTopic));
+
+
+        PulsarService pulsarService = pulsar;
+        Field field = PulsarService.class.getDeclaredField("brokerService");
+        field.setAccessible(true);
+        field.set(pulsarService, brokerService);
+
+        CompletableFuture> producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(persistentTopic)
+                .createAsync();
+        CompletableFuture> producer1 = pulsarClient.newProducer(Schema.STRING)
+                .topic(nonPersistentTopic)
+                .createAsync();
+
+        producer.whenComplete((v, t) -> {
+            if (t == null) {
+                try {
+                    v.close();
+                } catch (PulsarClientException e) {
+                    // ignore
+                }
+            }
+        });
+        producer1.whenComplete((v, t) -> {
+            if (t == null) {
+                try {
+                    v.close();
+                } catch (PulsarClientException e) {
+                    // ignore
+                }
+            }
+        });
+
+        Awaitility.waitAtMost(2, TimeUnit.MINUTES).until(() -> {
+            String json = admin.brokerStats().getMetrics();
+            JsonArray metrics = new Gson().fromJson(json, JsonArray.class);
+            AtomicBoolean flag = new AtomicBoolean(false);
+
+            metrics.forEach(ele -> {
+                JsonObject obj = ((JsonObject) ele);
+                JsonObject metrics0 = (JsonObject) obj.get("metrics");
+                JsonPrimitive v = (JsonPrimitive) metrics0.get("brk_topic_load_failed_count");
+                if (null != v && v.getAsDouble() >= 2D) {
+                    flag.set(true);
+                }
+            });
+
+            return flag.get();
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index a7a28afd8ac64..6cb7378330f09 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -333,6 +333,10 @@ public void testPerTopicStats() throws Exception {
         assertEquals(cm.size(), 1);
         assertEquals(cm.get(0).tags.get("cluster"), "test");
 
+        cm = (List) metrics.get("topic_load_failed_total");
+        assertEquals(cm.size(), 1);
+        assertEquals(cm.get(0).tags.get("cluster"), "test");
+
         cm = (List) metrics.get("pulsar_in_bytes_total");
         assertEquals(cm.size(), 2);
         assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");

From dd3bbbb982da5830fc0ca25374b490b72fbb96b1 Mon Sep 17 00:00:00 2001
From: daojun 
Date: Wed, 1 Mar 2023 01:35:52 +0800
Subject: [PATCH 277/280] fix imports

---
 .../org/apache/pulsar/broker/service/BrokerServiceTest.java     | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index b1f785aef8ea1..40f1bdb0b2a7a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -99,8 +99,6 @@
 import org.apache.pulsar.common.util.netty.EventLoopUtil;
 import org.awaitility.Awaitility;
 import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

From 496a0110f91066a49de8f57672ddd60d9c3618cc Mon Sep 17 00:00:00 2001
From: daojun 
Date: Wed, 1 Mar 2023 01:38:16 +0800
Subject: [PATCH 278/280] fix imports

---
 .../pulsar/broker/service/BrokerServiceTest.java   | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index 40f1bdb0b2a7a..1d45c87b2f256 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -43,7 +43,14 @@
 import java.io.InputStreamReader;
 import java.lang.reflect.Field;
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
@@ -94,7 +101,10 @@
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.*;
+import org.apache.pulsar.common.policies.data.BundlesData;
+import org.apache.pulsar.common.policies.data.LocalPolicies;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.netty.EventLoopUtil;
 import org.awaitility.Awaitility;

From 2e6567b3b90b4a9b0e56858d3f76c36d018cf354 Mon Sep 17 00:00:00 2001
From: daojun 
Date: Thu, 2 Mar 2023 14:26:09 +0800
Subject: [PATCH 279/280] fix test issue

---
 .../apache/pulsar/broker/stats/BrokerOperabilityMetrics.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
index 71521b42282ac..e6f264cfcfdb7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
@@ -39,7 +39,7 @@ public class BrokerOperabilityMetrics {
     private final LongAdder connectionCreateFailCount;
     private final LongAdder connectionTotalClosedCount;
     private final LongAdder connectionActive;
-    private final Counter topicLoadFailed = Counter.build("topic_load_failed", "-").register();
+    private static final Counter topicLoadFailed = Counter.build("topic_load_failed", "-").register();
 
     public BrokerOperabilityMetrics(String localCluster, String brokerName) {
         this.metricsList = new ArrayList<>();

From 4d1fab7241a81f8bb9b495c5a981dc39b7b25595 Mon Sep 17 00:00:00 2001
From: daojun 
Date: Thu, 2 Mar 2023 14:26:54 +0800
Subject: [PATCH 280/280] fix test issue

---
 .../pulsar/broker/stats/BrokerOperabilityMetrics.java       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
index e6f264cfcfdb7..400dbd3335a2a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
@@ -30,6 +30,7 @@
 /**
  */
 public class BrokerOperabilityMetrics {
+    private static final Counter TOPIC_LOAD_FAILED = Counter.build("topic_load_failed", "-").register();
     private final List metricsList;
     private final String localCluster;
     private final DimensionStats topicLoadStats;
@@ -39,7 +40,6 @@ public class BrokerOperabilityMetrics {
     private final LongAdder connectionCreateFailCount;
     private final LongAdder connectionTotalClosedCount;
     private final LongAdder connectionActive;
-    private static final Counter topicLoadFailed = Counter.build("topic_load_failed", "-").register();
 
     public BrokerOperabilityMetrics(String localCluster, String brokerName) {
         this.metricsList = new ArrayList<>();
@@ -87,7 +87,7 @@ Map getDimensionMap(String metricsName) {
 
     Metrics getTopicLoadMetrics() {
         Metrics metrics = getDimensionMetrics("topic_load_times", "topic_load", topicLoadStats);
-        metrics.put("brk_topic_load_failed_count", topicLoadFailed.get());
+        metrics.put("brk_topic_load_failed_count", TOPIC_LOAD_FAILED.get());
         return metrics;
     }
 
@@ -117,7 +117,7 @@ public void recordTopicLoadTimeValue(long topicLoadLatencyMs) {
     }
 
     public void recordTopicLoadFailed() {
-        this.topicLoadFailed.inc();
+        this.TOPIC_LOAD_FAILED.inc();
     }
 
     public void recordConnectionCreate() {