From 10490278e98b1250c09ec36d8c92c0a707429dee Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 7 Nov 2023 11:53:51 +0530 Subject: [PATCH 01/27] session window Signed-off-by: Yashash H L --- pkg/apis/proto/map/v1/map.pb.go | 2 +- pkg/apis/proto/map/v1/map_grpc.pb.go | 2 +- pkg/apis/proto/mapstream/v1/mapstream.pb.go | 2 +- .../proto/mapstream/v1/mapstream_grpc.pb.go | 2 +- pkg/apis/proto/reduce/v1/reduce.pb.go | 2 +- pkg/apis/proto/reduce/v1/reduce_grpc.pb.go | 2 +- pkg/apis/proto/sessionreduce/v1/mockgen.go | 3 + .../sessionreduce/v1/sessionreduce.pb.go | 848 ++++++++++++++++++ .../sessionreduce/v1/sessionreduce.proto | 95 ++ .../sessionreduce/v1/sessionreduce_grpc.pb.go | 179 ++++ .../v1/sessionreducemock/sessionreducemock.go | 216 +++++ pkg/apis/proto/sideinput/v1/sideinput.pb.go | 2 +- .../proto/sideinput/v1/sideinput_grpc.pb.go | 2 +- pkg/apis/proto/sink/v1/sink.pb.go | 2 +- pkg/apis/proto/sink/v1/sink_grpc.pb.go | 2 +- pkg/apis/proto/source/v1/source.pb.go | 2 +- pkg/apis/proto/source/v1/source_grpc.pb.go | 2 +- .../proto/sourcetransform/v1/transform.pb.go | 2 +- .../sourcetransform/v1/transform_grpc.pb.go | 2 +- pkg/globalreducer/task_manager.go | 210 +++++ pkg/sessionreducer/doc.go | 5 + pkg/sessionreducer/interface.go | 22 + pkg/sessionreducer/message.go | 70 ++ pkg/sessionreducer/options.go | 43 + pkg/sessionreducer/options_test.go | 18 + pkg/sessionreducer/server.go | 56 ++ pkg/sessionreducer/service.go | 115 +++ pkg/sessionreducer/task_manager.go | 227 +++++ pkg/sessionreducer/types.go | 65 ++ 29 files changed, 2186 insertions(+), 14 deletions(-) create mode 100644 pkg/apis/proto/sessionreduce/v1/mockgen.go create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreduce.proto create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go create mode 100644 pkg/globalreducer/task_manager.go create mode 100644 pkg/sessionreducer/doc.go create mode 100644 pkg/sessionreducer/interface.go create mode 100644 pkg/sessionreducer/message.go create mode 100644 pkg/sessionreducer/options.go create mode 100644 pkg/sessionreducer/options_test.go create mode 100644 pkg/sessionreducer/server.go create mode 100644 pkg/sessionreducer/service.go create mode 100644 pkg/sessionreducer/task_manager.go create mode 100644 pkg/sessionreducer/types.go diff --git a/pkg/apis/proto/map/v1/map.pb.go b/pkg/apis/proto/map/v1/map.pb.go index 6dcf613c..03bc0967 100644 --- a/pkg/apis/proto/map/v1/map.pb.go +++ b/pkg/apis/proto/map/v1/map.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v4.23.4 +// protoc v3.21.9 // source: pkg/apis/proto/map/v1/map.proto package v1 diff --git a/pkg/apis/proto/map/v1/map_grpc.pb.go b/pkg/apis/proto/map/v1/map_grpc.pb.go index 9026727d..39b713bb 100644 --- a/pkg/apis/proto/map/v1/map_grpc.pb.go +++ b/pkg/apis/proto/map/v1/map_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v4.23.4 +// - protoc v3.21.9 // source: pkg/apis/proto/map/v1/map.proto package v1 diff --git a/pkg/apis/proto/mapstream/v1/mapstream.pb.go b/pkg/apis/proto/mapstream/v1/mapstream.pb.go index 5681354a..889b7648 100644 --- a/pkg/apis/proto/mapstream/v1/mapstream.pb.go +++ b/pkg/apis/proto/mapstream/v1/mapstream.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v4.23.4 +// protoc v3.21.9 // source: pkg/apis/proto/mapstream/v1/mapstream.proto package v1 diff --git a/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go b/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go index ad88a113..1ad13424 100644 --- a/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go +++ b/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v4.23.4 +// - protoc v3.21.9 // source: pkg/apis/proto/mapstream/v1/mapstream.proto package v1 diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index e952a775..e1963f5d 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v4.23.4 +// protoc v3.21.9 // source: pkg/apis/proto/reduce/v1/reduce.proto package v1 diff --git a/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go b/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go index 11919bac..e93cca9c 100644 --- a/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v4.23.4 +// - protoc v3.21.9 // source: pkg/apis/proto/reduce/v1/reduce.proto package v1 diff --git a/pkg/apis/proto/sessionreduce/v1/mockgen.go b/pkg/apis/proto/sessionreduce/v1/mockgen.go new file mode 100644 index 00000000..7338cedd --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/mockgen.go @@ -0,0 +1,3 @@ +package v1 + +//go:generate mockgen -destination sessionreducemock/sessionreducemock -package sessionreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1 SessionReduceClient,SessionReduce_SessionReduceFnClient diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go new file mode 100644 index 00000000..6a892b74 --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -0,0 +1,848 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.9 +// source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto + +package v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// WindowType represents a window type. +// REVISIT Do we need this? Can this be passed using metadata? +type SessionReduceRequest_WindowType int32 + +const ( + SessionReduceRequest_FIXED SessionReduceRequest_WindowType = 0 + SessionReduceRequest_SLIDING SessionReduceRequest_WindowType = 1 + SessionReduceRequest_SESSION SessionReduceRequest_WindowType = 2 + SessionReduceRequest_GLOBAL SessionReduceRequest_WindowType = 3 +) + +// Enum value maps for SessionReduceRequest_WindowType. +var ( + SessionReduceRequest_WindowType_name = map[int32]string{ + 0: "FIXED", + 1: "SLIDING", + 2: "SESSION", + 3: "GLOBAL", + } + SessionReduceRequest_WindowType_value = map[string]int32{ + "FIXED": 0, + "SLIDING": 1, + "SESSION": 2, + "GLOBAL": 3, + } +) + +func (x SessionReduceRequest_WindowType) Enum() *SessionReduceRequest_WindowType { + p := new(SessionReduceRequest_WindowType) + *p = x + return p +} + +func (x SessionReduceRequest_WindowType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SessionReduceRequest_WindowType) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0].Descriptor() +} + +func (SessionReduceRequest_WindowType) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0] +} + +func (x SessionReduceRequest_WindowType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SessionReduceRequest_WindowType.Descriptor instead. +func (SessionReduceRequest_WindowType) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0} +} + +type SessionReduceRequest_WindowOperation_Event int32 + +const ( + SessionReduceRequest_WindowOperation_OPEN SessionReduceRequest_WindowOperation_Event = 0 + SessionReduceRequest_WindowOperation_CLOSE SessionReduceRequest_WindowOperation_Event = 1 + SessionReduceRequest_WindowOperation_EXPAND SessionReduceRequest_WindowOperation_Event = 2 + SessionReduceRequest_WindowOperation_MERGE SessionReduceRequest_WindowOperation_Event = 3 + SessionReduceRequest_WindowOperation_APPEND SessionReduceRequest_WindowOperation_Event = 4 +) + +// Enum value maps for SessionReduceRequest_WindowOperation_Event. +var ( + SessionReduceRequest_WindowOperation_Event_name = map[int32]string{ + 0: "OPEN", + 1: "CLOSE", + 2: "EXPAND", + 3: "MERGE", + 4: "APPEND", + } + SessionReduceRequest_WindowOperation_Event_value = map[string]int32{ + "OPEN": 0, + "CLOSE": 1, + "EXPAND": 2, + "MERGE": 3, + "APPEND": 4, + } +) + +func (x SessionReduceRequest_WindowOperation_Event) Enum() *SessionReduceRequest_WindowOperation_Event { + p := new(SessionReduceRequest_WindowOperation_Event) + *p = x + return p +} + +func (x SessionReduceRequest_WindowOperation_Event) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SessionReduceRequest_WindowOperation_Event) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[1].Descriptor() +} + +func (SessionReduceRequest_WindowOperation_Event) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[1] +} + +func (x SessionReduceRequest_WindowOperation_Event) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SessionReduceRequest_WindowOperation_Event.Descriptor instead. +func (SessionReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0, 0} +} + +type Partition struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` + End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` +} + +func (x *Partition) Reset() { + *x = Partition{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Partition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Partition) ProtoMessage() {} + +func (x *Partition) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Partition.ProtoReflect.Descriptor instead. +func (*Partition) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{0} +} + +func (x *Partition) GetStart() *timestamppb.Timestamp { + if x != nil { + return x.Start + } + return nil +} + +func (x *Partition) GetEnd() *timestamppb.Timestamp { + if x != nil { + return x.End + } + return nil +} + +func (x *Partition) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +// * +// SessionReduceRequest represents a request element. +type SessionReduceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Payload *SessionReduceRequest_Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + Operation *SessionReduceRequest_WindowOperation `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` + Type SessionReduceRequest_WindowType `protobuf:"varint,3,opt,name=type,proto3,enum=sessionreduce.v1.SessionReduceRequest_WindowType" json:"type,omitempty"` +} + +func (x *SessionReduceRequest) Reset() { + *x = SessionReduceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceRequest) ProtoMessage() {} + +func (x *SessionReduceRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceRequest.ProtoReflect.Descriptor instead. +func (*SessionReduceRequest) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1} +} + +func (x *SessionReduceRequest) GetPayload() *SessionReduceRequest_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SessionReduceRequest) GetOperation() *SessionReduceRequest_WindowOperation { + if x != nil { + return x.Operation + } + return nil +} + +func (x *SessionReduceRequest) GetType() SessionReduceRequest_WindowType { + if x != nil { + return x.Type + } + return SessionReduceRequest_FIXED +} + +// * +// SessionReduceResponse represents a response element. +type SessionReduceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Result []*SessionReduceResponse_Result `protobuf:"bytes,1,rep,name=result,proto3" json:"result,omitempty"` + // Partition represents a window partition to which the result belongs. + Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` + CombinedKey string `protobuf:"bytes,3,opt,name=combinedKey,proto3" json:"combinedKey,omitempty"` + // Cob to indicate that there won't be any more results for this partition. + // This is used to indicate that the partition is closed so that the downstream + // can release the resources associated with the partition. + EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` +} + +func (x *SessionReduceResponse) Reset() { + *x = SessionReduceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceResponse) ProtoMessage() {} + +func (x *SessionReduceResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceResponse.ProtoReflect.Descriptor instead. +func (*SessionReduceResponse) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{2} +} + +func (x *SessionReduceResponse) GetResult() []*SessionReduceResponse_Result { + if x != nil { + return x.Result + } + return nil +} + +func (x *SessionReduceResponse) GetPartition() *Partition { + if x != nil { + return x.Partition + } + return nil +} + +func (x *SessionReduceResponse) GetCombinedKey() string { + if x != nil { + return x.CombinedKey + } + return "" +} + +func (x *SessionReduceResponse) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +// * +// ReadyResponse is the health check result. +type ReadyResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` +} + +func (x *ReadyResponse) Reset() { + *x = ReadyResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReadyResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadyResponse) ProtoMessage() {} + +func (x *ReadyResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. +func (*ReadyResponse) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{3} +} + +func (x *ReadyResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +// WindowOperation represents a window operation. +// it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. +type SessionReduceRequest_WindowOperation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Event SessionReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=sessionreduce.v1.SessionReduceRequest_WindowOperation_Event" json:"event,omitempty"` + Partitions []*Partition `protobuf:"bytes,2,rep,name=partitions,proto3" json:"partitions,omitempty"` +} + +func (x *SessionReduceRequest_WindowOperation) Reset() { + *x = SessionReduceRequest_WindowOperation{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceRequest_WindowOperation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceRequest_WindowOperation) ProtoMessage() {} + +func (x *SessionReduceRequest_WindowOperation) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceRequest_WindowOperation.ProtoReflect.Descriptor instead. +func (*SessionReduceRequest_WindowOperation) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *SessionReduceRequest_WindowOperation) GetEvent() SessionReduceRequest_WindowOperation_Event { + if x != nil { + return x.Event + } + return SessionReduceRequest_WindowOperation_OPEN +} + +func (x *SessionReduceRequest_WindowOperation) GetPartitions() []*Partition { + if x != nil { + return x.Partitions + } + return nil +} + +// Payload represents a payload element. +type SessionReduceRequest_Payload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` +} + +func (x *SessionReduceRequest_Payload) Reset() { + *x = SessionReduceRequest_Payload{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceRequest_Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceRequest_Payload) ProtoMessage() {} + +func (x *SessionReduceRequest_Payload) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceRequest_Payload.ProtoReflect.Descriptor instead. +func (*SessionReduceRequest_Payload) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *SessionReduceRequest_Payload) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *SessionReduceRequest_Payload) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *SessionReduceRequest_Payload) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *SessionReduceRequest_Payload) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +// Result represents a result element. It contains the result of the reduce function. +type SessionReduceResponse_Result struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` +} + +func (x *SessionReduceResponse_Result) Reset() { + *x = SessionReduceResponse_Result{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceResponse_Result) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceResponse_Result) ProtoMessage() {} + +func (x *SessionReduceResponse_Result) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceResponse_Result.ProtoReflect.Descriptor instead. +func (*SessionReduceResponse_Result) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{2, 0} +} + +func (x *SessionReduceResponse_Result) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *SessionReduceResponse_Result) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *SessionReduceResponse_Result) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +var File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto protoreflect.FileDescriptor + +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ + 0x0a, 0x33, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, + 0x31, 0x2f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7d, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, + 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x22, 0xcd, 0x05, 0x0a, 0x14, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, + 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, + 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, + 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x45, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x1a, 0xe3, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0a, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, + 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, + 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, + 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, + 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, + 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0x3d, 0x0a, 0x0a, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x49, 0x58, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, + 0x0a, 0x07, 0x53, 0x4c, 0x49, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, + 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x47, 0x4c, 0x4f, 0x42, + 0x41, 0x4c, 0x10, 0x03, 0x22, 0xbf, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, + 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, + 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, + 0x4b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, + 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, + 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, + 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, + 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, + 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescOnce sync.Once + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData = file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc +) + +func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP() []byte { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescOnce.Do(func() { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData) + }) + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData +} + +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []interface{}{ + (SessionReduceRequest_WindowType)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowType + (SessionReduceRequest_WindowOperation_Event)(0), // 1: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + (*Partition)(nil), // 2: sessionreduce.v1.Partition + (*SessionReduceRequest)(nil), // 3: sessionreduce.v1.SessionReduceRequest + (*SessionReduceResponse)(nil), // 4: sessionreduce.v1.SessionReduceResponse + (*ReadyResponse)(nil), // 5: sessionreduce.v1.ReadyResponse + (*SessionReduceRequest_WindowOperation)(nil), // 6: sessionreduce.v1.SessionReduceRequest.WindowOperation + (*SessionReduceRequest_Payload)(nil), // 7: sessionreduce.v1.SessionReduceRequest.Payload + (*SessionReduceResponse_Result)(nil), // 8: sessionreduce.v1.SessionReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 10: google.protobuf.Empty +} +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ + 9, // 0: sessionreduce.v1.Partition.start:type_name -> google.protobuf.Timestamp + 9, // 1: sessionreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp + 7, // 2: sessionreduce.v1.SessionReduceRequest.payload:type_name -> sessionreduce.v1.SessionReduceRequest.Payload + 6, // 3: sessionreduce.v1.SessionReduceRequest.operation:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation + 0, // 4: sessionreduce.v1.SessionReduceRequest.type:type_name -> sessionreduce.v1.SessionReduceRequest.WindowType + 8, // 5: sessionreduce.v1.SessionReduceResponse.result:type_name -> sessionreduce.v1.SessionReduceResponse.Result + 2, // 6: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition + 9, // 7: sessionreduce.v1.SessionReduceResponse.event_time:type_name -> google.protobuf.Timestamp + 1, // 8: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + 2, // 9: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition + 9, // 10: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 9, // 11: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 3, // 12: sessionreduce.v1.SessionReduce.SessionReduceFn:input_type -> sessionreduce.v1.SessionReduceRequest + 10, // 13: sessionreduce.v1.SessionReduce.IsReady:input_type -> google.protobuf.Empty + 4, // 14: sessionreduce.v1.SessionReduce.SessionReduceFn:output_type -> sessionreduce.v1.SessionReduceResponse + 5, // 15: sessionreduce.v1.SessionReduce.IsReady:output_type -> sessionreduce.v1.ReadyResponse + 14, // [14:16] is the sub-list for method output_type + 12, // [12:14] is the sub-list for method input_type + 12, // [12:12] is the sub-list for extension type_name + 12, // [12:12] is the sub-list for extension extendee + 0, // [0:12] is the sub-list for field type_name +} + +func init() { file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() } +func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { + if File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Partition); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReadyResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceRequest_WindowOperation); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceRequest_Payload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceResponse_Result); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc, + NumEnums: 2, + NumMessages: 7, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes, + DependencyIndexes: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs, + EnumInfos: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes, + MessageInfos: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes, + }.Build() + File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto = out.File + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = nil + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = nil + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = nil +} diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto new file mode 100644 index 00000000..cb11781b --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +option go_package = "github.com/numaproj/numaflow-go/pkg/apis/proto/reducestream/v1"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + + +package sessionreduce.v1; + +service SessionReduce { + // SessionReduceFn applies a reduce function to a request stream. + rpc SessionReduceFn(stream SessionReduceRequest) returns (stream SessionReduceResponse); + + // IsReady is the heartbeat endpoint for gRPC. + rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); +} + +message Partition { + google.protobuf.Timestamp start = 1; + google.protobuf.Timestamp end = 2; + string key = 3; +} + +/** + * SessionReduceRequest represents a request element. + */ +message SessionReduceRequest { + // WindowOperation represents a window operation. + // it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. + message WindowOperation { + enum Event { + OPEN = 0; + CLOSE = 1; + EXPAND = 2; + MERGE = 3; + APPEND = 4; + } + + Event event = 1; + repeated Partition partitions = 2; + } + + // WindowType represents a window type. + //REVISIT Do we need this? Can this be passed using metadata? + enum WindowType { + FIXED = 0; + SLIDING = 1; + SESSION = 2; + GLOBAL = 3; + } + + // Payload represents a payload element. + message Payload { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + } + + Payload payload = 1; + WindowOperation operation = 2; + WindowType type = 3; +} + +/** + * SessionReduceResponse represents a response element. + */ +message SessionReduceResponse { + // Result represents a result element. It contains the result of the reduce function. + message Result { + repeated string keys = 1; + bytes value = 2; + repeated string tags = 3; + } + + + repeated Result result = 1; + + // Partition represents a window partition to which the result belongs. + Partition partition = 2; + + string combinedKey = 3; + // Cob to indicate that there won't be any more results for this partition. + // This is used to indicate that the partition is closed so that the downstream + // can release the resources associated with the partition. + google.protobuf.Timestamp event_time = 4; +} + +/** + * ReadyResponse is the health check result. + */ +message ReadyResponse { + bool ready = 1; +} \ No newline at end of file diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go new file mode 100644 index 00000000..c56ea476 --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go @@ -0,0 +1,179 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.21.9 +// source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SessionReduceClient is the client API for SessionReduce service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SessionReduceClient interface { + // SessionReduceFn applies a reduce function to a request stream. + SessionReduceFn(ctx context.Context, opts ...grpc.CallOption) (SessionReduce_SessionReduceFnClient, error) + // IsReady is the heartbeat endpoint for gRPC. + IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) +} + +type sessionReduceClient struct { + cc grpc.ClientConnInterface +} + +func NewSessionReduceClient(cc grpc.ClientConnInterface) SessionReduceClient { + return &sessionReduceClient{cc} +} + +func (c *sessionReduceClient) SessionReduceFn(ctx context.Context, opts ...grpc.CallOption) (SessionReduce_SessionReduceFnClient, error) { + stream, err := c.cc.NewStream(ctx, &SessionReduce_ServiceDesc.Streams[0], "/sessionreduce.v1.SessionReduce/SessionReduceFn", opts...) + if err != nil { + return nil, err + } + x := &sessionReduceSessionReduceFnClient{stream} + return x, nil +} + +type SessionReduce_SessionReduceFnClient interface { + Send(*SessionReduceRequest) error + Recv() (*SessionReduceResponse, error) + grpc.ClientStream +} + +type sessionReduceSessionReduceFnClient struct { + grpc.ClientStream +} + +func (x *sessionReduceSessionReduceFnClient) Send(m *SessionReduceRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *sessionReduceSessionReduceFnClient) Recv() (*SessionReduceResponse, error) { + m := new(SessionReduceResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *sessionReduceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + out := new(ReadyResponse) + err := c.cc.Invoke(ctx, "/sessionreduce.v1.SessionReduce/IsReady", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SessionReduceServer is the server API for SessionReduce service. +// All implementations must embed UnimplementedSessionReduceServer +// for forward compatibility +type SessionReduceServer interface { + // SessionReduceFn applies a reduce function to a request stream. + SessionReduceFn(SessionReduce_SessionReduceFnServer) error + // IsReady is the heartbeat endpoint for gRPC. + IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) + mustEmbedUnimplementedSessionReduceServer() +} + +// UnimplementedSessionReduceServer must be embedded to have forward compatible implementations. +type UnimplementedSessionReduceServer struct { +} + +func (UnimplementedSessionReduceServer) SessionReduceFn(SessionReduce_SessionReduceFnServer) error { + return status.Errorf(codes.Unimplemented, "method SessionReduceFn not implemented") +} +func (UnimplementedSessionReduceServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") +} +func (UnimplementedSessionReduceServer) mustEmbedUnimplementedSessionReduceServer() {} + +// UnsafeSessionReduceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SessionReduceServer will +// result in compilation errors. +type UnsafeSessionReduceServer interface { + mustEmbedUnimplementedSessionReduceServer() +} + +func RegisterSessionReduceServer(s grpc.ServiceRegistrar, srv SessionReduceServer) { + s.RegisterService(&SessionReduce_ServiceDesc, srv) +} + +func _SessionReduce_SessionReduceFn_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SessionReduceServer).SessionReduceFn(&sessionReduceSessionReduceFnServer{stream}) +} + +type SessionReduce_SessionReduceFnServer interface { + Send(*SessionReduceResponse) error + Recv() (*SessionReduceRequest, error) + grpc.ServerStream +} + +type sessionReduceSessionReduceFnServer struct { + grpc.ServerStream +} + +func (x *sessionReduceSessionReduceFnServer) Send(m *SessionReduceResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *sessionReduceSessionReduceFnServer) Recv() (*SessionReduceRequest, error) { + m := new(SessionReduceRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _SessionReduce_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(emptypb.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SessionReduceServer).IsReady(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/sessionreduce.v1.SessionReduce/IsReady", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SessionReduceServer).IsReady(ctx, req.(*emptypb.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// SessionReduce_ServiceDesc is the grpc.ServiceDesc for SessionReduce service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SessionReduce_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "sessionreduce.v1.SessionReduce", + HandlerType: (*SessionReduceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "IsReady", + Handler: _SessionReduce_IsReady_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "SessionReduceFn", + Handler: _SessionReduce_SessionReduceFn_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "pkg/apis/proto/sessionreduce/v1/sessionreduce.proto", +} diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go b/pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go new file mode 100644 index 00000000..d1572efb --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go @@ -0,0 +1,216 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1 (interfaces: SessionReduceClient,SessionReduce_SessionReduceFnClient) + +// Package sessionreducemock is a generated GoMock package. +package sessionreducemock + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" + grpc "google.golang.org/grpc" + metadata "google.golang.org/grpc/metadata" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// MockSessionReduceClient is a mock of SessionReduceClient interface. +type MockSessionReduceClient struct { + ctrl *gomock.Controller + recorder *MockSessionReduceClientMockRecorder +} + +// MockSessionReduceClientMockRecorder is the mock recorder for MockSessionReduceClient. +type MockSessionReduceClientMockRecorder struct { + mock *MockSessionReduceClient +} + +// NewMockSessionReduceClient creates a new mock instance. +func NewMockSessionReduceClient(ctrl *gomock.Controller) *MockSessionReduceClient { + mock := &MockSessionReduceClient{ctrl: ctrl} + mock.recorder = &MockSessionReduceClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSessionReduceClient) EXPECT() *MockSessionReduceClientMockRecorder { + return m.recorder +} + +// IsReady mocks base method. +func (m *MockSessionReduceClient) IsReady(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*v1.ReadyResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "IsReady", varargs...) + ret0, _ := ret[0].(*v1.ReadyResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsReady indicates an expected call of IsReady. +func (mr *MockSessionReduceClientMockRecorder) IsReady(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsReady", reflect.TypeOf((*MockSessionReduceClient)(nil).IsReady), varargs...) +} + +// SessionReduceFn mocks base method. +func (m *MockSessionReduceClient) SessionReduceFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.SessionReduce_SessionReduceFnClient, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "SessionReduceFn", varargs...) + ret0, _ := ret[0].(v1.SessionReduce_SessionReduceFnClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SessionReduceFn indicates an expected call of SessionReduceFn. +func (mr *MockSessionReduceClientMockRecorder) SessionReduceFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SessionReduceFn", reflect.TypeOf((*MockSessionReduceClient)(nil).SessionReduceFn), varargs...) +} + +// MockSessionReduce_SessionReduceFnClient is a mock of SessionReduce_SessionReduceFnClient interface. +type MockSessionReduce_SessionReduceFnClient struct { + ctrl *gomock.Controller + recorder *MockSessionReduce_SessionReduceFnClientMockRecorder +} + +// MockSessionReduce_SessionReduceFnClientMockRecorder is the mock recorder for MockSessionReduce_SessionReduceFnClient. +type MockSessionReduce_SessionReduceFnClientMockRecorder struct { + mock *MockSessionReduce_SessionReduceFnClient +} + +// NewMockSessionReduce_SessionReduceFnClient creates a new mock instance. +func NewMockSessionReduce_SessionReduceFnClient(ctrl *gomock.Controller) *MockSessionReduce_SessionReduceFnClient { + mock := &MockSessionReduce_SessionReduceFnClient{ctrl: ctrl} + mock.recorder = &MockSessionReduce_SessionReduceFnClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSessionReduce_SessionReduceFnClient) EXPECT() *MockSessionReduce_SessionReduceFnClientMockRecorder { + return m.recorder +} + +// CloseSend mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) CloseSend() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloseSend") + ret0, _ := ret[0].(error) + return ret0 +} + +// CloseSend indicates an expected call of CloseSend. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) CloseSend() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseSend", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).CloseSend)) +} + +// Context mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Context() context.Context { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Context") + ret0, _ := ret[0].(context.Context) + return ret0 +} + +// Context indicates an expected call of Context. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Context() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Context)) +} + +// Header mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Header() (metadata.MD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Header") + ret0, _ := ret[0].(metadata.MD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Header indicates an expected call of Header. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Header() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Header)) +} + +// Recv mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Recv() (*v1.SessionReduceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Recv") + ret0, _ := ret[0].(*v1.SessionReduceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Recv indicates an expected call of Recv. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Recv() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Recv)) +} + +// RecvMsg mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) RecvMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecvMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecvMsg indicates an expected call of RecvMsg. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).RecvMsg), arg0) +} + +// Send mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Send(arg0 *v1.SessionReduceRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Send(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Send), arg0) +} + +// SendMsg mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) SendMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMsg indicates an expected call of SendMsg. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).SendMsg), arg0) +} + +// Trailer mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Trailer() metadata.MD { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Trailer") + ret0, _ := ret[0].(metadata.MD) + return ret0 +} + +// Trailer indicates an expected call of Trailer. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Trailer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Trailer)) +} diff --git a/pkg/apis/proto/sideinput/v1/sideinput.pb.go b/pkg/apis/proto/sideinput/v1/sideinput.pb.go index 7f8b7280..95a32865 100644 --- a/pkg/apis/proto/sideinput/v1/sideinput.pb.go +++ b/pkg/apis/proto/sideinput/v1/sideinput.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v4.23.4 +// protoc v3.21.9 // source: pkg/apis/proto/sideinput/v1/sideinput.proto package v1 diff --git a/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go b/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go index 9d38558e..3f966308 100644 --- a/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go +++ b/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v4.23.4 +// - protoc v3.21.9 // source: pkg/apis/proto/sideinput/v1/sideinput.proto package v1 diff --git a/pkg/apis/proto/sink/v1/sink.pb.go b/pkg/apis/proto/sink/v1/sink.pb.go index 50658427..e746cedd 100644 --- a/pkg/apis/proto/sink/v1/sink.pb.go +++ b/pkg/apis/proto/sink/v1/sink.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v4.23.4 +// protoc v3.21.9 // source: pkg/apis/proto/sink/v1/sink.proto package v1 diff --git a/pkg/apis/proto/sink/v1/sink_grpc.pb.go b/pkg/apis/proto/sink/v1/sink_grpc.pb.go index 9af7b4e4..eddd9ef8 100644 --- a/pkg/apis/proto/sink/v1/sink_grpc.pb.go +++ b/pkg/apis/proto/sink/v1/sink_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v4.23.4 +// - protoc v3.21.9 // source: pkg/apis/proto/sink/v1/sink.proto package v1 diff --git a/pkg/apis/proto/source/v1/source.pb.go b/pkg/apis/proto/source/v1/source.pb.go index 1c8dcba4..e13121b2 100644 --- a/pkg/apis/proto/source/v1/source.pb.go +++ b/pkg/apis/proto/source/v1/source.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v4.23.4 +// protoc v3.21.9 // source: pkg/apis/proto/source/v1/source.proto package v1 diff --git a/pkg/apis/proto/source/v1/source_grpc.pb.go b/pkg/apis/proto/source/v1/source_grpc.pb.go index 79c769d6..1dce880a 100644 --- a/pkg/apis/proto/source/v1/source_grpc.pb.go +++ b/pkg/apis/proto/source/v1/source_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v4.23.4 +// - protoc v3.21.9 // source: pkg/apis/proto/source/v1/source.proto package v1 diff --git a/pkg/apis/proto/sourcetransform/v1/transform.pb.go b/pkg/apis/proto/sourcetransform/v1/transform.pb.go index f20a8d88..e38b6247 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform.pb.go +++ b/pkg/apis/proto/sourcetransform/v1/transform.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v4.23.4 +// protoc v3.21.9 // source: pkg/apis/proto/sourcetransform/v1/transform.proto package v1 diff --git a/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go b/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go index 5aac6e78..2bd5561b 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go +++ b/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v4.23.4 +// - protoc v3.21.9 // source: pkg/apis/proto/sourcetransform/v1/transform.proto package v1 diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go new file mode 100644 index 00000000..90d5b0fb --- /dev/null +++ b/pkg/globalreducer/task_manager.go @@ -0,0 +1,210 @@ +package globalreducer + +// +//import ( +// "context" +// "fmt" +// "strings" +// "sync" +// "time" +// +// v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" +//) +// +//type reduceTask struct { +// combinedKey string +// partition *v1.Partition +// reduceStreamer SessionReducer +// watermark time.Time +// inputCh chan Datum +// outputCh chan Message +// Done chan struct{} +//} +// +//func (rt *reduceTask) buildSessionReduceResponse(message Message) *v1.SessionReduceResponse { +// response := &v1.SessionReduceResponse{ +// Result: &v1.SessionReduceResponse_Result{ +// Keys: message.Keys(), +// Value: message.Value(), +// Tags: message.Tags(), +// }, +// Partition: rt.partition, +// CombinedKey: rt.combinedKey, +// Cob: false, +// } +// +// return response +//} +// +//func (rt *reduceTask) buildCobSessionReduceResponse() *v1.SessionReduceResponse { +// response := &v1.SessionReduceResponse{ +// Result: &v1.SessionReduceResponse_Result{ +// Keys: []string{}, +// Value: []byte{}, +// Tags: []string{}, +// }, +// Partition: rt.partition, +// CombinedKey: rt.combinedKey, +// Cob: true, +// } +// +// return response +//} +// +//func (rt *reduceTask) uniqueKey() string { +// return fmt.Sprintf("%d:%d:%s", +// rt.partition.GetStart().AsTime().UnixMilli(), +// rt.partition.GetEnd().AsTime().UnixMilli(), +// rt.combinedKey) +//} +// +//type reduceTaskManager struct { +// Tasks map[string]*reduceTask +// Output chan *v1.SessionReduceResponse +// Mutex sync.RWMutex +//} +// +//func newReduceTaskManager() *reduceTaskManager { +// return &reduceTaskManager{ +// Tasks: make(map[string]*reduceTask), +// Output: make(chan *v1.SessionReduceResponse), +// } +//} +// +//func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, reduceStreamer SessionReducer, partition *v1.Partition) { +// rtm.Mutex.Lock() +// defer rtm.Mutex.Unlock() +// +// task := &reduceTask{ +// combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), +// partition: partition, +// reduceStreamer: reduceStreamer, +// inputCh: make(chan Datum), +// outputCh: make(chan Message), +// Done: make(chan struct{}), +// } +// +// go func() { +// defer close(task.Done) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// defer wg.Done() +// for { +// select { +// case <-ctx.Done(): +// return +// case message, ok := <-task.outputCh: +// if !ok { +// rtm.Output <- task.buildCobSessionReduceResponse() +// return +// } +// rtm.Output <- task.buildSessionReduceResponse(message) +// } +// } +// }() +// reduceStreamer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) +// close(task.outputCh) +// wg.Wait() +// }() +// +// rtm.Tasks[task.uniqueKey()] = task +//} +// +//func (rtm *reduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) error { +// rtm.Mutex.RLock() +// task, ok := rtm.Tasks[generateKey(request.Operation.Partitions[0], request.Payload.Keys)] +// rtm.Mutex.RUnlock() +// +// if !ok { +// return fmt.Errorf("task not found") +// } +// +// task.inputCh <- buildDatum(request) +// return nil +//} +// +//func (rtm *reduceTaskManager) CloseTask(request *v1.SessionReduceRequest) error { +// rtm.Mutex.Lock() +// tasksToBeClosed := make([]*reduceTask, 0, len(request.Operation.Partitions)) +// for _, partition := range request.Operation.Partitions { +// key := generateKey(partition, request.Payload.Keys) +// task, ok := rtm.Tasks[key] +// if ok { +// tasksToBeClosed = append(tasksToBeClosed, task) +// } +// delete(rtm.Tasks, key) +// } +// rtm.Mutex.Unlock() +// +// for _, task := range tasksToBeClosed { +// close(task.inputCh) +// } +// +// return nil +//} +// +//func (rtm *reduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { +// rtm.Mutex.Lock() +// tasks := make([]*reduceTask, 0, len(request.Operation.Partitions)) +// for _, partition := range request.Operation.Partitions { +// key := generateKey(partition, request.Payload.Keys) +// task, ok := rtm.Tasks[key] +// if !ok { +// rtm.Mutex.Unlock() +// return fmt.Errorf("task not found") +// } +// tasks = append(tasks, task) +// } +// rtm.Mutex.Unlock() +// +// if len(tasks) == 0 { +// return nil +// } +// +// mainTask := tasks[0] +// aggregators := make([][]byte, 0, len(tasks)-1) +// +// for _, task := range tasks[1:] { +// close(task.inputCh) +// aggregators = append(aggregators, task.reduceStreamer.Aggregator(ctx)) +// } +// +// for _, aggregator := range aggregators { +// mainTask.reduceStreamer.MergeAggregator(ctx, aggregator) +// } +// +// if request.Payload != nil { +// mainTask.inputCh <- buildDatum(request) +// } +// +// return nil +//} +// +//func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { +// return rtm.Output +//} +// +//func (rtm *reduceTaskManager) WaitAll() { +// rtm.Mutex.RLock() +// tasks := make([]*reduceTask, 0, len(rtm.Tasks)) +// for _, task := range rtm.Tasks { +// tasks = append(tasks, task) +// } +// rtm.Mutex.RUnlock() +// +// for _, task := range tasks { +// <-task.Done +// } +//} +// +//func generateKey(partition *v1.Partition, keys []string) string { +// return fmt.Sprintf("%d:%d:%s", +// partition.GetStart().AsTime().UnixMilli(), +// partition.GetEnd().AsTime().UnixMilli(), +// strings.Join(keys, delimiter)) +//} +// +//func buildDatum(request *v1.SessionReduceRequest) Datum { +// return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +//} diff --git a/pkg/sessionreducer/doc.go b/pkg/sessionreducer/doc.go new file mode 100644 index 00000000..dc190ed2 --- /dev/null +++ b/pkg/sessionreducer/doc.go @@ -0,0 +1,5 @@ +// Package reducer implements the server code for reduce operation. + +// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/ + +package sessionreducer diff --git a/pkg/sessionreducer/interface.go b/pkg/sessionreducer/interface.go new file mode 100644 index 00000000..66930a49 --- /dev/null +++ b/pkg/sessionreducer/interface.go @@ -0,0 +1,22 @@ +package sessionreducer + +import ( + "context" + "time" +) + +// Datum contains methods to get the payload information. +type Datum interface { + Value() []byte + EventTime() time.Time + Watermark() time.Time +} + +// SessionReducer is the interface which can be used to implement a session reduce operation. +type SessionReducer interface { + SessionReduce(ctx context.Context, keys []string, input <-chan Datum) Messages + Aggregator(ctx context.Context) []byte + MergeAggregator(ctx context.Context, aggregator []byte) +} + +type CreateSessionReducer func() SessionReducer diff --git a/pkg/sessionreducer/message.go b/pkg/sessionreducer/message.go new file mode 100644 index 00000000..e00d8659 --- /dev/null +++ b/pkg/sessionreducer/message.go @@ -0,0 +1,70 @@ +package sessionreducer + +import "fmt" + +var ( + DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ +) + +// Message is used to wrap the data return by reduce functions +type Message struct { + value []byte + keys []string + tags []string +} + +// NewMessage creates a Message with value +func NewMessage(value []byte) Message { + return Message{value: value} +} + +// MessageToDrop creates a Message to be dropped +func MessageToDrop() Message { + return Message{value: []byte{}, tags: []string{DROP}} +} + +// WithKeys is used to assign the keys to the message +func (m Message) WithKeys(keys []string) Message { + m.keys = keys + return m +} + +// WithTags is used to assign the tags to the message +// tags will be used for conditional forwarding +func (m Message) WithTags(tags []string) Message { + m.tags = tags + return m +} + +// Keys returns message keys +func (m Message) Keys() []string { + return m.keys +} + +// Value returns message value +func (m Message) Value() []byte { + return m.value +} + +// Tags returns message tags +func (m Message) Tags() []string { + return m.tags +} + +type Messages []Message + +// MessagesBuilder returns an empty instance of Messages +func MessagesBuilder() Messages { + return Messages{} +} + +// Append appends a Message +func (m Messages) Append(msg Message) Messages { + m = append(m, msg) + return m +} + +// Items returns the message list +func (m Messages) Items() []Message { + return m +} diff --git a/pkg/sessionreducer/options.go b/pkg/sessionreducer/options.go new file mode 100644 index 00000000..6e5da5b7 --- /dev/null +++ b/pkg/sessionreducer/options.go @@ -0,0 +1,43 @@ +package sessionreducer + +import ( + "github.com/numaproj/numaflow-go/pkg/info" +) + +type options struct { + sockAddr string + maxMessageSize int + serverInfoFilePath string +} + +// Option is the interface to apply options. +type Option func(*options) + +func DefaultOptions() *options { + return &options{ + sockAddr: address, + maxMessageSize: defaultMaxMessageSize, + serverInfoFilePath: info.ServerInfoFilePath, + } +} + +// WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. +func WithMaxMessageSize(size int) Option { + return func(opts *options) { + opts.maxMessageSize = size + } +} + +// WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. +func WithSockAddr(addr string) Option { + return func(opts *options) { + opts.sockAddr = addr + } +} + +// WithServerInfoFilePath sets the server info file path to the given path. +func WithServerInfoFilePath(f string) Option { + return func(opts *options) { + opts.serverInfoFilePath = f + } +} diff --git a/pkg/sessionreducer/options_test.go b/pkg/sessionreducer/options_test.go new file mode 100644 index 00000000..9dfbcfbf --- /dev/null +++ b/pkg/sessionreducer/options_test.go @@ -0,0 +1,18 @@ +package sessionreducer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithMaxMessageSize(t *testing.T) { + var ( + size = 1024 * 1024 * 10 + opts = &options{ + maxMessageSize: defaultMaxMessageSize, + } + ) + WithMaxMessageSize(1024 * 1024 * 10)(opts) + assert.Equal(t, size, opts.maxMessageSize) +} diff --git a/pkg/sessionreducer/server.go b/pkg/sessionreducer/server.go new file mode 100644 index 00000000..c7aa1634 --- /dev/null +++ b/pkg/sessionreducer/server.go @@ -0,0 +1,56 @@ +package sessionreducer + +import ( + "context" + "fmt" + "os/signal" + "syscall" + + "github.com/numaproj/numaflow-go/pkg" + sessionreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" + "github.com/numaproj/numaflow-go/pkg/shared" +) + +// server is a session reduce gRPC server. +type server struct { + svc *Service + opts *options +} + +// NewServer creates a new session reduce server. +func NewServer(r CreateSessionReducer, inputOptions ...Option) numaflow.Server { + opts := DefaultOptions() + for _, inputOption := range inputOptions { + inputOption(opts) + } + s := new(server) + s.svc = new(Service) + s.svc.createSessionReducer = r + s.opts = opts + return s +} + +// Start starts the session reduce gRPC server. +func (r *server) Start(ctx context.Context) error { + ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // write server info to the file + // start listening on unix domain socket + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath) + if err != nil { + return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) + } + // close the listener + defer func() { _ = lis.Close() }() + + // create a grpc server + grpcServer := shared.CreateGRPCServer(r.opts.maxMessageSize) + defer grpcServer.GracefulStop() + + // register the sessionReduce service + sessionreducepb.RegisterSessionReduceServer(grpcServer, r.svc) + + // start the grpc server + return shared.StartGRPCServer(ctxWithSignal, grpcServer, lis) +} diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go new file mode 100644 index 00000000..3397d2d1 --- /dev/null +++ b/pkg/sessionreducer/service.go @@ -0,0 +1,115 @@ +package sessionreducer + +import ( + "context" + "io" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + sessionreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" +) + +const ( + uds = "unix" + defaultMaxMessageSize = 1024 * 1024 * 64 + address = "/var/run/numaflow/sesionreduce.sock" + delimiter = ":" +) + +// Service implements the proto gen server interface and contains the sesionreduce operation handler. +type Service struct { + sessionreducepb.UnimplementedSessionReduceServer + createSessionReducer CreateSessionReducer +} + +// IsReady returns true to indicate the gRPC connection is ready. +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*sessionreducepb.ReadyResponse, error) { + return &sessionreducepb.ReadyResponse{Ready: true}, nil +} + +// SessionReduceFn applies a session reduce function to a request stream and streams the results. +func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionReduceFnServer) error { + + ctx := stream.Context() + taskManager := newReduceTaskManager() + // err group for the go routine which reads from the output channel and sends to the stream + var g errgroup.Group + + g.Go(func() error { + for output := range taskManager.OutputChannel() { + err := stream.Send(output) + if err != nil { + return err + } + } + return nil + }) + + for { + d, recvErr := stream.Recv() + + // if the stream is closed, close all the tasks and break + if recvErr == io.EOF { + taskManager.CloseAll() + break + } + + if recvErr != nil { + statusErr := status.Errorf(codes.Internal, recvErr.Error()) + return statusErr + } + + // invoke the appropriate task manager method based on the operation + switch d.Operation.Event { + case sessionreducepb.SessionReduceRequest_WindowOperation_OPEN: + // create a new task and start the session reduce operation + // also append the datum to the task + taskManager.CreateTask(ctx, d, fs.createSessionReducer()) + err := taskManager.AppendToTask(d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE: + // close the task + taskManager.CloseTask(d) + case sessionreducepb.SessionReduceRequest_WindowOperation_APPEND: + // append the datum to the task + err := taskManager.AppendToTask(d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case sessionreducepb.SessionReduceRequest_WindowOperation_MERGE: + // merge the tasks + err := taskManager.MergeTasks(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND: + // expand the task + err := taskManager.ExpandTask(d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + } + + } + + // wait for all the tasks to return + taskManager.WaitAll() + + // wait for the go routine which reads from the output channel and sends to the stream to return + err := g.Wait() + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + + return nil +} diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go new file mode 100644 index 00000000..bc632bad --- /dev/null +++ b/pkg/sessionreducer/task_manager.go @@ -0,0 +1,227 @@ +package sessionreducer + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" + + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" +) + +// reduceTask represents a reduce task for a session reduce operation. +type reduceTask struct { + combinedKey string + partition *v1.Partition + reduceStreamer SessionReducer + watermark time.Time + inputCh chan Datum + Done chan struct{} +} + +// buildSessionReduceResponse builds the session reduce response from the messages. +func (rt *reduceTask) buildSessionReduceResponse(messages Messages) *v1.SessionReduceResponse { + result := make([]*v1.SessionReduceResponse_Result, 0, len(messages)) + for _, message := range messages { + result = append(result, &v1.SessionReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + }) + } + + response := &v1.SessionReduceResponse{ + Result: result, + Partition: rt.partition, + CombinedKey: rt.combinedKey, + EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + } + + return response +} + +// uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. +func (rt *reduceTask) uniqueKey() string { + return fmt.Sprintf("%d:%d:%s", + rt.partition.GetStart().AsTime().UnixMilli(), + rt.partition.GetEnd().AsTime().UnixMilli(), + rt.combinedKey) +} + +// reduceTaskManager manages the reduce tasks for a session reduce operation. +type reduceTaskManager struct { + Tasks map[string]*reduceTask + Output chan *v1.SessionReduceResponse + Mutex sync.RWMutex +} + +func newReduceTaskManager() *reduceTaskManager { + return &reduceTaskManager{ + Tasks: make(map[string]*reduceTask), + Output: make(chan *v1.SessionReduceResponse), + } +} + +// CreateTask creates a new reduce task and starts the session reduce operation. +func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, reduceStreamer SessionReducer) { + rtm.Mutex.Lock() + defer rtm.Mutex.Unlock() + + task := &reduceTask{ + combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), + partition: request.Operation.Partitions[0], + reduceStreamer: reduceStreamer, + inputCh: make(chan Datum), + Done: make(chan struct{}), + } + + go func() { + defer close(task.Done) + msgs := reduceStreamer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh) + rtm.Output <- task.buildSessionReduceResponse(msgs) + }() + + rtm.Tasks[task.uniqueKey()] = task +} + +// AppendToTask writes the message to the reduce task. +func (rtm *reduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) error { + rtm.Mutex.RLock() + task, ok := rtm.Tasks[generateKey(request.Operation.Partitions[0], request.Payload.Keys)] + rtm.Mutex.RUnlock() + + if !ok { + return fmt.Errorf("task not found") + } + + task.inputCh <- buildDatum(request) + return nil +} + +// CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. +func (rtm *reduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { + rtm.Mutex.Lock() + tasksToBeClosed := make([]*reduceTask, 0, len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + key := generateKey(partition, request.Payload.Keys) + task, ok := rtm.Tasks[key] + if ok { + tasksToBeClosed = append(tasksToBeClosed, task) + } + delete(rtm.Tasks, key) + } + rtm.Mutex.Unlock() + + for _, task := range tasksToBeClosed { + close(task.inputCh) + } +} + +// MergeTasks merges the session reduce tasks. It will invoke close on all the tasks except the main task. The main task will +// merge the aggregators from the other tasks. The main task will be first task in the list of tasks. +func (rtm *reduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { + rtm.Mutex.Lock() + tasks := make([]*reduceTask, 0, len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + key := generateKey(partition, request.Payload.Keys) + task, ok := rtm.Tasks[key] + if !ok { + rtm.Mutex.Unlock() + return fmt.Errorf("task not found") + } + tasks = append(tasks, task) + } + rtm.Mutex.Unlock() + + if len(tasks) == 0 { + return nil + } + + mainTask := tasks[0] + aggregators := make([][]byte, 0, len(tasks)-1) + + for _, task := range tasks[1:] { + close(task.inputCh) + aggregators = append(aggregators, task.reduceStreamer.Aggregator(ctx)) + } + + for _, aggregator := range aggregators { + mainTask.reduceStreamer.MergeAggregator(ctx, aggregator) + } + + if request.Payload != nil { + mainTask.inputCh <- buildDatum(request) + } + + return nil +} + +// ExpandTask expands the reduce task. It will expand the window for the reduce task. +// deletes the old task and creates a new task with the new window. +// expects request.Operation.Partitions to have exactly two partitions. The first partition is the old partition and the second +// partition is the new partition. +func (rtm *reduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { + if len(request.Operation.Partitions) != 2 { + return fmt.Errorf("expected exactly two partitions") + } + rtm.Mutex.Lock() + key := generateKey(request.Operation.Partitions[0], request.Payload.Keys) + task, ok := rtm.Tasks[key] + task.partition = request.Operation.Partitions[1] + delete(rtm.Tasks, key) + rtm.Tasks[task.uniqueKey()] = task + rtm.Mutex.Unlock() + + if !ok { + return fmt.Errorf("task not found") + } + + return nil +} + +// OutputChannel returns the output channel for the reduce task manager to read the results. +func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { + return rtm.Output +} + +// WaitAll waits for all the reduce tasks to complete. +func (rtm *reduceTaskManager) WaitAll() { + rtm.Mutex.RLock() + tasks := make([]*reduceTask, 0, len(rtm.Tasks)) + for _, task := range rtm.Tasks { + tasks = append(tasks, task) + } + rtm.Mutex.RUnlock() + + for _, task := range tasks { + <-task.Done + } +} + +// CloseAll closes all the reduce tasks. +func (rtm *reduceTaskManager) CloseAll() { + rtm.Mutex.Lock() + tasks := make([]*reduceTask, 0, len(rtm.Tasks)) + for _, task := range rtm.Tasks { + tasks = append(tasks, task) + } + rtm.Mutex.Unlock() + + for _, task := range tasks { + close(task.inputCh) + } +} + +func generateKey(partition *v1.Partition, keys []string) string { + return fmt.Sprintf("%d:%d:%s", + partition.GetStart().AsTime().UnixMilli(), + partition.GetEnd().AsTime().UnixMilli(), + strings.Join(keys, delimiter)) +} + +func buildDatum(request *v1.SessionReduceRequest) Datum { + return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +} diff --git a/pkg/sessionreducer/types.go b/pkg/sessionreducer/types.go new file mode 100644 index 00000000..6266c01a --- /dev/null +++ b/pkg/sessionreducer/types.go @@ -0,0 +1,65 @@ +package sessionreducer + +import "time" + +// handlerDatum implements the Datum interface and is used in the reduce functions. +type handlerDatum struct { + value []byte + eventTime time.Time + watermark time.Time +} + +func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time) Datum { + return &handlerDatum{ + value: value, + eventTime: eventTime, + watermark: watermark, + } +} + +func (h *handlerDatum) Value() []byte { + return h.value +} + +func (h *handlerDatum) EventTime() time.Time { + return h.eventTime +} + +func (h *handlerDatum) Watermark() time.Time { + return h.watermark +} + +// intervalWindow implements IntervalWindow interface which will be passed as metadata +// to reduce handlers +type intervalWindow struct { + startTime time.Time + endTime time.Time +} + +func NewIntervalWindow(startTime time.Time, endTime time.Time) IntervalWindow { + return &intervalWindow{ + startTime: startTime, + endTime: endTime, + } +} + +func (i *intervalWindow) StartTime() time.Time { + return i.startTime +} + +func (i *intervalWindow) EndTime() time.Time { + return i.endTime +} + +// metadata implements Metadata interface which will be passed to reduce function. +type metadata struct { + intervalWindow IntervalWindow +} + +func NewMetadata(window IntervalWindow) Metadata { + return &metadata{intervalWindow: window} +} + +func (m *metadata) IntervalWindow() IntervalWindow { + return m.intervalWindow +} From 7325fc205f651931e2e017b40e61fc8c526d5ee0 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 7 Nov 2023 16:06:10 +0530 Subject: [PATCH 02/27] refactor existing reducer Signed-off-by: Yashash H L --- pkg/apis/proto/reduce/v1/reduce.pb.go | 489 +++++++++++++++--- pkg/apis/proto/reduce/v1/reduce.proto | 42 +- .../sessionreduce/v1/sessionreduce.pb.go | 308 +++++------ .../sessionreduce/v1/sessionreduce.proto | 20 +- pkg/reducer/service.go | 149 ++---- pkg/reducer/service_test.go | 326 +++++++++--- pkg/reducer/task_manager.go | 178 +++++++ pkg/sessionreducer/service.go | 5 +- pkg/sessionreducer/task_manager.go | 60 ++- 9 files changed, 1093 insertions(+), 484 deletions(-) create mode 100644 pkg/reducer/task_manager.go diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index e1963f5d..e24d241a 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -22,6 +22,55 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ReduceRequest_WindowOperation_Event int32 + +const ( + ReduceRequest_WindowOperation_OPEN ReduceRequest_WindowOperation_Event = 0 + ReduceRequest_WindowOperation_CLOSE ReduceRequest_WindowOperation_Event = 1 + ReduceRequest_WindowOperation_APPEND ReduceRequest_WindowOperation_Event = 4 +) + +// Enum value maps for ReduceRequest_WindowOperation_Event. +var ( + ReduceRequest_WindowOperation_Event_name = map[int32]string{ + 0: "OPEN", + 1: "CLOSE", + 4: "APPEND", + } + ReduceRequest_WindowOperation_Event_value = map[string]int32{ + "OPEN": 0, + "CLOSE": 1, + "APPEND": 4, + } +) + +func (x ReduceRequest_WindowOperation_Event) Enum() *ReduceRequest_WindowOperation_Event { + p := new(ReduceRequest_WindowOperation_Event) + *p = x + return p +} + +func (x ReduceRequest_WindowOperation_Event) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ReduceRequest_WindowOperation_Event) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes[0].Descriptor() +} + +func (ReduceRequest_WindowOperation_Event) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes[0] +} + +func (x ReduceRequest_WindowOperation_Event) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ReduceRequest_WindowOperation_Event.Descriptor instead. +func (ReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0, 0, 0} +} + // * // ReduceRequest represents a request element. type ReduceRequest struct { @@ -29,10 +78,8 @@ type ReduceRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` + Payload *ReduceRequest_Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + Operation *ReduceRequest_WindowOperation `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` } func (x *ReduceRequest) Reset() { @@ -67,34 +114,84 @@ func (*ReduceRequest) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0} } -func (x *ReduceRequest) GetKeys() []string { +func (x *ReduceRequest) GetPayload() *ReduceRequest_Payload { if x != nil { - return x.Keys + return x.Payload } return nil } -func (x *ReduceRequest) GetValue() []byte { +func (x *ReduceRequest) GetOperation() *ReduceRequest_WindowOperation { if x != nil { - return x.Value + return x.Operation } return nil } -func (x *ReduceRequest) GetEventTime() *timestamppb.Timestamp { +// Partition represents a window partition. +type Partition struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` + End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` + Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` +} + +func (x *Partition) Reset() { + *x = Partition{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Partition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Partition) ProtoMessage() {} + +func (x *Partition) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Partition.ProtoReflect.Descriptor instead. +func (*Partition) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{1} +} + +func (x *Partition) GetStart() *timestamppb.Timestamp { if x != nil { - return x.EventTime + return x.Start } return nil } -func (x *ReduceRequest) GetWatermark() *timestamppb.Timestamp { +func (x *Partition) GetEnd() *timestamppb.Timestamp { if x != nil { - return x.Watermark + return x.End } return nil } +func (x *Partition) GetSlot() string { + if x != nil { + return x.Slot + } + return "" +} + // * // ReduceResponse represents a response element. type ReduceResponse struct { @@ -103,12 +200,16 @@ type ReduceResponse struct { unknownFields protoimpl.UnknownFields Results []*ReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + // Partition represents a window partition to which the result belongs. + Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` + // EventTime represents the event time of the result. + EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } func (x *ReduceResponse) Reset() { *x = ReduceResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -121,7 +222,7 @@ func (x *ReduceResponse) String() string { func (*ReduceResponse) ProtoMessage() {} func (x *ReduceResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -134,7 +235,7 @@ func (x *ReduceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReduceResponse.ProtoReflect.Descriptor instead. func (*ReduceResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{1} + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{2} } func (x *ReduceResponse) GetResults() []*ReduceResponse_Result { @@ -144,6 +245,20 @@ func (x *ReduceResponse) GetResults() []*ReduceResponse_Result { return nil } +func (x *ReduceResponse) GetPartition() *Partition { + if x != nil { + return x.Partition + } + return nil +} + +func (x *ReduceResponse) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + // * // ReadyResponse is the health check result. type ReadyResponse struct { @@ -157,7 +272,7 @@ type ReadyResponse struct { func (x *ReadyResponse) Reset() { *x = ReadyResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -170,7 +285,7 @@ func (x *ReadyResponse) String() string { func (*ReadyResponse) ProtoMessage() {} func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -183,7 +298,7 @@ func (x *ReadyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{2} + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{3} } func (x *ReadyResponse) GetReady() bool { @@ -193,6 +308,136 @@ func (x *ReadyResponse) GetReady() bool { return false } +// WindowOperation represents a window operation. +// for fixed and sliding windows, OPEN, APPEND and CLOSE events are sent. +type ReduceRequest_WindowOperation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Event ReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=reduce.v1.ReduceRequest_WindowOperation_Event" json:"event,omitempty"` + Partitions []*Partition `protobuf:"bytes,2,rep,name=partitions,proto3" json:"partitions,omitempty"` +} + +func (x *ReduceRequest_WindowOperation) Reset() { + *x = ReduceRequest_WindowOperation{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReduceRequest_WindowOperation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReduceRequest_WindowOperation) ProtoMessage() {} + +func (x *ReduceRequest_WindowOperation) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReduceRequest_WindowOperation.ProtoReflect.Descriptor instead. +func (*ReduceRequest_WindowOperation) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *ReduceRequest_WindowOperation) GetEvent() ReduceRequest_WindowOperation_Event { + if x != nil { + return x.Event + } + return ReduceRequest_WindowOperation_OPEN +} + +func (x *ReduceRequest_WindowOperation) GetPartitions() []*Partition { + if x != nil { + return x.Partitions + } + return nil +} + +// Payload represents a payload element. +type ReduceRequest_Payload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` +} + +func (x *ReduceRequest_Payload) Reset() { + *x = ReduceRequest_Payload{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReduceRequest_Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReduceRequest_Payload) ProtoMessage() {} + +func (x *ReduceRequest_Payload) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReduceRequest_Payload.ProtoReflect.Descriptor instead. +func (*ReduceRequest_Payload) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0, 1} +} + +func (x *ReduceRequest_Payload) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *ReduceRequest_Payload) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *ReduceRequest_Payload) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *ReduceRequest_Payload) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +// Result represents a result element. It contains the result of the reduce function. type ReduceResponse_Result struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -206,7 +451,7 @@ type ReduceResponse_Result struct { func (x *ReduceResponse_Result) Reset() { *x = ReduceResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -219,7 +464,7 @@ func (x *ReduceResponse_Result) String() string { func (*ReduceResponse_Result) ProtoMessage() {} func (x *ReduceResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -232,7 +477,7 @@ func (x *ReduceResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use ReduceResponse_Result.ProtoReflect.Descriptor instead. func (*ReduceResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{1, 0} + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{2, 0} } func (x *ReduceResponse_Result) GetKeys() []string { @@ -266,42 +511,78 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0xae, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, - 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x22, 0xf8, 0x03, 0x0a, 0x0d, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x46, + 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xb7, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x05, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x72, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x12, 0x34, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, + 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, + 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, + 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, - 0x6b, 0x22, 0x94, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, - 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, - 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, - 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, - 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, - 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, - 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0x7f, 0x0a, 0x09, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x22, 0x83, 0x02, 0x0a, + 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x3a, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x09, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, + 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, + 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, + 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -316,28 +597,41 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP() []byte { return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescData } -var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []interface{}{ - (*ReduceRequest)(nil), // 0: reduce.v1.ReduceRequest - (*ReduceResponse)(nil), // 1: reduce.v1.ReduceResponse - (*ReadyResponse)(nil), // 2: reduce.v1.ReadyResponse - (*ReduceResponse_Result)(nil), // 3: reduce.v1.ReduceResponse.Result - (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 5: google.protobuf.Empty + (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Event + (*ReduceRequest)(nil), // 1: reduce.v1.ReduceRequest + (*Partition)(nil), // 2: reduce.v1.Partition + (*ReduceResponse)(nil), // 3: reduce.v1.ReduceResponse + (*ReadyResponse)(nil), // 4: reduce.v1.ReadyResponse + (*ReduceRequest_WindowOperation)(nil), // 5: reduce.v1.ReduceRequest.WindowOperation + (*ReduceRequest_Payload)(nil), // 6: reduce.v1.ReduceRequest.Payload + (*ReduceResponse_Result)(nil), // 7: reduce.v1.ReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs = []int32{ - 4, // 0: reduce.v1.ReduceRequest.event_time:type_name -> google.protobuf.Timestamp - 4, // 1: reduce.v1.ReduceRequest.watermark:type_name -> google.protobuf.Timestamp - 3, // 2: reduce.v1.ReduceResponse.results:type_name -> reduce.v1.ReduceResponse.Result - 0, // 3: reduce.v1.Reduce.ReduceFn:input_type -> reduce.v1.ReduceRequest - 5, // 4: reduce.v1.Reduce.IsReady:input_type -> google.protobuf.Empty - 1, // 5: reduce.v1.Reduce.ReduceFn:output_type -> reduce.v1.ReduceResponse - 2, // 6: reduce.v1.Reduce.IsReady:output_type -> reduce.v1.ReadyResponse - 5, // [5:7] is the sub-list for method output_type - 3, // [3:5] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 6, // 0: reduce.v1.ReduceRequest.payload:type_name -> reduce.v1.ReduceRequest.Payload + 5, // 1: reduce.v1.ReduceRequest.operation:type_name -> reduce.v1.ReduceRequest.WindowOperation + 8, // 2: reduce.v1.Partition.start:type_name -> google.protobuf.Timestamp + 8, // 3: reduce.v1.Partition.end:type_name -> google.protobuf.Timestamp + 7, // 4: reduce.v1.ReduceResponse.results:type_name -> reduce.v1.ReduceResponse.Result + 2, // 5: reduce.v1.ReduceResponse.partition:type_name -> reduce.v1.Partition + 8, // 6: reduce.v1.ReduceResponse.event_time:type_name -> google.protobuf.Timestamp + 0, // 7: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Event + 2, // 8: reduce.v1.ReduceRequest.WindowOperation.partitions:type_name -> reduce.v1.Partition + 8, // 9: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 10: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 1, // 11: reduce.v1.Reduce.ReduceFn:input_type -> reduce.v1.ReduceRequest + 9, // 12: reduce.v1.Reduce.IsReady:input_type -> google.protobuf.Empty + 3, // 13: reduce.v1.Reduce.ReduceFn:output_type -> reduce.v1.ReduceResponse + 4, // 14: reduce.v1.Reduce.IsReady:output_type -> reduce.v1.ReadyResponse + 13, // [13:15] is the sub-list for method output_type + 11, // [11:13] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_pkg_apis_proto_reduce_v1_reduce_proto_init() } @@ -359,7 +653,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { } } file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReduceResponse); i { + switch v := v.(*Partition); i { case 0: return &v.state case 1: @@ -371,7 +665,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { } } file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadyResponse); i { + switch v := v.(*ReduceResponse); i { case 0: return &v.state case 1: @@ -383,6 +677,42 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { } } file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReadyResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReduceRequest_WindowOperation); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReduceRequest_Payload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ReduceResponse_Result); i { case 0: return &v.state @@ -400,13 +730,14 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc, - NumEnums: 0, - NumMessages: 4, + NumEnums: 1, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, GoTypes: file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes, DependencyIndexes: file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs, + EnumInfos: file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes, MessageInfos: file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes, }.Build() File_pkg_apis_proto_reduce_v1_reduce_proto = out.File diff --git a/pkg/apis/proto/reduce/v1/reduce.proto b/pkg/apis/proto/reduce/v1/reduce.proto index 5f596a81..3bafc9f7 100644 --- a/pkg/apis/proto/reduce/v1/reduce.proto +++ b/pkg/apis/proto/reduce/v1/reduce.proto @@ -20,22 +20,56 @@ service Reduce { * ReduceRequest represents a request element. */ message ReduceRequest { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; + // WindowOperation represents a window operation. + // for fixed and sliding windows, OPEN, APPEND and CLOSE events are sent. + message WindowOperation { + enum Event { + OPEN = 0; + CLOSE = 1; + APPEND = 4; + } + + Event event = 1; + repeated Partition partitions = 2; + } + + // Payload represents a payload element. + message Payload { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + } + + Payload payload = 1; + WindowOperation operation = 2; +} + +// Partition represents a window partition. +message Partition { + google.protobuf.Timestamp start = 1; + google.protobuf.Timestamp end = 2; + string slot = 3; } /** * ReduceResponse represents a response element. */ message ReduceResponse { + // Result represents a result element. It contains the result of the reduce function. message Result { repeated string keys = 1; bytes value = 2; repeated string tags = 3; } + repeated Result results = 1; + + // Partition represents a window partition to which the result belongs. + Partition partition = 2; + + // EventTime represents the event time of the result. + google.protobuf.Timestamp event_time = 4; } /** diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index 6a892b74..d777356d 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -22,60 +22,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// WindowType represents a window type. -// REVISIT Do we need this? Can this be passed using metadata? -type SessionReduceRequest_WindowType int32 - -const ( - SessionReduceRequest_FIXED SessionReduceRequest_WindowType = 0 - SessionReduceRequest_SLIDING SessionReduceRequest_WindowType = 1 - SessionReduceRequest_SESSION SessionReduceRequest_WindowType = 2 - SessionReduceRequest_GLOBAL SessionReduceRequest_WindowType = 3 -) - -// Enum value maps for SessionReduceRequest_WindowType. -var ( - SessionReduceRequest_WindowType_name = map[int32]string{ - 0: "FIXED", - 1: "SLIDING", - 2: "SESSION", - 3: "GLOBAL", - } - SessionReduceRequest_WindowType_value = map[string]int32{ - "FIXED": 0, - "SLIDING": 1, - "SESSION": 2, - "GLOBAL": 3, - } -) - -func (x SessionReduceRequest_WindowType) Enum() *SessionReduceRequest_WindowType { - p := new(SessionReduceRequest_WindowType) - *p = x - return p -} - -func (x SessionReduceRequest_WindowType) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (SessionReduceRequest_WindowType) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0].Descriptor() -} - -func (SessionReduceRequest_WindowType) Type() protoreflect.EnumType { - return &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0] -} - -func (x SessionReduceRequest_WindowType) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use SessionReduceRequest_WindowType.Descriptor instead. -func (SessionReduceRequest_WindowType) EnumDescriptor() ([]byte, []int) { - return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0} -} - type SessionReduceRequest_WindowOperation_Event int32 const ( @@ -115,11 +61,11 @@ func (x SessionReduceRequest_WindowOperation_Event) String() string { } func (SessionReduceRequest_WindowOperation_Event) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[1].Descriptor() + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0].Descriptor() } func (SessionReduceRequest_WindowOperation_Event) Type() protoreflect.EnumType { - return &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[1] + return &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0] } func (x SessionReduceRequest_WindowOperation_Event) Number() protoreflect.EnumNumber { @@ -131,6 +77,7 @@ func (SessionReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []in return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0, 0} } +// Partition represents a window partition. type Partition struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -138,7 +85,7 @@ type Partition struct { Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` - Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` } func (x *Partition) Reset() { @@ -187,9 +134,9 @@ func (x *Partition) GetEnd() *timestamppb.Timestamp { return nil } -func (x *Partition) GetKey() string { +func (x *Partition) GetSlot() string { if x != nil { - return x.Key + return x.Slot } return "" } @@ -203,7 +150,6 @@ type SessionReduceRequest struct { Payload *SessionReduceRequest_Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` Operation *SessionReduceRequest_WindowOperation `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` - Type SessionReduceRequest_WindowType `protobuf:"varint,3,opt,name=type,proto3,enum=sessionreduce.v1.SessionReduceRequest_WindowType" json:"type,omitempty"` } func (x *SessionReduceRequest) Reset() { @@ -252,13 +198,6 @@ func (x *SessionReduceRequest) GetOperation() *SessionReduceRequest_WindowOperat return nil } -func (x *SessionReduceRequest) GetType() SessionReduceRequest_WindowType { - if x != nil { - return x.Type - } - return SessionReduceRequest_FIXED -} - // * // SessionReduceResponse represents a response element. type SessionReduceResponse struct { @@ -268,11 +207,10 @@ type SessionReduceResponse struct { Result []*SessionReduceResponse_Result `protobuf:"bytes,1,rep,name=result,proto3" json:"result,omitempty"` // Partition represents a window partition to which the result belongs. - Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - CombinedKey string `protobuf:"bytes,3,opt,name=combinedKey,proto3" json:"combinedKey,omitempty"` - // Cob to indicate that there won't be any more results for this partition. - // This is used to indicate that the partition is closed so that the downstream - // can release the resources associated with the partition. + Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` + // the key which was used for demultiplexing the request stream. + CombinedKey string `protobuf:"bytes,3,opt,name=combinedKey,proto3" json:"combinedKey,omitempty"` + // EventTime represents the event time of the result. EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } @@ -589,99 +527,91 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7d, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7f, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, - 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x22, 0xcd, 0x05, 0x0a, 0x14, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, - 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, - 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x45, 0x0a, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x1a, 0xe3, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0a, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, - 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, - 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, - 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, - 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, - 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0x3d, 0x0a, 0x0a, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x49, 0x58, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, - 0x0a, 0x07, 0x53, 0x4c, 0x49, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, - 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x47, 0x4c, 0x4f, 0x42, - 0x41, 0x4c, 0x10, 0x03, 0x22, 0xbf, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, - 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, - 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, - 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, - 0x4b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, - 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, - 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, - 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, + 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x22, 0xc7, 0x04, 0x0a, 0x14, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x48, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, + 0xe3, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, + 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, + 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, 0x44, 0x10, 0x02, 0x12, 0x09, + 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, + 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, + 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, + 0x22, 0xbf, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x06, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, - 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, - 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, - 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, + 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, + 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, 0x0a, 0x0d, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x0f, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x26, + 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, + 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, + 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, + 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -696,43 +626,41 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP() []by return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData } -var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []interface{}{ - (SessionReduceRequest_WindowType)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowType - (SessionReduceRequest_WindowOperation_Event)(0), // 1: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event - (*Partition)(nil), // 2: sessionreduce.v1.Partition - (*SessionReduceRequest)(nil), // 3: sessionreduce.v1.SessionReduceRequest - (*SessionReduceResponse)(nil), // 4: sessionreduce.v1.SessionReduceResponse - (*ReadyResponse)(nil), // 5: sessionreduce.v1.ReadyResponse - (*SessionReduceRequest_WindowOperation)(nil), // 6: sessionreduce.v1.SessionReduceRequest.WindowOperation - (*SessionReduceRequest_Payload)(nil), // 7: sessionreduce.v1.SessionReduceRequest.Payload - (*SessionReduceResponse_Result)(nil), // 8: sessionreduce.v1.SessionReduceResponse.Result - (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 10: google.protobuf.Empty + (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + (*Partition)(nil), // 1: sessionreduce.v1.Partition + (*SessionReduceRequest)(nil), // 2: sessionreduce.v1.SessionReduceRequest + (*SessionReduceResponse)(nil), // 3: sessionreduce.v1.SessionReduceResponse + (*ReadyResponse)(nil), // 4: sessionreduce.v1.ReadyResponse + (*SessionReduceRequest_WindowOperation)(nil), // 5: sessionreduce.v1.SessionReduceRequest.WindowOperation + (*SessionReduceRequest_Payload)(nil), // 6: sessionreduce.v1.SessionReduceRequest.Payload + (*SessionReduceResponse_Result)(nil), // 7: sessionreduce.v1.SessionReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ - 9, // 0: sessionreduce.v1.Partition.start:type_name -> google.protobuf.Timestamp - 9, // 1: sessionreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp - 7, // 2: sessionreduce.v1.SessionReduceRequest.payload:type_name -> sessionreduce.v1.SessionReduceRequest.Payload - 6, // 3: sessionreduce.v1.SessionReduceRequest.operation:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation - 0, // 4: sessionreduce.v1.SessionReduceRequest.type:type_name -> sessionreduce.v1.SessionReduceRequest.WindowType - 8, // 5: sessionreduce.v1.SessionReduceResponse.result:type_name -> sessionreduce.v1.SessionReduceResponse.Result - 2, // 6: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition - 9, // 7: sessionreduce.v1.SessionReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 1, // 8: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event - 2, // 9: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition - 9, // 10: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp - 9, // 11: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp - 3, // 12: sessionreduce.v1.SessionReduce.SessionReduceFn:input_type -> sessionreduce.v1.SessionReduceRequest - 10, // 13: sessionreduce.v1.SessionReduce.IsReady:input_type -> google.protobuf.Empty - 4, // 14: sessionreduce.v1.SessionReduce.SessionReduceFn:output_type -> sessionreduce.v1.SessionReduceResponse - 5, // 15: sessionreduce.v1.SessionReduce.IsReady:output_type -> sessionreduce.v1.ReadyResponse - 14, // [14:16] is the sub-list for method output_type - 12, // [12:14] is the sub-list for method input_type - 12, // [12:12] is the sub-list for extension type_name - 12, // [12:12] is the sub-list for extension extendee - 0, // [0:12] is the sub-list for field type_name + 8, // 0: sessionreduce.v1.Partition.start:type_name -> google.protobuf.Timestamp + 8, // 1: sessionreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp + 6, // 2: sessionreduce.v1.SessionReduceRequest.payload:type_name -> sessionreduce.v1.SessionReduceRequest.Payload + 5, // 3: sessionreduce.v1.SessionReduceRequest.operation:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation + 7, // 4: sessionreduce.v1.SessionReduceResponse.result:type_name -> sessionreduce.v1.SessionReduceResponse.Result + 1, // 5: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition + 8, // 6: sessionreduce.v1.SessionReduceResponse.event_time:type_name -> google.protobuf.Timestamp + 0, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + 1, // 8: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition + 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 10: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 2, // 11: sessionreduce.v1.SessionReduce.SessionReduceFn:input_type -> sessionreduce.v1.SessionReduceRequest + 9, // 12: sessionreduce.v1.SessionReduce.IsReady:input_type -> google.protobuf.Empty + 3, // 13: sessionreduce.v1.SessionReduce.SessionReduceFn:output_type -> sessionreduce.v1.SessionReduceResponse + 4, // 14: sessionreduce.v1.SessionReduce.IsReady:output_type -> sessionreduce.v1.ReadyResponse + 13, // [13:15] is the sub-list for method output_type + 11, // [11:13] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() } @@ -831,7 +759,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc, - NumEnums: 2, + NumEnums: 1, NumMessages: 7, NumExtensions: 0, NumServices: 1, diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto index cb11781b..c07c6f24 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -16,10 +16,11 @@ service SessionReduce { rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); } +// Partition represents a window partition. message Partition { google.protobuf.Timestamp start = 1; google.protobuf.Timestamp end = 2; - string key = 3; + string slot = 3; } /** @@ -41,15 +42,6 @@ message SessionReduceRequest { repeated Partition partitions = 2; } - // WindowType represents a window type. - //REVISIT Do we need this? Can this be passed using metadata? - enum WindowType { - FIXED = 0; - SLIDING = 1; - SESSION = 2; - GLOBAL = 3; - } - // Payload represents a payload element. message Payload { repeated string keys = 1; @@ -60,7 +52,6 @@ message SessionReduceRequest { Payload payload = 1; WindowOperation operation = 2; - WindowType type = 3; } /** @@ -74,16 +65,15 @@ message SessionReduceResponse { repeated string tags = 3; } - repeated Result result = 1; // Partition represents a window partition to which the result belongs. Partition partition = 2; + // the key which was used for demultiplexing the request stream. string combinedKey = 3; - // Cob to indicate that there won't be any more results for this partition. - // This is used to indicate that the partition is closed so that the downstream - // can release the resources associated with the partition. + + // EventTime represents the event time of the result. google.protobuf.Timestamp event_time = 4; } diff --git a/pkg/reducer/service.go b/pkg/reducer/service.go index 76610ef6..3ebd0e51 100644 --- a/pkg/reducer/service.go +++ b/pkg/reducer/service.go @@ -2,16 +2,10 @@ package reducer import ( "context" - "fmt" "io" - "strconv" - "strings" - "sync" - "time" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" - grpcmd "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" @@ -30,7 +24,6 @@ const ( // Service implements the proto gen server interface and contains the reduce operation handler. type Service struct { reducepb.UnimplementedReduceServer - Reducer Reducer } @@ -42,44 +35,23 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*reducepb.ReadyResp // ReduceFn applies a reduce function to a request stream and returns a list of results. func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { var ( - md Metadata - err error - startTime int64 - endTime int64 - ctx = stream.Context() - chanMap = make(map[string]chan Datum) - mu sync.RWMutex - g errgroup.Group + err error + ctx = stream.Context() + g errgroup.Group ) - grpcMD, ok := grpcmd.FromIncomingContext(ctx) - if !ok { - statusErr := status.Errorf(codes.InvalidArgument, "keys and window information are not passed in grpc metadata") - return statusErr - } - - // get window start and end time from grpc metadata - var st, et string - st, err = getValueFromMetadata(grpcMD, winStartTime) - if err != nil { - statusErr := status.Errorf(codes.InvalidArgument, err.Error()) - return statusErr - } - - et, err = getValueFromMetadata(grpcMD, winEndTime) - if err != nil { - statusErr := status.Errorf(codes.InvalidArgument, err.Error()) - return statusErr - } + taskManager := newReduceTaskManager() - startTime, _ = strconv.ParseInt(st, 10, 64) - endTime, _ = strconv.ParseInt(et, 10, 64) - - // create interval window interface using the start and end time - iw := NewIntervalWindow(time.UnixMilli(startTime), time.UnixMilli(endTime)) - - // create metadata using interval window interface - md = NewMetadata(iw) + // err group for the go routine which reads from the output channel and sends to the stream + g.Go(func() error { + for output := range taskManager.OutputChannel() { + err := stream.Send(output) + if err != nil { + return err + } + } + return nil + }) // read messages from the stream and write the messages to corresponding channels // if the channel is not created, create the channel and invoke the reduceFn @@ -87,84 +59,43 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { d, recvErr := stream.Recv() // if EOF, close all the channels if recvErr == io.EOF { - closeChannels(chanMap) + taskManager.CloseAll() break } if recvErr != nil { - closeChannels(chanMap) // the error here is returned by stream.Recv() // it's already a gRPC error return recvErr } - unifiedKey := strings.Join(d.GetKeys(), delimiter) - var hd = NewHandlerDatum(d.GetValue(), d.EventTime.AsTime(), d.Watermark.AsTime()) - ch, chok := chanMap[unifiedKey] - if !chok { - ch = make(chan Datum) - chanMap[unifiedKey] = ch - - func(k []string, ch chan Datum) { - g.Go(func() error { - // we stream the messages to the user by writing messages - // to channel and wait until we get the result and stream - // the result back to the client (numaflow). - messages := fs.Reducer.Reduce(ctx, k, ch, md) - datumList := buildDatumList(messages) - - // stream.Send() is not thread safe. - mu.Lock() - defer mu.Unlock() - sendErr := stream.Send(datumList) - if sendErr != nil { - // the error here is returned by stream.Send() - // it's already a gRPC error - return sendErr - } - return nil - }) - }(d.GetKeys(), ch) + switch d.Operation.Event { + case reducepb.ReduceRequest_WindowOperation_OPEN: + // create a new reduce task and start the reduce operation + err = taskManager.CreateTask(ctx, d, fs.Reducer) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case reducepb.ReduceRequest_WindowOperation_CLOSE: + // close the reduce task + taskManager.CloseTask(d) + case reducepb.ReduceRequest_WindowOperation_APPEND: + // append the datum to the reduce task + err = taskManager.AppendToTask(d, fs.Reducer) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } } - ch <- hd } - // wait until all the mapfn return - return g.Wait() -} - -func buildDatumList(messages Messages) *reducepb.ReduceResponse { - response := &reducepb.ReduceResponse{} - for _, msg := range messages { - response.Results = append(response.Results, &reducepb.ReduceResponse_Result{ - Keys: msg.Keys(), - Value: msg.Value(), - Tags: msg.Tags(), - }) - } - - return response -} - -func closeChannels(chanMap map[string]chan Datum) { - for _, ch := range chanMap { - close(ch) + taskManager.WaitAll() + // wait for the go routine which reads from the output channel and sends to the stream to return + err = g.Wait() + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr } -} -func getValueFromMetadata(md grpcmd.MD, k string) (string, error) { - var value string - - keyValue := md.Get(k) - - if len(keyValue) > 1 { - return value, fmt.Errorf("expected extactly one value for keys %s in metadata but got %d values, %s", k, len(keyValue), keyValue) - } else if len(keyValue) == 1 { - value = keyValue[0] - } else { - // the length equals zero is invalid for reduce - // since we are using a global keys, and start and end time - // cannot be empty - return value, fmt.Errorf("expected non empty value for keys %s in metadata but got an empty value", k) - } - return value, nil + return nil } diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index 8eb6e488..336025aa 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -72,22 +72,58 @@ func TestService_ReduceFn(t *testing.T) { }), input: []*reducepb.ReduceRequest{ { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, expected: &reducepb.ReduceResponse{ @@ -97,6 +133,12 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(60)), }, }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EventTime: timestamppb.New(time.UnixMilli(119999)), }, expectedErr: false, }, @@ -112,40 +154,112 @@ func TestService_ReduceFn(t *testing.T) { }), input: []*reducepb.ReduceRequest{ { - Keys: []string{"client1"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client2"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client3"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client1"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client2"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client3"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, expected: &reducepb.ReduceResponse{ @@ -163,6 +277,12 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(60)), }, }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EventTime: timestamppb.New(time.UnixMilli(119999)), }, expectedErr: false, }, @@ -178,22 +298,58 @@ func TestService_ReduceFn(t *testing.T) { }), input: []*reducepb.ReduceRequest{ { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, expected: &reducepb.ReduceResponse{ @@ -202,6 +358,12 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(60)), }, }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EventTime: timestamppb.New(time.UnixMilli(119999)), }, expectedErr: false, }, @@ -217,22 +379,58 @@ func TestService_ReduceFn(t *testing.T) { }), input: []*reducepb.ReduceRequest{ { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Partitions: []*reducepb.Partition{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, expected: &reducepb.ReduceResponse{ @@ -242,6 +440,12 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte{}, }, }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EventTime: timestamppb.New(time.UnixMilli(119999)), }, expectedErr: false, }, @@ -277,6 +481,8 @@ func TestService_ReduceFn(t *testing.T) { defer wg.Done() for msg := range outputCh { result.Results = append(result.Results, msg.Results...) + result.Partition = msg.Partition + result.EventTime = msg.EventTime } }() diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go new file mode 100644 index 00000000..dc31cf96 --- /dev/null +++ b/pkg/reducer/task_manager.go @@ -0,0 +1,178 @@ +package reducer + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" + + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +// reduceTask represents a reduce task for a reduce operation. +type reduceTask struct { + combinedKey string + partition *v1.Partition + reduceHandler Reducer + inputCh chan Datum + Done chan struct{} +} + +// buildReduceResponse builds the reduce response from the messages. +func (rt *reduceTask) buildReduceResponse(messages Messages) *v1.ReduceResponse { + results := make([]*v1.ReduceResponse_Result, 0, len(messages)) + for _, message := range messages { + results = append(results, &v1.ReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + }) + } + + response := &v1.ReduceResponse{ + Results: results, + Partition: rt.partition, + EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + } + + return response +} + +// uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. +func (rt *reduceTask) uniqueKey() string { + return fmt.Sprintf("%d:%d:%s", + rt.partition.GetStart().AsTime().UnixMilli(), + rt.partition.GetEnd().AsTime().UnixMilli(), + rt.combinedKey) +} + +// reduceTaskManager manages the reduce tasks for a reduce operation. +type reduceTaskManager struct { + Tasks map[string]*reduceTask + Output chan *v1.ReduceResponse + Mutex sync.RWMutex +} + +func newReduceTaskManager() *reduceTaskManager { + return &reduceTaskManager{ + Tasks: make(map[string]*reduceTask), + Output: make(chan *v1.ReduceResponse), + } +} + +// CreateTask creates a new reduce task and starts the reduce operation. +func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest, reducer Reducer) error { + rtm.Mutex.Lock() + defer rtm.Mutex.Unlock() + + if len(request.Operation.Partitions) != 1 { + return fmt.Errorf("invalid number of partitions") + } + + md := NewMetadata(NewIntervalWindow(request.Operation.Partitions[0].GetStart().AsTime(), + request.Operation.Partitions[0].GetEnd().AsTime())) + + task := &reduceTask{ + combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), + partition: request.Operation.Partitions[0], + inputCh: make(chan Datum), + Done: make(chan struct{}), + } + + go func() { + defer close(task.Done) + msgs := reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) + rtm.Output <- task.buildReduceResponse(msgs) + }() + + // write the first message to the input channel + task.inputCh <- buildDatum(request) + + rtm.Tasks[task.uniqueKey()] = task + return nil +} + +// AppendToTask writes the message to the reduce task. +// If the task is not found, it creates a new task and starts the reduce operation. +func (rtm *reduceTaskManager) AppendToTask(request *v1.ReduceRequest, reducer Reducer) error { + rtm.Mutex.RLock() + gKey := generateKey(request.Operation.Partitions[0], request.Payload.Keys) + task, ok := rtm.Tasks[gKey] + rtm.Mutex.RUnlock() + + // if the task is not found, create a new task + if !ok { + return rtm.CreateTask(context.Background(), request, reducer) + } + + task.inputCh <- buildDatum(request) + return nil +} + +// CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. +func (rtm *reduceTaskManager) CloseTask(request *v1.ReduceRequest) { + rtm.Mutex.Lock() + tasksToBeClosed := make([]*reduceTask, 0, len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + key := generateKey(partition, request.Payload.Keys) + task, ok := rtm.Tasks[key] + if ok { + tasksToBeClosed = append(tasksToBeClosed, task) + } + delete(rtm.Tasks, key) + } + rtm.Mutex.Unlock() + + for _, task := range tasksToBeClosed { + close(task.inputCh) + } +} + +// OutputChannel returns the output channel for the reduce task manager to read the results. +func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { + return rtm.Output +} + +// WaitAll waits for all the reduce tasks to complete. +func (rtm *reduceTaskManager) WaitAll() { + rtm.Mutex.RLock() + tasks := make([]*reduceTask, 0, len(rtm.Tasks)) + for _, task := range rtm.Tasks { + tasks = append(tasks, task) + } + rtm.Mutex.RUnlock() + + for _, task := range tasks { + <-task.Done + } + // after all the tasks are completed, close the output channel + close(rtm.Output) +} + +// CloseAll closes all the reduce tasks. +func (rtm *reduceTaskManager) CloseAll() { + rtm.Mutex.Lock() + tasks := make([]*reduceTask, 0, len(rtm.Tasks)) + for _, task := range rtm.Tasks { + tasks = append(tasks, task) + } + rtm.Mutex.Unlock() + + for _, task := range tasks { + close(task.inputCh) + } +} + +func generateKey(partition *v1.Partition, keys []string) string { + return fmt.Sprintf("%d:%d:%s", + partition.GetStart().AsTime().UnixMilli(), + partition.GetEnd().AsTime().UnixMilli(), + strings.Join(keys, delimiter)) +} + +func buildDatum(request *v1.ReduceRequest) Datum { + return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +} diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go index 3397d2d1..88935267 100644 --- a/pkg/sessionreducer/service.go +++ b/pkg/sessionreducer/service.go @@ -67,8 +67,7 @@ func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionR case sessionreducepb.SessionReduceRequest_WindowOperation_OPEN: // create a new task and start the session reduce operation // also append the datum to the task - taskManager.CreateTask(ctx, d, fs.createSessionReducer()) - err := taskManager.AppendToTask(d) + err := taskManager.CreateTask(ctx, d, fs.createSessionReducer()) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr @@ -78,7 +77,7 @@ func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionR taskManager.CloseTask(d) case sessionreducepb.SessionReduceRequest_WindowOperation_APPEND: // append the datum to the task - err := taskManager.AppendToTask(d) + err := taskManager.AppendToTask(d, fs.createSessionReducer()) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index bc632bad..e0d4e8ce 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -12,8 +12,8 @@ import ( v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" ) -// reduceTask represents a reduce task for a session reduce operation. -type reduceTask struct { +// sessionReduceTask represents a session reduce task for a session reduce operation. +type sessionReduceTask struct { combinedKey string partition *v1.Partition reduceStreamer SessionReducer @@ -23,7 +23,7 @@ type reduceTask struct { } // buildSessionReduceResponse builds the session reduce response from the messages. -func (rt *reduceTask) buildSessionReduceResponse(messages Messages) *v1.SessionReduceResponse { +func (rt *sessionReduceTask) buildSessionReduceResponse(messages Messages) *v1.SessionReduceResponse { result := make([]*v1.SessionReduceResponse_Result, 0, len(messages)) for _, message := range messages { result = append(result, &v1.SessionReduceResponse_Result{ @@ -44,33 +44,37 @@ func (rt *reduceTask) buildSessionReduceResponse(messages Messages) *v1.SessionR } // uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. -func (rt *reduceTask) uniqueKey() string { +func (rt *sessionReduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", rt.partition.GetStart().AsTime().UnixMilli(), rt.partition.GetEnd().AsTime().UnixMilli(), rt.combinedKey) } -// reduceTaskManager manages the reduce tasks for a session reduce operation. -type reduceTaskManager struct { - Tasks map[string]*reduceTask +// sessionReduceTaskManager manages the reduce tasks for a session reduce operation. +type sessionReduceTaskManager struct { + Tasks map[string]*sessionReduceTask Output chan *v1.SessionReduceResponse Mutex sync.RWMutex } -func newReduceTaskManager() *reduceTaskManager { - return &reduceTaskManager{ - Tasks: make(map[string]*reduceTask), +func newReduceTaskManager() *sessionReduceTaskManager { + return &sessionReduceTaskManager{ + Tasks: make(map[string]*sessionReduceTask), Output: make(chan *v1.SessionReduceResponse), } } // CreateTask creates a new reduce task and starts the session reduce operation. -func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, reduceStreamer SessionReducer) { +func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, reduceStreamer SessionReducer) error { rtm.Mutex.Lock() defer rtm.Mutex.Unlock() - task := &reduceTask{ + if len(request.Operation.Partitions) != 1 { + return fmt.Errorf("invalid number of partitions") + } + + task := &sessionReduceTask{ combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), partition: request.Operation.Partitions[0], reduceStreamer: reduceStreamer, @@ -85,16 +89,22 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Sessio }() rtm.Tasks[task.uniqueKey()] = task + return nil } // AppendToTask writes the message to the reduce task. -func (rtm *reduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) error { +// If the reduce task is not found, it will create a new reduce task and start the reduce operation. +func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceRequest, sessionReducer SessionReducer) error { + if len(request.Operation.Partitions) != 1 { + return fmt.Errorf("invalid number of partitions") + } + rtm.Mutex.RLock() task, ok := rtm.Tasks[generateKey(request.Operation.Partitions[0], request.Payload.Keys)] rtm.Mutex.RUnlock() if !ok { - return fmt.Errorf("task not found") + return rtm.CreateTask(context.Background(), request, sessionReducer) } task.inputCh <- buildDatum(request) @@ -102,9 +112,9 @@ func (rtm *reduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) err } // CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. -func (rtm *reduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { +func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { rtm.Mutex.Lock() - tasksToBeClosed := make([]*reduceTask, 0, len(request.Operation.Partitions)) + tasksToBeClosed := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { key := generateKey(partition, request.Payload.Keys) task, ok := rtm.Tasks[key] @@ -122,9 +132,9 @@ func (rtm *reduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { // MergeTasks merges the session reduce tasks. It will invoke close on all the tasks except the main task. The main task will // merge the aggregators from the other tasks. The main task will be first task in the list of tasks. -func (rtm *reduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { +func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { rtm.Mutex.Lock() - tasks := make([]*reduceTask, 0, len(request.Operation.Partitions)) + tasks := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { key := generateKey(partition, request.Payload.Keys) task, ok := rtm.Tasks[key] @@ -163,7 +173,7 @@ func (rtm *reduceTaskManager) MergeTasks(ctx context.Context, request *v1.Sessio // deletes the old task and creates a new task with the new window. // expects request.Operation.Partitions to have exactly two partitions. The first partition is the old partition and the second // partition is the new partition. -func (rtm *reduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { +func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { if len(request.Operation.Partitions) != 2 { return fmt.Errorf("expected exactly two partitions") } @@ -183,14 +193,14 @@ func (rtm *reduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error } // OutputChannel returns the output channel for the reduce task manager to read the results. -func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { +func (rtm *sessionReduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { return rtm.Output } // WaitAll waits for all the reduce tasks to complete. -func (rtm *reduceTaskManager) WaitAll() { +func (rtm *sessionReduceTaskManager) WaitAll() { rtm.Mutex.RLock() - tasks := make([]*reduceTask, 0, len(rtm.Tasks)) + tasks := make([]*sessionReduceTask, 0, len(rtm.Tasks)) for _, task := range rtm.Tasks { tasks = append(tasks, task) } @@ -199,12 +209,14 @@ func (rtm *reduceTaskManager) WaitAll() { for _, task := range tasks { <-task.Done } + // after all the tasks are completed, close the output channel + close(rtm.Output) } // CloseAll closes all the reduce tasks. -func (rtm *reduceTaskManager) CloseAll() { +func (rtm *sessionReduceTaskManager) CloseAll() { rtm.Mutex.Lock() - tasks := make([]*reduceTask, 0, len(rtm.Tasks)) + tasks := make([]*sessionReduceTask, 0, len(rtm.Tasks)) for _, task := range rtm.Tasks { tasks = append(tasks, task) } From 05e425d6bb712b2f1a1f7983ff4771613fbb875d Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 7 Nov 2023 17:34:50 +0530 Subject: [PATCH 03/27] support for global reduce Signed-off-by: Yashash H L --- go.mod | 1 + go.sum | 2 + .../proto/globalreduce/v1/globalreduce.pb.go | 774 ++++++++++++++++++ .../proto/globalreduce/v1/globalreduce.proto | 85 ++ .../globalreduce/v1/globalreduce_grpc.pb.go | 179 ++++ .../v1/globalreducemock/globalreducemock.go | 216 +++++ pkg/apis/proto/globalreduce/v1/mockgen.go | 3 + .../sessionreduce/v1/sessionreduce.pb.go | 88 +- .../sessionreduce/v1/sessionreduce.proto | 2 +- pkg/globalreducer/doc.go | 5 + pkg/globalreducer/interface.go | 18 + pkg/globalreducer/message.go | 70 ++ pkg/globalreducer/options.go | 43 + pkg/globalreducer/options_test.go | 18 + pkg/globalreducer/server.go | 56 ++ pkg/globalreducer/service.go | 100 +++ pkg/globalreducer/task_manager.go | 420 +++++----- pkg/globalreducer/types.go | 65 ++ pkg/sessionreducer/interface.go | 1 + pkg/sessionreducer/service.go | 2 +- pkg/sessionreducer/task_manager.go | 17 +- 21 files changed, 1904 insertions(+), 261 deletions(-) create mode 100644 pkg/apis/proto/globalreduce/v1/globalreduce.pb.go create mode 100644 pkg/apis/proto/globalreduce/v1/globalreduce.proto create mode 100644 pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go create mode 100644 pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go create mode 100644 pkg/apis/proto/globalreduce/v1/mockgen.go create mode 100644 pkg/globalreducer/doc.go create mode 100644 pkg/globalreducer/interface.go create mode 100644 pkg/globalreducer/message.go create mode 100644 pkg/globalreducer/options.go create mode 100644 pkg/globalreducer/options_test.go create mode 100644 pkg/globalreducer/server.go create mode 100644 pkg/globalreducer/service.go create mode 100644 pkg/globalreducer/types.go diff --git a/go.mod b/go.mod index a6ed38f4..423fb94f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/golang/mock v1.6.0 github.com/stretchr/testify v1.8.1 + go.uber.org/atomic v1.11.0 golang.org/x/net v0.9.0 golang.org/x/sync v0.1.0 google.golang.org/grpc v1.57.0 diff --git a/go.sum b/go.sum index 90eb1d42..c20884dc 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go new file mode 100644 index 00000000..cbefc269 --- /dev/null +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go @@ -0,0 +1,774 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.9 +// source: pkg/apis/proto/globalreduce/v1/globalreduce.proto + +package v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GlobalReduceRequest_WindowOperation_Event int32 + +const ( + GlobalReduceRequest_WindowOperation_OPEN GlobalReduceRequest_WindowOperation_Event = 0 + GlobalReduceRequest_WindowOperation_CLOSE GlobalReduceRequest_WindowOperation_Event = 1 + GlobalReduceRequest_WindowOperation_EXPAND GlobalReduceRequest_WindowOperation_Event = 2 + GlobalReduceRequest_WindowOperation_MERGE GlobalReduceRequest_WindowOperation_Event = 3 + GlobalReduceRequest_WindowOperation_APPEND GlobalReduceRequest_WindowOperation_Event = 4 +) + +// Enum value maps for GlobalReduceRequest_WindowOperation_Event. +var ( + GlobalReduceRequest_WindowOperation_Event_name = map[int32]string{ + 0: "OPEN", + 1: "CLOSE", + 2: "EXPAND", + 3: "MERGE", + 4: "APPEND", + } + GlobalReduceRequest_WindowOperation_Event_value = map[string]int32{ + "OPEN": 0, + "CLOSE": 1, + "EXPAND": 2, + "MERGE": 3, + "APPEND": 4, + } +) + +func (x GlobalReduceRequest_WindowOperation_Event) Enum() *GlobalReduceRequest_WindowOperation_Event { + p := new(GlobalReduceRequest_WindowOperation_Event) + *p = x + return p +} + +func (x GlobalReduceRequest_WindowOperation_Event) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GlobalReduceRequest_WindowOperation_Event) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes[0].Descriptor() +} + +func (GlobalReduceRequest_WindowOperation_Event) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes[0] +} + +func (x GlobalReduceRequest_WindowOperation_Event) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GlobalReduceRequest_WindowOperation_Event.Descriptor instead. +func (GlobalReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1, 0, 0} +} + +// Partition represents a window partition. +type Partition struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` + End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` + Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` +} + +func (x *Partition) Reset() { + *x = Partition{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Partition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Partition) ProtoMessage() {} + +func (x *Partition) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Partition.ProtoReflect.Descriptor instead. +func (*Partition) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{0} +} + +func (x *Partition) GetStart() *timestamppb.Timestamp { + if x != nil { + return x.Start + } + return nil +} + +func (x *Partition) GetEnd() *timestamppb.Timestamp { + if x != nil { + return x.End + } + return nil +} + +func (x *Partition) GetSlot() string { + if x != nil { + return x.Slot + } + return "" +} + +// * +// GlobalReduceRequest represents a request element. +type GlobalReduceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Payload *GlobalReduceRequest_Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + Operation *GlobalReduceRequest_WindowOperation `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` +} + +func (x *GlobalReduceRequest) Reset() { + *x = GlobalReduceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobalReduceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobalReduceRequest) ProtoMessage() {} + +func (x *GlobalReduceRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobalReduceRequest.ProtoReflect.Descriptor instead. +func (*GlobalReduceRequest) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1} +} + +func (x *GlobalReduceRequest) GetPayload() *GlobalReduceRequest_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *GlobalReduceRequest) GetOperation() *GlobalReduceRequest_WindowOperation { + if x != nil { + return x.Operation + } + return nil +} + +// * +// GlobalReduceResponse represents a response element. +type GlobalReduceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Results []*GlobalReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + // Partition represents a window partition to which the result belongs. + Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` + // the key which was used for demultiplexing the request stream. + CombinedKey string `protobuf:"bytes,3,opt,name=combinedKey,proto3" json:"combinedKey,omitempty"` + // EventTime represents the event time of the result. + EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` +} + +func (x *GlobalReduceResponse) Reset() { + *x = GlobalReduceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobalReduceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobalReduceResponse) ProtoMessage() {} + +func (x *GlobalReduceResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobalReduceResponse.ProtoReflect.Descriptor instead. +func (*GlobalReduceResponse) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{2} +} + +func (x *GlobalReduceResponse) GetResults() []*GlobalReduceResponse_Result { + if x != nil { + return x.Results + } + return nil +} + +func (x *GlobalReduceResponse) GetPartition() *Partition { + if x != nil { + return x.Partition + } + return nil +} + +func (x *GlobalReduceResponse) GetCombinedKey() string { + if x != nil { + return x.CombinedKey + } + return "" +} + +func (x *GlobalReduceResponse) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +// * +// ReadyResponse is the health check result. +type ReadyResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` +} + +func (x *ReadyResponse) Reset() { + *x = ReadyResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReadyResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadyResponse) ProtoMessage() {} + +func (x *ReadyResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. +func (*ReadyResponse) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{3} +} + +func (x *ReadyResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +// WindowOperation represents a window operation. +// it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. +type GlobalReduceRequest_WindowOperation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Event GlobalReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=globalreduce.v1.GlobalReduceRequest_WindowOperation_Event" json:"event,omitempty"` + Partitions []*Partition `protobuf:"bytes,2,rep,name=partitions,proto3" json:"partitions,omitempty"` +} + +func (x *GlobalReduceRequest_WindowOperation) Reset() { + *x = GlobalReduceRequest_WindowOperation{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobalReduceRequest_WindowOperation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobalReduceRequest_WindowOperation) ProtoMessage() {} + +func (x *GlobalReduceRequest_WindowOperation) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobalReduceRequest_WindowOperation.ProtoReflect.Descriptor instead. +func (*GlobalReduceRequest_WindowOperation) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *GlobalReduceRequest_WindowOperation) GetEvent() GlobalReduceRequest_WindowOperation_Event { + if x != nil { + return x.Event + } + return GlobalReduceRequest_WindowOperation_OPEN +} + +func (x *GlobalReduceRequest_WindowOperation) GetPartitions() []*Partition { + if x != nil { + return x.Partitions + } + return nil +} + +// Payload represents a payload element. +type GlobalReduceRequest_Payload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` +} + +func (x *GlobalReduceRequest_Payload) Reset() { + *x = GlobalReduceRequest_Payload{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobalReduceRequest_Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobalReduceRequest_Payload) ProtoMessage() {} + +func (x *GlobalReduceRequest_Payload) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobalReduceRequest_Payload.ProtoReflect.Descriptor instead. +func (*GlobalReduceRequest_Payload) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *GlobalReduceRequest_Payload) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *GlobalReduceRequest_Payload) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *GlobalReduceRequest_Payload) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *GlobalReduceRequest_Payload) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +// Result represents a result element. It contains the result of the reduce function. +type GlobalReduceResponse_Result struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` +} + +func (x *GlobalReduceResponse_Result) Reset() { + *x = GlobalReduceResponse_Result{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobalReduceResponse_Result) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobalReduceResponse_Result) ProtoMessage() {} + +func (x *GlobalReduceResponse_Result) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobalReduceResponse_Result.ProtoReflect.Descriptor instead. +func (*GlobalReduceResponse_Result) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{2, 0} +} + +func (x *GlobalReduceResponse_Result) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *GlobalReduceResponse_Result) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *GlobalReduceResponse_Result) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +var File_pkg_apis_proto_globalreduce_v1_globalreduce_proto protoreflect.FileDescriptor + +var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ + 0x0a, 0x31, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, 0x31, + 0x2f, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0x7f, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, + 0x6c, 0x6f, 0x74, 0x22, 0xbf, 0x04, 0x0a, 0x13, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x46, 0x0a, 0x07, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x12, 0x52, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xe0, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x05, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x67, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3a, 0x0a, + 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, + 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, + 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, + 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, + 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, + 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xbd, 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, + 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, + 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, + 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, + 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, + 0x0c, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, + 0x0e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, + 0x24, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, + 0x12, 0x41, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, + 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescOnce sync.Once + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData = file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc +) + +func file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP() []byte { + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescOnce.Do(func() { + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData) + }) + return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData +} + +var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes = []interface{}{ + (GlobalReduceRequest_WindowOperation_Event)(0), // 0: globalreduce.v1.GlobalReduceRequest.WindowOperation.Event + (*Partition)(nil), // 1: globalreduce.v1.Partition + (*GlobalReduceRequest)(nil), // 2: globalreduce.v1.GlobalReduceRequest + (*GlobalReduceResponse)(nil), // 3: globalreduce.v1.GlobalReduceResponse + (*ReadyResponse)(nil), // 4: globalreduce.v1.ReadyResponse + (*GlobalReduceRequest_WindowOperation)(nil), // 5: globalreduce.v1.GlobalReduceRequest.WindowOperation + (*GlobalReduceRequest_Payload)(nil), // 6: globalreduce.v1.GlobalReduceRequest.Payload + (*GlobalReduceResponse_Result)(nil), // 7: globalreduce.v1.GlobalReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty +} +var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = []int32{ + 8, // 0: globalreduce.v1.Partition.start:type_name -> google.protobuf.Timestamp + 8, // 1: globalreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp + 6, // 2: globalreduce.v1.GlobalReduceRequest.payload:type_name -> globalreduce.v1.GlobalReduceRequest.Payload + 5, // 3: globalreduce.v1.GlobalReduceRequest.operation:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation + 7, // 4: globalreduce.v1.GlobalReduceResponse.results:type_name -> globalreduce.v1.GlobalReduceResponse.Result + 1, // 5: globalreduce.v1.GlobalReduceResponse.partition:type_name -> globalreduce.v1.Partition + 8, // 6: globalreduce.v1.GlobalReduceResponse.event_time:type_name -> google.protobuf.Timestamp + 0, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Event + 1, // 8: globalreduce.v1.GlobalReduceRequest.WindowOperation.partitions:type_name -> globalreduce.v1.Partition + 8, // 9: globalreduce.v1.GlobalReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 10: globalreduce.v1.GlobalReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 2, // 11: globalreduce.v1.GlobalReduce.GlobalReduceFn:input_type -> globalreduce.v1.GlobalReduceRequest + 9, // 12: globalreduce.v1.GlobalReduce.IsReady:input_type -> google.protobuf.Empty + 3, // 13: globalreduce.v1.GlobalReduce.GlobalReduceFn:output_type -> globalreduce.v1.GlobalReduceResponse + 4, // 14: globalreduce.v1.GlobalReduce.IsReady:output_type -> globalreduce.v1.ReadyResponse + 13, // [13:15] is the sub-list for method output_type + 11, // [11:13] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_init() } +func file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_init() { + if File_pkg_apis_proto_globalreduce_v1_globalreduce_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Partition); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobalReduceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobalReduceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReadyResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobalReduceRequest_WindowOperation); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobalReduceRequest_Payload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobalReduceResponse_Result); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc, + NumEnums: 1, + NumMessages: 7, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes, + DependencyIndexes: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs, + EnumInfos: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes, + MessageInfos: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes, + }.Build() + File_pkg_apis_proto_globalreduce_v1_globalreduce_proto = out.File + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = nil + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes = nil + file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = nil +} diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.proto b/pkg/apis/proto/globalreduce/v1/globalreduce.proto new file mode 100644 index 00000000..75a28fbc --- /dev/null +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.proto @@ -0,0 +1,85 @@ +syntax = "proto3"; + +option go_package = "github.com/numaproj/numaflow-go/pkg/apis/proto/reducestream/v1"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + + +package globalreduce.v1; + +service GlobalReduce { + // GlobalReduceFn applies a reduce function to a request stream. + rpc GlobalReduceFn(stream GlobalReduceRequest) returns (stream GlobalReduceResponse); + + // IsReady is the heartbeat endpoint for gRPC. + rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); +} + +// Partition represents a window partition. +message Partition { + google.protobuf.Timestamp start = 1; + google.protobuf.Timestamp end = 2; + string slot = 3; +} + +/** + * GlobalReduceRequest represents a request element. + */ +message GlobalReduceRequest { + // WindowOperation represents a window operation. + // it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. + message WindowOperation { + enum Event { + OPEN = 0; + CLOSE = 1; + EXPAND = 2; + MERGE = 3; + APPEND = 4; + } + + Event event = 1; + repeated Partition partitions = 2; + } + + // Payload represents a payload element. + message Payload { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + } + + Payload payload = 1; + WindowOperation operation = 2; +} + +/** + * GlobalReduceResponse represents a response element. + */ +message GlobalReduceResponse { + // Result represents a result element. It contains the result of the reduce function. + message Result { + repeated string keys = 1; + bytes value = 2; + repeated string tags = 3; + } + + repeated Result results = 1; + + // Partition represents a window partition to which the result belongs. + Partition partition = 2; + + // the key which was used for demultiplexing the request stream. + string combinedKey = 3; + + // EventTime represents the event time of the result. + google.protobuf.Timestamp event_time = 4; +} + +/** + * ReadyResponse is the health check result. + */ +message ReadyResponse { + bool ready = 1; +} \ No newline at end of file diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go new file mode 100644 index 00000000..67ce9cb8 --- /dev/null +++ b/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go @@ -0,0 +1,179 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.21.9 +// source: pkg/apis/proto/globalreduce/v1/globalreduce.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// GlobalReduceClient is the client API for GlobalReduce service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type GlobalReduceClient interface { + // GlobalReduceFn applies a reduce function to a request stream. + GlobalReduceFn(ctx context.Context, opts ...grpc.CallOption) (GlobalReduce_GlobalReduceFnClient, error) + // IsReady is the heartbeat endpoint for gRPC. + IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) +} + +type globalReduceClient struct { + cc grpc.ClientConnInterface +} + +func NewGlobalReduceClient(cc grpc.ClientConnInterface) GlobalReduceClient { + return &globalReduceClient{cc} +} + +func (c *globalReduceClient) GlobalReduceFn(ctx context.Context, opts ...grpc.CallOption) (GlobalReduce_GlobalReduceFnClient, error) { + stream, err := c.cc.NewStream(ctx, &GlobalReduce_ServiceDesc.Streams[0], "/globalreduce.v1.GlobalReduce/GlobalReduceFn", opts...) + if err != nil { + return nil, err + } + x := &globalReduceGlobalReduceFnClient{stream} + return x, nil +} + +type GlobalReduce_GlobalReduceFnClient interface { + Send(*GlobalReduceRequest) error + Recv() (*GlobalReduceResponse, error) + grpc.ClientStream +} + +type globalReduceGlobalReduceFnClient struct { + grpc.ClientStream +} + +func (x *globalReduceGlobalReduceFnClient) Send(m *GlobalReduceRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *globalReduceGlobalReduceFnClient) Recv() (*GlobalReduceResponse, error) { + m := new(GlobalReduceResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *globalReduceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + out := new(ReadyResponse) + err := c.cc.Invoke(ctx, "/globalreduce.v1.GlobalReduce/IsReady", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// GlobalReduceServer is the server API for GlobalReduce service. +// All implementations must embed UnimplementedGlobalReduceServer +// for forward compatibility +type GlobalReduceServer interface { + // GlobalReduceFn applies a reduce function to a request stream. + GlobalReduceFn(GlobalReduce_GlobalReduceFnServer) error + // IsReady is the heartbeat endpoint for gRPC. + IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) + mustEmbedUnimplementedGlobalReduceServer() +} + +// UnimplementedGlobalReduceServer must be embedded to have forward compatible implementations. +type UnimplementedGlobalReduceServer struct { +} + +func (UnimplementedGlobalReduceServer) GlobalReduceFn(GlobalReduce_GlobalReduceFnServer) error { + return status.Errorf(codes.Unimplemented, "method GlobalReduceFn not implemented") +} +func (UnimplementedGlobalReduceServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") +} +func (UnimplementedGlobalReduceServer) mustEmbedUnimplementedGlobalReduceServer() {} + +// UnsafeGlobalReduceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to GlobalReduceServer will +// result in compilation errors. +type UnsafeGlobalReduceServer interface { + mustEmbedUnimplementedGlobalReduceServer() +} + +func RegisterGlobalReduceServer(s grpc.ServiceRegistrar, srv GlobalReduceServer) { + s.RegisterService(&GlobalReduce_ServiceDesc, srv) +} + +func _GlobalReduce_GlobalReduceFn_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(GlobalReduceServer).GlobalReduceFn(&globalReduceGlobalReduceFnServer{stream}) +} + +type GlobalReduce_GlobalReduceFnServer interface { + Send(*GlobalReduceResponse) error + Recv() (*GlobalReduceRequest, error) + grpc.ServerStream +} + +type globalReduceGlobalReduceFnServer struct { + grpc.ServerStream +} + +func (x *globalReduceGlobalReduceFnServer) Send(m *GlobalReduceResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *globalReduceGlobalReduceFnServer) Recv() (*GlobalReduceRequest, error) { + m := new(GlobalReduceRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _GlobalReduce_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(emptypb.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GlobalReduceServer).IsReady(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/globalreduce.v1.GlobalReduce/IsReady", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GlobalReduceServer).IsReady(ctx, req.(*emptypb.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// GlobalReduce_ServiceDesc is the grpc.ServiceDesc for GlobalReduce service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var GlobalReduce_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "globalreduce.v1.GlobalReduce", + HandlerType: (*GlobalReduceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "IsReady", + Handler: _GlobalReduce_IsReady_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "GlobalReduceFn", + Handler: _GlobalReduce_GlobalReduceFn_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "pkg/apis/proto/globalreduce/v1/globalreduce.proto", +} diff --git a/pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go b/pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go new file mode 100644 index 00000000..108a3d72 --- /dev/null +++ b/pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go @@ -0,0 +1,216 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1 (interfaces: GlobalReduceClient,GlobalReduce_GlobalReduceFnClient) + +// Package globalreducemock is a generated GoMock package. +package globalreducemock + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" + grpc "google.golang.org/grpc" + metadata "google.golang.org/grpc/metadata" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// MockGlobalReduceClient is a mock of GlobalReduceClient interface. +type MockGlobalReduceClient struct { + ctrl *gomock.Controller + recorder *MockGlobalReduceClientMockRecorder +} + +// MockGlobalReduceClientMockRecorder is the mock recorder for MockGlobalReduceClient. +type MockGlobalReduceClientMockRecorder struct { + mock *MockGlobalReduceClient +} + +// NewMockGlobalReduceClient creates a new mock instance. +func NewMockGlobalReduceClient(ctrl *gomock.Controller) *MockGlobalReduceClient { + mock := &MockGlobalReduceClient{ctrl: ctrl} + mock.recorder = &MockGlobalReduceClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGlobalReduceClient) EXPECT() *MockGlobalReduceClientMockRecorder { + return m.recorder +} + +// GlobalReduceFn mocks base method. +func (m *MockGlobalReduceClient) GlobalReduceFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.GlobalReduce_GlobalReduceFnClient, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GlobalReduceFn", varargs...) + ret0, _ := ret[0].(v1.GlobalReduce_GlobalReduceFnClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GlobalReduceFn indicates an expected call of GlobalReduceFn. +func (mr *MockGlobalReduceClientMockRecorder) GlobalReduceFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GlobalReduceFn", reflect.TypeOf((*MockGlobalReduceClient)(nil).GlobalReduceFn), varargs...) +} + +// IsReady mocks base method. +func (m *MockGlobalReduceClient) IsReady(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*v1.ReadyResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "IsReady", varargs...) + ret0, _ := ret[0].(*v1.ReadyResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsReady indicates an expected call of IsReady. +func (mr *MockGlobalReduceClientMockRecorder) IsReady(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsReady", reflect.TypeOf((*MockGlobalReduceClient)(nil).IsReady), varargs...) +} + +// MockGlobalReduce_GlobalReduceFnClient is a mock of GlobalReduce_GlobalReduceFnClient interface. +type MockGlobalReduce_GlobalReduceFnClient struct { + ctrl *gomock.Controller + recorder *MockGlobalReduce_GlobalReduceFnClientMockRecorder +} + +// MockGlobalReduce_GlobalReduceFnClientMockRecorder is the mock recorder for MockGlobalReduce_GlobalReduceFnClient. +type MockGlobalReduce_GlobalReduceFnClientMockRecorder struct { + mock *MockGlobalReduce_GlobalReduceFnClient +} + +// NewMockGlobalReduce_GlobalReduceFnClient creates a new mock instance. +func NewMockGlobalReduce_GlobalReduceFnClient(ctrl *gomock.Controller) *MockGlobalReduce_GlobalReduceFnClient { + mock := &MockGlobalReduce_GlobalReduceFnClient{ctrl: ctrl} + mock.recorder = &MockGlobalReduce_GlobalReduceFnClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGlobalReduce_GlobalReduceFnClient) EXPECT() *MockGlobalReduce_GlobalReduceFnClientMockRecorder { + return m.recorder +} + +// CloseSend mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) CloseSend() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloseSend") + ret0, _ := ret[0].(error) + return ret0 +} + +// CloseSend indicates an expected call of CloseSend. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) CloseSend() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseSend", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).CloseSend)) +} + +// Context mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) Context() context.Context { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Context") + ret0, _ := ret[0].(context.Context) + return ret0 +} + +// Context indicates an expected call of Context. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Context() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Context)) +} + +// Header mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) Header() (metadata.MD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Header") + ret0, _ := ret[0].(metadata.MD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Header indicates an expected call of Header. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Header() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Header)) +} + +// Recv mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) Recv() (*v1.GlobalReduceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Recv") + ret0, _ := ret[0].(*v1.GlobalReduceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Recv indicates an expected call of Recv. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Recv() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Recv)) +} + +// RecvMsg mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) RecvMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecvMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecvMsg indicates an expected call of RecvMsg. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).RecvMsg), arg0) +} + +// Send mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) Send(arg0 *v1.GlobalReduceRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Send(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Send), arg0) +} + +// SendMsg mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) SendMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMsg indicates an expected call of SendMsg. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).SendMsg), arg0) +} + +// Trailer mocks base method. +func (m *MockGlobalReduce_GlobalReduceFnClient) Trailer() metadata.MD { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Trailer") + ret0, _ := ret[0].(metadata.MD) + return ret0 +} + +// Trailer indicates an expected call of Trailer. +func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Trailer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Trailer)) +} diff --git a/pkg/apis/proto/globalreduce/v1/mockgen.go b/pkg/apis/proto/globalreduce/v1/mockgen.go new file mode 100644 index 00000000..1927cb2e --- /dev/null +++ b/pkg/apis/proto/globalreduce/v1/mockgen.go @@ -0,0 +1,3 @@ +package v1 + +//go:generate mockgen -destination globalreducemock/globalreducemock -package globalreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1 GlobalReduceClient,GlobalReduce_GlobalReduceFnClient diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index d777356d..4658e168 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -205,7 +205,7 @@ type SessionReduceResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Result []*SessionReduceResponse_Result `protobuf:"bytes,1,rep,name=result,proto3" json:"result,omitempty"` + Results []*SessionReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` // the key which was used for demultiplexing the request stream. @@ -246,9 +246,9 @@ func (*SessionReduceResponse) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{2} } -func (x *SessionReduceResponse) GetResult() []*SessionReduceResponse_Result { +func (x *SessionReduceResponse) GetResults() []*SessionReduceResponse_Result { if x != nil { - return x.Result + return x.Results } return nil } @@ -572,46 +572,46 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, - 0x22, 0xbf, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x06, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, - 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, - 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, - 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, - 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, 0x0a, 0x0d, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x0f, 0x53, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x26, - 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, - 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, - 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, - 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x22, 0xc1, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x07, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, + 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, + 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, 0x0a, 0x0d, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x66, 0x0a, + 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, + 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, + 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, + 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, + 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, + 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -645,7 +645,7 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ 8, // 1: sessionreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp 6, // 2: sessionreduce.v1.SessionReduceRequest.payload:type_name -> sessionreduce.v1.SessionReduceRequest.Payload 5, // 3: sessionreduce.v1.SessionReduceRequest.operation:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation - 7, // 4: sessionreduce.v1.SessionReduceResponse.result:type_name -> sessionreduce.v1.SessionReduceResponse.Result + 7, // 4: sessionreduce.v1.SessionReduceResponse.results:type_name -> sessionreduce.v1.SessionReduceResponse.Result 1, // 5: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition 8, // 6: sessionreduce.v1.SessionReduceResponse.event_time:type_name -> google.protobuf.Timestamp 0, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto index c07c6f24..0682e131 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -65,7 +65,7 @@ message SessionReduceResponse { repeated string tags = 3; } - repeated Result result = 1; + repeated Result results = 1; // Partition represents a window partition to which the result belongs. Partition partition = 2; diff --git a/pkg/globalreducer/doc.go b/pkg/globalreducer/doc.go new file mode 100644 index 00000000..e8c00ec9 --- /dev/null +++ b/pkg/globalreducer/doc.go @@ -0,0 +1,5 @@ +// Package reducer implements the server code for reduce operation. + +// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/ + +package globalreducer diff --git a/pkg/globalreducer/interface.go b/pkg/globalreducer/interface.go new file mode 100644 index 00000000..9e213024 --- /dev/null +++ b/pkg/globalreducer/interface.go @@ -0,0 +1,18 @@ +package globalreducer + +import ( + "context" + "time" +) + +// Datum contains methods to get the payload information. +type Datum interface { + Value() []byte + EventTime() time.Time + Watermark() time.Time +} + +// GlobalReducer is the interface which can be used to implement a session reduce operation. +type GlobalReducer interface { + GlobalReduce(ctx context.Context, keys []string, input <-chan Datum, output <-chan Messages) +} diff --git a/pkg/globalreducer/message.go b/pkg/globalreducer/message.go new file mode 100644 index 00000000..91167f34 --- /dev/null +++ b/pkg/globalreducer/message.go @@ -0,0 +1,70 @@ +package globalreducer + +import "fmt" + +var ( + DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ +) + +// Message is used to wrap the data return by reduce functions +type Message struct { + value []byte + keys []string + tags []string +} + +// NewMessage creates a Message with value +func NewMessage(value []byte) Message { + return Message{value: value} +} + +// MessageToDrop creates a Message to be dropped +func MessageToDrop() Message { + return Message{value: []byte{}, tags: []string{DROP}} +} + +// WithKeys is used to assign the keys to the message +func (m Message) WithKeys(keys []string) Message { + m.keys = keys + return m +} + +// WithTags is used to assign the tags to the message +// tags will be used for conditional forwarding +func (m Message) WithTags(tags []string) Message { + m.tags = tags + return m +} + +// Keys returns message keys +func (m Message) Keys() []string { + return m.keys +} + +// Value returns message value +func (m Message) Value() []byte { + return m.value +} + +// Tags returns message tags +func (m Message) Tags() []string { + return m.tags +} + +type Messages []Message + +// MessagesBuilder returns an empty instance of Messages +func MessagesBuilder() Messages { + return Messages{} +} + +// Append appends a Message +func (m Messages) Append(msg Message) Messages { + m = append(m, msg) + return m +} + +// Items returns the message list +func (m Messages) Items() []Message { + return m +} diff --git a/pkg/globalreducer/options.go b/pkg/globalreducer/options.go new file mode 100644 index 00000000..76a59c69 --- /dev/null +++ b/pkg/globalreducer/options.go @@ -0,0 +1,43 @@ +package globalreducer + +import ( + "github.com/numaproj/numaflow-go/pkg/info" +) + +type options struct { + sockAddr string + maxMessageSize int + serverInfoFilePath string +} + +// Option is the interface to apply options. +type Option func(*options) + +func DefaultOptions() *options { + return &options{ + sockAddr: address, + maxMessageSize: defaultMaxMessageSize, + serverInfoFilePath: info.ServerInfoFilePath, + } +} + +// WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. +func WithMaxMessageSize(size int) Option { + return func(opts *options) { + opts.maxMessageSize = size + } +} + +// WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. +func WithSockAddr(addr string) Option { + return func(opts *options) { + opts.sockAddr = addr + } +} + +// WithServerInfoFilePath sets the server info file path to the given path. +func WithServerInfoFilePath(f string) Option { + return func(opts *options) { + opts.serverInfoFilePath = f + } +} diff --git a/pkg/globalreducer/options_test.go b/pkg/globalreducer/options_test.go new file mode 100644 index 00000000..93029ba7 --- /dev/null +++ b/pkg/globalreducer/options_test.go @@ -0,0 +1,18 @@ +package globalreducer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithMaxMessageSize(t *testing.T) { + var ( + size = 1024 * 1024 * 10 + opts = &options{ + maxMessageSize: defaultMaxMessageSize, + } + ) + WithMaxMessageSize(1024 * 1024 * 10)(opts) + assert.Equal(t, size, opts.maxMessageSize) +} diff --git a/pkg/globalreducer/server.go b/pkg/globalreducer/server.go new file mode 100644 index 00000000..56833765 --- /dev/null +++ b/pkg/globalreducer/server.go @@ -0,0 +1,56 @@ +package globalreducer + +import ( + "context" + "fmt" + "os/signal" + "syscall" + + "github.com/numaproj/numaflow-go/pkg" + globalreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" + "github.com/numaproj/numaflow-go/pkg/shared" +) + +// server is a global reduce gRPC server. +type server struct { + svc *Service + opts *options +} + +// NewServer creates a new global reduce server. +func NewServer(r GlobalReducer, inputOptions ...Option) numaflow.Server { + opts := DefaultOptions() + for _, inputOption := range inputOptions { + inputOption(opts) + } + s := new(server) + s.svc = new(Service) + s.svc.globalReducer = r + s.opts = opts + return s +} + +// Start starts the global reduce gRPC server. +func (r *server) Start(ctx context.Context) error { + ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // write server info to the file + // start listening on unix domain socket + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath) + if err != nil { + return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) + } + // close the listener + defer func() { _ = lis.Close() }() + + // create a grpc server + grpcServer := shared.CreateGRPCServer(r.opts.maxMessageSize) + defer grpcServer.GracefulStop() + + // register the globalReduce service + globalreducepb.RegisterGlobalReduceServer(grpcServer, r.svc) + + // start the grpc server + return shared.StartGRPCServer(ctxWithSignal, grpcServer, lis) +} diff --git a/pkg/globalreducer/service.go b/pkg/globalreducer/service.go new file mode 100644 index 00000000..d570686c --- /dev/null +++ b/pkg/globalreducer/service.go @@ -0,0 +1,100 @@ +package globalreducer + +import ( + "context" + "io" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + globalreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" +) + +const ( + uds = "unix" + defaultMaxMessageSize = 1024 * 1024 * 64 + address = "/var/run/numaflow/globalreduce.sock" + delimiter = ":" +) + +// Service implements the proto gen server interface and contains the globalreduce operation handler. +type Service struct { + globalreducepb.UnimplementedGlobalReduceServer + globalReducer GlobalReducer +} + +// IsReady returns true to indicate the gRPC connection is ready. +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*globalreducepb.ReadyResponse, error) { + return &globalreducepb.ReadyResponse{Ready: true}, nil +} + +// GlobalReduceFn applies a global reduce function to a request stream and streams the results. +func (fs *Service) GlobalReduceFn(stream globalreducepb.GlobalReduce_GlobalReduceFnServer) error { + + ctx := stream.Context() + taskManager := newReduceTaskManager() + // err group for the go routine which reads from the output channel and sends to the stream + var g errgroup.Group + + g.Go(func() error { + for output := range taskManager.OutputChannel() { + err := stream.Send(output) + if err != nil { + return err + } + } + return nil + }) + + for { + d, recvErr := stream.Recv() + + // if the stream is closed, close all the tasks and break + if recvErr == io.EOF { + taskManager.CloseAll() + break + } + + if recvErr != nil { + statusErr := status.Errorf(codes.Internal, recvErr.Error()) + return statusErr + } + + // invoke the appropriate task manager method based on the operation + switch d.Operation.Event { + case globalreducepb.GlobalReduceRequest_WindowOperation_OPEN: + // create a new task and start the global reduce operation + // also append the datum to the task + err := taskManager.CreateTask(ctx, d, fs.globalReducer) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case globalreducepb.GlobalReduceRequest_WindowOperation_CLOSE: + // close the task + taskManager.CloseTask(d) + case globalreducepb.GlobalReduceRequest_WindowOperation_APPEND: + // append the datum to the task + err := taskManager.AppendToTask(d, fs.globalReducer) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + } + + } + + // wait for all the tasks to return + taskManager.WaitAll() + + // wait for the go routine which reads from the output channel and sends to the stream to return + err := g.Wait() + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + + return nil +} diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go index 90d5b0fb..08de9353 100644 --- a/pkg/globalreducer/task_manager.go +++ b/pkg/globalreducer/task_manager.go @@ -1,210 +1,214 @@ package globalreducer -// -//import ( -// "context" -// "fmt" -// "strings" -// "sync" -// "time" -// -// v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" -//) -// -//type reduceTask struct { -// combinedKey string -// partition *v1.Partition -// reduceStreamer SessionReducer -// watermark time.Time -// inputCh chan Datum -// outputCh chan Message -// Done chan struct{} -//} -// -//func (rt *reduceTask) buildSessionReduceResponse(message Message) *v1.SessionReduceResponse { -// response := &v1.SessionReduceResponse{ -// Result: &v1.SessionReduceResponse_Result{ -// Keys: message.Keys(), -// Value: message.Value(), -// Tags: message.Tags(), -// }, -// Partition: rt.partition, -// CombinedKey: rt.combinedKey, -// Cob: false, -// } -// -// return response -//} -// -//func (rt *reduceTask) buildCobSessionReduceResponse() *v1.SessionReduceResponse { -// response := &v1.SessionReduceResponse{ -// Result: &v1.SessionReduceResponse_Result{ -// Keys: []string{}, -// Value: []byte{}, -// Tags: []string{}, -// }, -// Partition: rt.partition, -// CombinedKey: rt.combinedKey, -// Cob: true, -// } -// -// return response -//} -// -//func (rt *reduceTask) uniqueKey() string { -// return fmt.Sprintf("%d:%d:%s", -// rt.partition.GetStart().AsTime().UnixMilli(), -// rt.partition.GetEnd().AsTime().UnixMilli(), -// rt.combinedKey) -//} -// -//type reduceTaskManager struct { -// Tasks map[string]*reduceTask -// Output chan *v1.SessionReduceResponse -// Mutex sync.RWMutex -//} -// -//func newReduceTaskManager() *reduceTaskManager { -// return &reduceTaskManager{ -// Tasks: make(map[string]*reduceTask), -// Output: make(chan *v1.SessionReduceResponse), -// } -//} -// -//func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, reduceStreamer SessionReducer, partition *v1.Partition) { -// rtm.Mutex.Lock() -// defer rtm.Mutex.Unlock() -// -// task := &reduceTask{ -// combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), -// partition: partition, -// reduceStreamer: reduceStreamer, -// inputCh: make(chan Datum), -// outputCh: make(chan Message), -// Done: make(chan struct{}), -// } -// -// go func() { -// defer close(task.Done) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// defer wg.Done() -// for { -// select { -// case <-ctx.Done(): -// return -// case message, ok := <-task.outputCh: -// if !ok { -// rtm.Output <- task.buildCobSessionReduceResponse() -// return -// } -// rtm.Output <- task.buildSessionReduceResponse(message) -// } -// } -// }() -// reduceStreamer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) -// close(task.outputCh) -// wg.Wait() -// }() -// -// rtm.Tasks[task.uniqueKey()] = task -//} -// -//func (rtm *reduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) error { -// rtm.Mutex.RLock() -// task, ok := rtm.Tasks[generateKey(request.Operation.Partitions[0], request.Payload.Keys)] -// rtm.Mutex.RUnlock() -// -// if !ok { -// return fmt.Errorf("task not found") -// } -// -// task.inputCh <- buildDatum(request) -// return nil -//} -// -//func (rtm *reduceTaskManager) CloseTask(request *v1.SessionReduceRequest) error { -// rtm.Mutex.Lock() -// tasksToBeClosed := make([]*reduceTask, 0, len(request.Operation.Partitions)) -// for _, partition := range request.Operation.Partitions { -// key := generateKey(partition, request.Payload.Keys) -// task, ok := rtm.Tasks[key] -// if ok { -// tasksToBeClosed = append(tasksToBeClosed, task) -// } -// delete(rtm.Tasks, key) -// } -// rtm.Mutex.Unlock() -// -// for _, task := range tasksToBeClosed { -// close(task.inputCh) -// } -// -// return nil -//} -// -//func (rtm *reduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { -// rtm.Mutex.Lock() -// tasks := make([]*reduceTask, 0, len(request.Operation.Partitions)) -// for _, partition := range request.Operation.Partitions { -// key := generateKey(partition, request.Payload.Keys) -// task, ok := rtm.Tasks[key] -// if !ok { -// rtm.Mutex.Unlock() -// return fmt.Errorf("task not found") -// } -// tasks = append(tasks, task) -// } -// rtm.Mutex.Unlock() -// -// if len(tasks) == 0 { -// return nil -// } -// -// mainTask := tasks[0] -// aggregators := make([][]byte, 0, len(tasks)-1) -// -// for _, task := range tasks[1:] { -// close(task.inputCh) -// aggregators = append(aggregators, task.reduceStreamer.Aggregator(ctx)) -// } -// -// for _, aggregator := range aggregators { -// mainTask.reduceStreamer.MergeAggregator(ctx, aggregator) -// } -// -// if request.Payload != nil { -// mainTask.inputCh <- buildDatum(request) -// } -// -// return nil -//} -// -//func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { -// return rtm.Output -//} -// -//func (rtm *reduceTaskManager) WaitAll() { -// rtm.Mutex.RLock() -// tasks := make([]*reduceTask, 0, len(rtm.Tasks)) -// for _, task := range rtm.Tasks { -// tasks = append(tasks, task) -// } -// rtm.Mutex.RUnlock() -// -// for _, task := range tasks { -// <-task.Done -// } -//} -// -//func generateKey(partition *v1.Partition, keys []string) string { -// return fmt.Sprintf("%d:%d:%s", -// partition.GetStart().AsTime().UnixMilli(), -// partition.GetEnd().AsTime().UnixMilli(), -// strings.Join(keys, delimiter)) -//} -// -//func buildDatum(request *v1.SessionReduceRequest) Datum { -// return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) -//} +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "go.uber.org/atomic" + "google.golang.org/protobuf/types/known/timestamppb" + + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" +) + +// globalReduceTask represents a global reduce task for a global reduce operation. +type globalReduceTask struct { + combinedKey string + partition *v1.Partition + reduceStreamer GlobalReducer + inputCh chan Datum + outputCh chan Messages + latestWatermark *atomic.Time + Done chan struct{} +} + +// buildGlobalReduceResponse builds the global reduce response from the messages. +func (rt *globalReduceTask) buildGlobalReduceResponse(messages Messages) *v1.GlobalReduceResponse { + results := make([]*v1.GlobalReduceResponse_Result, 0, len(messages)) + for _, message := range messages { + results = append(results, &v1.GlobalReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + }) + } + + // the event time is the latest watermark + // since for global window, partition start and end time will be -1 + response := &v1.GlobalReduceResponse{ + Results: results, + Partition: rt.partition, + CombinedKey: rt.combinedKey, + EventTime: timestamppb.New(rt.latestWatermark.Load()), + } + + return response +} + +// uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. +func (rt *globalReduceTask) uniqueKey() string { + return fmt.Sprintf("%d:%d:%s", + rt.partition.GetStart().AsTime().UnixMilli(), + rt.partition.GetEnd().AsTime().UnixMilli(), + rt.combinedKey) +} + +// globalReduceTaskManager manages the reduce tasks for a global reduce operation. +type globalReduceTaskManager struct { + Tasks map[string]*globalReduceTask + Output chan *v1.GlobalReduceResponse + Mutex sync.RWMutex +} + +func newReduceTaskManager() *globalReduceTaskManager { + return &globalReduceTaskManager{ + Tasks: make(map[string]*globalReduceTask), + Output: make(chan *v1.GlobalReduceResponse), + } +} + +// CreateTask creates a new reduce task and starts the global reduce operation. +func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1.GlobalReduceRequest, globalReducer GlobalReducer) error { + rtm.Mutex.Lock() + defer rtm.Mutex.Unlock() + + if len(request.Operation.Partitions) != 1 { + return fmt.Errorf("invalid number of partitions") + } + + task := &globalReduceTask{ + combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), + partition: request.Operation.Partitions[0], + reduceStreamer: globalReducer, + inputCh: make(chan Datum), + outputCh: make(chan Messages), + Done: make(chan struct{}), + latestWatermark: atomic.NewTime(time.Time{}), + } + + go func() { + defer close(task.Done) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + case message, ok := <-task.outputCh: + if !ok { + return + } + rtm.Output <- task.buildGlobalReduceResponse(message) + } + } + }() + // start the global reduce operation + globalReducer.GlobalReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) + // close the output channel after the global reduce operation is completed + close(task.outputCh) + // wait for the output channel reader to complete + wg.Wait() + }() + + // send the datum to the task if the payload is not nil + // also update the latest watermark + if request.Payload != nil { + task.inputCh <- buildDatum(request) + task.latestWatermark.Store(request.Payload.Watermark.AsTime()) + } + + rtm.Tasks[task.uniqueKey()] = task + return nil +} + +// AppendToTask writes the message to the reduce task. +// If the reduce task is not found, it will create a new reduce task and start the reduce operation. +func (rtm *globalReduceTaskManager) AppendToTask(request *v1.GlobalReduceRequest, globalReducer GlobalReducer) error { + if len(request.Operation.Partitions) != 1 { + return fmt.Errorf("invalid number of partitions") + } + + rtm.Mutex.RLock() + task, ok := rtm.Tasks[generateKey(request.Operation.Partitions[0], request.Payload.Keys)] + rtm.Mutex.RUnlock() + + if !ok { + return rtm.CreateTask(context.Background(), request, globalReducer) + } + + if request.Payload != nil { + task.inputCh <- buildDatum(request) + task.latestWatermark.Store(request.Payload.Watermark.AsTime()) + } + + return nil +} + +// CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. +func (rtm *globalReduceTaskManager) CloseTask(request *v1.GlobalReduceRequest) { + rtm.Mutex.Lock() + tasksToBeClosed := make([]*globalReduceTask, 0, len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + key := generateKey(partition, request.Payload.Keys) + task, ok := rtm.Tasks[key] + if ok { + tasksToBeClosed = append(tasksToBeClosed, task) + } + delete(rtm.Tasks, key) + } + rtm.Mutex.Unlock() + + for _, task := range tasksToBeClosed { + close(task.inputCh) + } +} + +// OutputChannel returns the output channel for the reduce task manager to read the results. +func (rtm *globalReduceTaskManager) OutputChannel() <-chan *v1.GlobalReduceResponse { + return rtm.Output +} + +// WaitAll waits for all the reduce tasks to complete. +func (rtm *globalReduceTaskManager) WaitAll() { + rtm.Mutex.RLock() + tasks := make([]*globalReduceTask, 0, len(rtm.Tasks)) + for _, task := range rtm.Tasks { + tasks = append(tasks, task) + } + rtm.Mutex.RUnlock() + + for _, task := range tasks { + <-task.Done + } + // after all the tasks are completed, close the output channel + close(rtm.Output) +} + +// CloseAll closes all the reduce tasks. +func (rtm *globalReduceTaskManager) CloseAll() { + rtm.Mutex.Lock() + tasks := make([]*globalReduceTask, 0, len(rtm.Tasks)) + for _, task := range rtm.Tasks { + tasks = append(tasks, task) + } + rtm.Mutex.Unlock() + + for _, task := range tasks { + close(task.inputCh) + } +} + +func generateKey(partition *v1.Partition, keys []string) string { + return fmt.Sprintf("%d:%d:%s", + partition.GetStart().AsTime().UnixMilli(), + partition.GetEnd().AsTime().UnixMilli(), + strings.Join(keys, delimiter)) +} + +func buildDatum(request *v1.GlobalReduceRequest) Datum { + return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +} diff --git a/pkg/globalreducer/types.go b/pkg/globalreducer/types.go new file mode 100644 index 00000000..69778e65 --- /dev/null +++ b/pkg/globalreducer/types.go @@ -0,0 +1,65 @@ +package globalreducer + +import "time" + +// handlerDatum implements the Datum interface and is used in the reduce functions. +type handlerDatum struct { + value []byte + eventTime time.Time + watermark time.Time +} + +func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time) Datum { + return &handlerDatum{ + value: value, + eventTime: eventTime, + watermark: watermark, + } +} + +func (h *handlerDatum) Value() []byte { + return h.value +} + +func (h *handlerDatum) EventTime() time.Time { + return h.eventTime +} + +func (h *handlerDatum) Watermark() time.Time { + return h.watermark +} + +// intervalWindow implements IntervalWindow interface which will be passed as metadata +// to reduce handlers +type intervalWindow struct { + startTime time.Time + endTime time.Time +} + +func NewIntervalWindow(startTime time.Time, endTime time.Time) IntervalWindow { + return &intervalWindow{ + startTime: startTime, + endTime: endTime, + } +} + +func (i *intervalWindow) StartTime() time.Time { + return i.startTime +} + +func (i *intervalWindow) EndTime() time.Time { + return i.endTime +} + +// metadata implements Metadata interface which will be passed to reduce function. +type metadata struct { + intervalWindow IntervalWindow +} + +func NewMetadata(window IntervalWindow) Metadata { + return &metadata{intervalWindow: window} +} + +func (m *metadata) IntervalWindow() IntervalWindow { + return m.intervalWindow +} diff --git a/pkg/sessionreducer/interface.go b/pkg/sessionreducer/interface.go index 66930a49..d0f2a169 100644 --- a/pkg/sessionreducer/interface.go +++ b/pkg/sessionreducer/interface.go @@ -19,4 +19,5 @@ type SessionReducer interface { MergeAggregator(ctx context.Context, aggregator []byte) } +// CreateSessionReducer is a function which returns a new instance of SessionReducer. type CreateSessionReducer func() SessionReducer diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go index 88935267..d99f2c5d 100644 --- a/pkg/sessionreducer/service.go +++ b/pkg/sessionreducer/service.go @@ -15,7 +15,7 @@ import ( const ( uds = "unix" defaultMaxMessageSize = 1024 * 1024 * 64 - address = "/var/run/numaflow/sesionreduce.sock" + address = "/var/run/numaflow/sessionreduce.sock" delimiter = ":" ) diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index e0d4e8ce..434c9691 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -17,16 +17,15 @@ type sessionReduceTask struct { combinedKey string partition *v1.Partition reduceStreamer SessionReducer - watermark time.Time inputCh chan Datum Done chan struct{} } // buildSessionReduceResponse builds the session reduce response from the messages. func (rt *sessionReduceTask) buildSessionReduceResponse(messages Messages) *v1.SessionReduceResponse { - result := make([]*v1.SessionReduceResponse_Result, 0, len(messages)) + results := make([]*v1.SessionReduceResponse_Result, 0, len(messages)) for _, message := range messages { - result = append(result, &v1.SessionReduceResponse_Result{ + results = append(results, &v1.SessionReduceResponse_Result{ Keys: message.Keys(), Value: message.Value(), Tags: message.Tags(), @@ -34,7 +33,7 @@ func (rt *sessionReduceTask) buildSessionReduceResponse(messages Messages) *v1.S } response := &v1.SessionReduceResponse{ - Result: result, + Results: results, Partition: rt.partition, CombinedKey: rt.combinedKey, EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), @@ -66,7 +65,7 @@ func newReduceTaskManager() *sessionReduceTaskManager { } // CreateTask creates a new reduce task and starts the session reduce operation. -func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, reduceStreamer SessionReducer) error { +func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, sessionReducer SessionReducer) error { rtm.Mutex.Lock() defer rtm.Mutex.Unlock() @@ -77,17 +76,21 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 task := &sessionReduceTask{ combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), partition: request.Operation.Partitions[0], - reduceStreamer: reduceStreamer, + reduceStreamer: sessionReducer, inputCh: make(chan Datum), Done: make(chan struct{}), } go func() { defer close(task.Done) - msgs := reduceStreamer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh) + msgs := sessionReducer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh) rtm.Output <- task.buildSessionReduceResponse(msgs) }() + // send the datum to the task if the payload is not nil + if request.Payload != nil { + task.inputCh <- buildDatum(request) + } rtm.Tasks[task.uniqueKey()] = task return nil } From e46031d00aab2e2fbcfd8bda6ceb7af50ff3b6d1 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Mon, 13 Nov 2023 11:55:27 +0530 Subject: [PATCH 04/27] fix lock Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.proto | 4 +-- pkg/globalreducer/task_manager.go | 17 +++++++--- pkg/reducer/examples/counter/Makefile | 2 +- pkg/reducer/examples/counter/go.mod | 16 ---------- pkg/reducer/examples/counter/go.sum | 28 ---------------- pkg/reducer/service.go | 6 ++-- pkg/reducer/task_manager.go | 32 +++++-------------- pkg/sessionreducer/interface.go | 4 +-- pkg/sessionreducer/task_manager.go | 24 +++++++++----- 9 files changed, 44 insertions(+), 89 deletions(-) delete mode 100644 pkg/reducer/examples/counter/go.mod delete mode 100644 pkg/reducer/examples/counter/go.sum diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.proto b/pkg/apis/proto/globalreduce/v1/globalreduce.proto index 75a28fbc..69c72317 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.proto +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.proto @@ -28,13 +28,11 @@ message Partition { */ message GlobalReduceRequest { // WindowOperation represents a window operation. - // it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. + // it can be one of OPEN, CLOSE, and APPEND. message WindowOperation { enum Event { OPEN = 0; CLOSE = 1; - EXPAND = 2; - MERGE = 3; APPEND = 4; } diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go index 08de9353..b0749998 100644 --- a/pkg/globalreducer/task_manager.go +++ b/pkg/globalreducer/task_manager.go @@ -72,7 +72,6 @@ func newReduceTaskManager() *globalReduceTaskManager { // CreateTask creates a new reduce task and starts the global reduce operation. func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1.GlobalReduceRequest, globalReducer GlobalReducer) error { rtm.Mutex.Lock() - defer rtm.Mutex.Unlock() if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") @@ -88,23 +87,35 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. latestWatermark: atomic.NewTime(time.Time{}), } + key := task.uniqueKey() + rtm.Tasks[key] = task + + rtm.Mutex.Unlock() + go func() { defer close(task.Done) var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() + readLoop: for { select { case <-ctx.Done(): return case message, ok := <-task.outputCh: if !ok { - return + break readLoop } rtm.Output <- task.buildGlobalReduceResponse(message) } } + // send a done signal + close(task.Done) + // delete the task from the tasks list + rtm.Mutex.Lock() + delete(rtm.Tasks, key) + rtm.Mutex.Unlock() }() // start the global reduce operation globalReducer.GlobalReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) @@ -121,7 +132,6 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. task.latestWatermark.Store(request.Payload.Watermark.AsTime()) } - rtm.Tasks[task.uniqueKey()] = task return nil } @@ -158,7 +168,6 @@ func (rtm *globalReduceTaskManager) CloseTask(request *v1.GlobalReduceRequest) { if ok { tasksToBeClosed = append(tasksToBeClosed, task) } - delete(rtm.Tasks, key) } rtm.Mutex.Unlock() diff --git a/pkg/reducer/examples/counter/Makefile b/pkg/reducer/examples/counter/Makefile index c5696c14..072cd6ab 100644 --- a/pkg/reducer/examples/counter/Makefile +++ b/pkg/reducer/examples/counter/Makefile @@ -4,7 +4,7 @@ build: .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-counter:v0.5.0" --target counter . + docker build -t "quay.io/yhl25/numaflow-go/reduce-counter:v0.5.4" --target counter . clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/counter/go.mod b/pkg/reducer/examples/counter/go.mod deleted file mode 100644 index 8b968e1e..00000000 --- a/pkg/reducer/examples/counter/go.mod +++ /dev/null @@ -1,16 +0,0 @@ -module even_odd - -go 1.20 - -require github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe - -require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect -) diff --git a/pkg/reducer/examples/counter/go.sum b/pkg/reducer/examples/counter/go.sum deleted file mode 100644 index 18c03144..00000000 --- a/pkg/reducer/examples/counter/go.sum +++ /dev/null @@ -1,28 +0,0 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe h1:nK/BGffgwQ4L9pyllwzSZttPxMf+OOqK3DOP97KZdRk= -github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe/go.mod h1:zcJq1YAA/jnxCQLW7EFK4+HXWCd2QtW4tMOvRjHFa2g= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/reducer/service.go b/pkg/reducer/service.go index 3ebd0e51..e2b22abe 100644 --- a/pkg/reducer/service.go +++ b/pkg/reducer/service.go @@ -68,6 +68,9 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { return recvErr } + // for fixed and sliding, its just open or append operation + // close signal will be sent to all the reducers when grpc + // input stream gets EOF. switch d.Operation.Event { case reducepb.ReduceRequest_WindowOperation_OPEN: // create a new reduce task and start the reduce operation @@ -76,9 +79,6 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr } - case reducepb.ReduceRequest_WindowOperation_CLOSE: - // close the reduce task - taskManager.CloseTask(d) case reducepb.ReduceRequest_WindowOperation_APPEND: // append the datum to the reduce task err = taskManager.AppendToTask(d, fs.Reducer) diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index dc31cf96..33380dc8 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -66,8 +66,6 @@ func newReduceTaskManager() *reduceTaskManager { // CreateTask creates a new reduce task and starts the reduce operation. func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest, reducer Reducer) error { rtm.Mutex.Lock() - defer rtm.Mutex.Unlock() - if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } @@ -82,16 +80,21 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce Done: make(chan struct{}), } + key := task.uniqueKey() + rtm.Tasks[key] = task + + rtm.Mutex.Unlock() + go func() { - defer close(task.Done) msgs := reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) + // write the output to the output channel, service will forward it to downstream rtm.Output <- task.buildReduceResponse(msgs) + // send a done signal + close(task.Done) }() // write the first message to the input channel task.inputCh <- buildDatum(request) - - rtm.Tasks[task.uniqueKey()] = task return nil } @@ -112,25 +115,6 @@ func (rtm *reduceTaskManager) AppendToTask(request *v1.ReduceRequest, reducer Re return nil } -// CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. -func (rtm *reduceTaskManager) CloseTask(request *v1.ReduceRequest) { - rtm.Mutex.Lock() - tasksToBeClosed := make([]*reduceTask, 0, len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - key := generateKey(partition, request.Payload.Keys) - task, ok := rtm.Tasks[key] - if ok { - tasksToBeClosed = append(tasksToBeClosed, task) - } - delete(rtm.Tasks, key) - } - rtm.Mutex.Unlock() - - for _, task := range tasksToBeClosed { - close(task.inputCh) - } -} - // OutputChannel returns the output channel for the reduce task manager to read the results. func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { return rtm.Output diff --git a/pkg/sessionreducer/interface.go b/pkg/sessionreducer/interface.go index d0f2a169..ec2289f3 100644 --- a/pkg/sessionreducer/interface.go +++ b/pkg/sessionreducer/interface.go @@ -15,8 +15,8 @@ type Datum interface { // SessionReducer is the interface which can be used to implement a session reduce operation. type SessionReducer interface { SessionReduce(ctx context.Context, keys []string, input <-chan Datum) Messages - Aggregator(ctx context.Context) []byte - MergeAggregator(ctx context.Context, aggregator []byte) + Accumulator(ctx context.Context) []byte + MergeAccumulator(ctx context.Context, accumulator []byte) } // CreateSessionReducer is a function which returns a new instance of SessionReducer. diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index 434c9691..9f284c48 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -67,8 +67,6 @@ func newReduceTaskManager() *sessionReduceTaskManager { // CreateTask creates a new reduce task and starts the session reduce operation. func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, sessionReducer SessionReducer) error { rtm.Mutex.Lock() - defer rtm.Mutex.Unlock() - if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } @@ -81,17 +79,28 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 Done: make(chan struct{}), } + key := task.uniqueKey() + rtm.Tasks[key] = task + + rtm.Mutex.Unlock() + go func() { - defer close(task.Done) msgs := sessionReducer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh) + // write the output to the output channel, service will forward it to downstream rtm.Output <- task.buildSessionReduceResponse(msgs) + // send a done signal + close(task.Done) + // delete the task from the tasks list + rtm.Mutex.Lock() + delete(rtm.Tasks, key) + rtm.Mutex.Unlock() }() // send the datum to the task if the payload is not nil if request.Payload != nil { task.inputCh <- buildDatum(request) } - rtm.Tasks[task.uniqueKey()] = task + return nil } @@ -124,7 +133,6 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) if ok { tasksToBeClosed = append(tasksToBeClosed, task) } - delete(rtm.Tasks, key) } rtm.Mutex.Unlock() @@ -158,11 +166,11 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 for _, task := range tasks[1:] { close(task.inputCh) - aggregators = append(aggregators, task.reduceStreamer.Aggregator(ctx)) + aggregators = append(aggregators, task.reduceStreamer.Accumulator(ctx)) } for _, aggregator := range aggregators { - mainTask.reduceStreamer.MergeAggregator(ctx, aggregator) + mainTask.reduceStreamer.MergeAccumulator(ctx, aggregator) } if request.Payload != nil { @@ -200,7 +208,7 @@ func (rtm *sessionReduceTaskManager) OutputChannel() <-chan *v1.SessionReduceRes return rtm.Output } -// WaitAll waits for all the reduce tasks to complete. +// WaitAll waits for all the pending reduce tasks to complete. func (rtm *sessionReduceTaskManager) WaitAll() { rtm.Mutex.RLock() tasks := make([]*sessionReduceTask, 0, len(rtm.Tasks)) From 56ee7bac90fadba6eca3a4f12d718361a0c7ebeb Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Mon, 13 Nov 2023 21:09:07 +0530 Subject: [PATCH 05/27] fix global reduce Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.pb.go | 4 +-- pkg/apis/proto/reduce/v1/reduce.pb.go | 4 +-- .../sessionreduce/v1/sessionreduce.pb.go | 4 +-- pkg/mapper/examples/even_odd/main.go | 2 +- pkg/mapper/examples/flatmap/main.go | 2 +- .../examples/flatmap_stream/main.go | 2 +- pkg/sessionreducer/server_test.go | 31 ++++++++++++++++ pkg/sessionreducer/types.go | 35 ------------------- 8 files changed, 40 insertions(+), 44 deletions(-) create mode 100644 pkg/sessionreducer/server_test.go diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go index cbefc269..c55c1900 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go @@ -627,7 +627,7 @@ func file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP() []byte var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes = []interface{}{ - (GlobalReduceRequest_WindowOperation_Event)(0), // 0: globalreduce.v1.GlobalReduceRequest.WindowOperation.Event + (GlobalReduceRequest_WindowOperation_Event)(0), // 0: globalreduce.v1.GlobalReduceRequest.WindowOperation.Operation (*Partition)(nil), // 1: globalreduce.v1.Partition (*GlobalReduceRequest)(nil), // 2: globalreduce.v1.GlobalReduceRequest (*GlobalReduceResponse)(nil), // 3: globalreduce.v1.GlobalReduceResponse @@ -646,7 +646,7 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = []int32{ 7, // 4: globalreduce.v1.GlobalReduceResponse.results:type_name -> globalreduce.v1.GlobalReduceResponse.Result 1, // 5: globalreduce.v1.GlobalReduceResponse.partition:type_name -> globalreduce.v1.Partition 8, // 6: globalreduce.v1.GlobalReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Event + 0, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Operation 1, // 8: globalreduce.v1.GlobalReduceRequest.WindowOperation.partitions:type_name -> globalreduce.v1.Partition 8, // 9: globalreduce.v1.GlobalReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 10: globalreduce.v1.GlobalReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index e24d241a..97a7ee45 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -600,7 +600,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP() []byte { var file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []interface{}{ - (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Event + (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Operation (*ReduceRequest)(nil), // 1: reduce.v1.ReduceRequest (*Partition)(nil), // 2: reduce.v1.Partition (*ReduceResponse)(nil), // 3: reduce.v1.ReduceResponse @@ -619,7 +619,7 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs = []int32{ 7, // 4: reduce.v1.ReduceResponse.results:type_name -> reduce.v1.ReduceResponse.Result 2, // 5: reduce.v1.ReduceResponse.partition:type_name -> reduce.v1.Partition 8, // 6: reduce.v1.ReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Event + 0, // 7: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Operation 2, // 8: reduce.v1.ReduceRequest.WindowOperation.partitions:type_name -> reduce.v1.Partition 8, // 9: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 10: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index 4658e168..1d53838a 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -629,7 +629,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP() []by var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []interface{}{ - (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Operation (*Partition)(nil), // 1: sessionreduce.v1.Partition (*SessionReduceRequest)(nil), // 2: sessionreduce.v1.SessionReduceRequest (*SessionReduceResponse)(nil), // 3: sessionreduce.v1.SessionReduceResponse @@ -648,7 +648,7 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ 7, // 4: sessionreduce.v1.SessionReduceResponse.results:type_name -> sessionreduce.v1.SessionReduceResponse.Result 1, // 5: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition 8, // 6: sessionreduce.v1.SessionReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + 0, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Operation 1, // 8: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 10: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp diff --git a/pkg/mapper/examples/even_odd/main.go b/pkg/mapper/examples/even_odd/main.go index 92fd0040..ffec9ef2 100644 --- a/pkg/mapper/examples/even_odd/main.go +++ b/pkg/mapper/examples/even_odd/main.go @@ -13,7 +13,7 @@ type EvenOdd struct { func (e *EvenOdd) Map(ctx context.Context, keys []string, d mapper.Datum) mapper.Messages { msg := d.Value() - _ = d.EventTime() // Event time is available + _ = d.EventTime() // Operation time is available _ = d.Watermark() // Watermark is available // If msg is not an integer, drop it, otherwise return it with "even" or "odd" key. if num, err := strconv.Atoi(string(msg)); err != nil { diff --git a/pkg/mapper/examples/flatmap/main.go b/pkg/mapper/examples/flatmap/main.go index 2a5fa18e..b6ad0400 100644 --- a/pkg/mapper/examples/flatmap/main.go +++ b/pkg/mapper/examples/flatmap/main.go @@ -10,7 +10,7 @@ import ( func mapFn(_ context.Context, keys []string, d mapper.Datum) mapper.Messages { msg := d.Value() - _ = d.EventTime() // Event time is available + _ = d.EventTime() // Operation time is available _ = d.Watermark() // Watermark is available // Split the msg into an array with comma. strs := strings.Split(string(msg), ",") diff --git a/pkg/mapstreamer/examples/flatmap_stream/main.go b/pkg/mapstreamer/examples/flatmap_stream/main.go index a72ab3f3..f333be48 100644 --- a/pkg/mapstreamer/examples/flatmap_stream/main.go +++ b/pkg/mapstreamer/examples/flatmap_stream/main.go @@ -15,7 +15,7 @@ type FlatMap struct { func (f *FlatMap) MapStream(ctx context.Context, keys []string, d mapstreamer.Datum, messageCh chan<- mapstreamer.Message) { defer close(messageCh) msg := d.Value() - _ = d.EventTime() // Event time is available + _ = d.EventTime() // Operation time is available _ = d.Watermark() // Watermark is available // Split the msg into an array with comma. strs := strings.Split(string(msg), ",") diff --git a/pkg/sessionreducer/server_test.go b/pkg/sessionreducer/server_test.go new file mode 100644 index 00000000..c7ce0c9e --- /dev/null +++ b/pkg/sessionreducer/server_test.go @@ -0,0 +1,31 @@ +package sessionreducer + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestReduceServer_Start(t *testing.T) { + socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") + defer func() { + _ = os.RemoveAll(socketFile.Name()) + }() + + serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") + defer func() { + _ = os.RemoveAll(serverInfoFile.Name()) + }() + + var reduceHandler = CreateSessionReducer(func() SessionReducer { + return nil + }) + // note: using actual uds connection + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + err := NewServer(reduceHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) + assert.NoError(t, err) +} diff --git a/pkg/sessionreducer/types.go b/pkg/sessionreducer/types.go index 6266c01a..68bbec9f 100644 --- a/pkg/sessionreducer/types.go +++ b/pkg/sessionreducer/types.go @@ -28,38 +28,3 @@ func (h *handlerDatum) EventTime() time.Time { func (h *handlerDatum) Watermark() time.Time { return h.watermark } - -// intervalWindow implements IntervalWindow interface which will be passed as metadata -// to reduce handlers -type intervalWindow struct { - startTime time.Time - endTime time.Time -} - -func NewIntervalWindow(startTime time.Time, endTime time.Time) IntervalWindow { - return &intervalWindow{ - startTime: startTime, - endTime: endTime, - } -} - -func (i *intervalWindow) StartTime() time.Time { - return i.startTime -} - -func (i *intervalWindow) EndTime() time.Time { - return i.endTime -} - -// metadata implements Metadata interface which will be passed to reduce function. -type metadata struct { - intervalWindow IntervalWindow -} - -func NewMetadata(window IntervalWindow) Metadata { - return &metadata{intervalWindow: window} -} - -func (m *metadata) IntervalWindow() IntervalWindow { - return m.intervalWindow -} From 52c946f82e4f235d2a32d1972bf7cc6df41a4e1c Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Fri, 17 Nov 2023 07:52:01 +0530 Subject: [PATCH 06/27] minor changes Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.pb.go | 121 +++++++++--------- pkg/apis/proto/globalreduce/v1/mockgen.go | 2 +- pkg/apis/proto/reduce/v1/reduce.pb.go | 8 +- pkg/apis/proto/reduce/v1/reduce.proto | 2 +- .../sessionreduce/v1/sessionreduce.pb.go | 4 +- 5 files changed, 65 insertions(+), 72 deletions(-) diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go index c55c1900..cb6f0f15 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go @@ -27,8 +27,6 @@ type GlobalReduceRequest_WindowOperation_Event int32 const ( GlobalReduceRequest_WindowOperation_OPEN GlobalReduceRequest_WindowOperation_Event = 0 GlobalReduceRequest_WindowOperation_CLOSE GlobalReduceRequest_WindowOperation_Event = 1 - GlobalReduceRequest_WindowOperation_EXPAND GlobalReduceRequest_WindowOperation_Event = 2 - GlobalReduceRequest_WindowOperation_MERGE GlobalReduceRequest_WindowOperation_Event = 3 GlobalReduceRequest_WindowOperation_APPEND GlobalReduceRequest_WindowOperation_Event = 4 ) @@ -37,15 +35,11 @@ var ( GlobalReduceRequest_WindowOperation_Event_name = map[int32]string{ 0: "OPEN", 1: "CLOSE", - 2: "EXPAND", - 3: "MERGE", 4: "APPEND", } GlobalReduceRequest_WindowOperation_Event_value = map[string]int32{ "OPEN": 0, "CLOSE": 1, - "EXPAND": 2, - "MERGE": 3, "APPEND": 4, } ) @@ -324,7 +318,7 @@ func (x *ReadyResponse) GetReady() bool { } // WindowOperation represents a window operation. -// it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. +// it can be one of OPEN, CLOSE, and APPEND. type GlobalReduceRequest_WindowOperation struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -535,7 +529,7 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, - 0x6c, 0x6f, 0x74, 0x22, 0xbf, 0x04, 0x0a, 0x13, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, + 0x6c, 0x6f, 0x74, 0x22, 0xa8, 0x04, 0x0a, 0x13, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x46, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, @@ -546,7 +540,7 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xe0, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xc9, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, @@ -556,60 +550,59 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, - 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, - 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, - 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, - 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, - 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xbd, 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, - 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, - 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, - 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, - 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, - 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, - 0x0c, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, - 0x0e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, - 0x24, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, - 0x12, 0x41, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, - 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, + 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, + 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, + 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xbd, + 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, + 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, + 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, + 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, + 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, + 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, + 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, + 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, + 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, + 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x07, 0x49, 0x73, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x67, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, + 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -627,7 +620,7 @@ func file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP() []byte var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes = []interface{}{ - (GlobalReduceRequest_WindowOperation_Event)(0), // 0: globalreduce.v1.GlobalReduceRequest.WindowOperation.Operation + (GlobalReduceRequest_WindowOperation_Event)(0), // 0: globalreduce.v1.GlobalReduceRequest.WindowOperation.Event (*Partition)(nil), // 1: globalreduce.v1.Partition (*GlobalReduceRequest)(nil), // 2: globalreduce.v1.GlobalReduceRequest (*GlobalReduceResponse)(nil), // 3: globalreduce.v1.GlobalReduceResponse @@ -646,7 +639,7 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = []int32{ 7, // 4: globalreduce.v1.GlobalReduceResponse.results:type_name -> globalreduce.v1.GlobalReduceResponse.Result 1, // 5: globalreduce.v1.GlobalReduceResponse.partition:type_name -> globalreduce.v1.Partition 8, // 6: globalreduce.v1.GlobalReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Operation + 0, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Event 1, // 8: globalreduce.v1.GlobalReduceRequest.WindowOperation.partitions:type_name -> globalreduce.v1.Partition 8, // 9: globalreduce.v1.GlobalReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 10: globalreduce.v1.GlobalReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp diff --git a/pkg/apis/proto/globalreduce/v1/mockgen.go b/pkg/apis/proto/globalreduce/v1/mockgen.go index 1927cb2e..3dc4e37a 100644 --- a/pkg/apis/proto/globalreduce/v1/mockgen.go +++ b/pkg/apis/proto/globalreduce/v1/mockgen.go @@ -1,3 +1,3 @@ package v1 -//go:generate mockgen -destination globalreducemock/globalreducemock -package globalreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1 GlobalReduceClient,GlobalReduce_GlobalReduceFnClient +//go:generate mockgen -destination globalreducemock/globalreducemock.go -package globalreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1 GlobalReduceClient,GlobalReduce_GlobalReduceFnClient diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index 97a7ee45..4d597c4c 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -203,7 +203,7 @@ type ReduceResponse struct { // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` // EventTime represents the event time of the result. - EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } func (x *ReduceResponse) Reset() { @@ -559,7 +559,7 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, + 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, @@ -600,7 +600,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP() []byte { var file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []interface{}{ - (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Operation + (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Event (*ReduceRequest)(nil), // 1: reduce.v1.ReduceRequest (*Partition)(nil), // 2: reduce.v1.Partition (*ReduceResponse)(nil), // 3: reduce.v1.ReduceResponse @@ -619,7 +619,7 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs = []int32{ 7, // 4: reduce.v1.ReduceResponse.results:type_name -> reduce.v1.ReduceResponse.Result 2, // 5: reduce.v1.ReduceResponse.partition:type_name -> reduce.v1.Partition 8, // 6: reduce.v1.ReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Operation + 0, // 7: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Event 2, // 8: reduce.v1.ReduceRequest.WindowOperation.partitions:type_name -> reduce.v1.Partition 8, // 9: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 10: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp diff --git a/pkg/apis/proto/reduce/v1/reduce.proto b/pkg/apis/proto/reduce/v1/reduce.proto index 3bafc9f7..91b55b4e 100644 --- a/pkg/apis/proto/reduce/v1/reduce.proto +++ b/pkg/apis/proto/reduce/v1/reduce.proto @@ -69,7 +69,7 @@ message ReduceResponse { Partition partition = 2; // EventTime represents the event time of the result. - google.protobuf.Timestamp event_time = 4; + google.protobuf.Timestamp event_time = 3; } /** diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index 1d53838a..4658e168 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -629,7 +629,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP() []by var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []interface{}{ - (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Operation + (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event (*Partition)(nil), // 1: sessionreduce.v1.Partition (*SessionReduceRequest)(nil), // 2: sessionreduce.v1.SessionReduceRequest (*SessionReduceResponse)(nil), // 3: sessionreduce.v1.SessionReduceResponse @@ -648,7 +648,7 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ 7, // 4: sessionreduce.v1.SessionReduceResponse.results:type_name -> sessionreduce.v1.SessionReduceResponse.Result 1, // 5: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition 8, // 6: sessionreduce.v1.SessionReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Operation + 0, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event 1, // 8: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 10: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp From 35c5ff7256cfd8228fa77c02646bf2defda29ae5 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Fri, 17 Nov 2023 19:18:28 +0530 Subject: [PATCH 07/27] update partition Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.pb.go | 178 +++++++------- .../proto/globalreduce/v1/globalreduce.proto | 4 +- pkg/apis/proto/sessionreduce/v1/mockgen.go | 2 +- .../sessionreduce/v1/sessionreduce.pb.go | 176 +++++++------- .../sessionreduce/v1/sessionreduce.proto | 6 +- pkg/globalreducer/service.go | 6 +- pkg/globalreducer/task_manager.go | 101 ++++---- pkg/reducer/task_manager.go | 62 ++--- .../examples/counter/Dockerfile | 20 ++ pkg/sessionreducer/examples/counter/Makefile | 10 + pkg/sessionreducer/examples/counter/README.md | 3 + pkg/sessionreducer/examples/counter/main.go | 42 ++++ pkg/sessionreducer/service.go | 6 +- pkg/sessionreducer/task_manager.go | 222 +++++++++++------- 14 files changed, 485 insertions(+), 353 deletions(-) create mode 100644 pkg/sessionreducer/examples/counter/Dockerfile create mode 100644 pkg/sessionreducer/examples/counter/Makefile create mode 100644 pkg/sessionreducer/examples/counter/README.md create mode 100644 pkg/sessionreducer/examples/counter/main.go diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go index cb6f0f15..48e40d54 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go @@ -80,6 +80,7 @@ type Partition struct { Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` + Keys []string `protobuf:"bytes,4,rep,name=keys,proto3" json:"keys,omitempty"` } func (x *Partition) Reset() { @@ -135,6 +136,13 @@ func (x *Partition) GetSlot() string { return "" } +func (x *Partition) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + // * // GlobalReduceRequest represents a request element. type GlobalReduceRequest struct { @@ -202,8 +210,6 @@ type GlobalReduceResponse struct { Results []*GlobalReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - // the key which was used for demultiplexing the request stream. - CombinedKey string `protobuf:"bytes,3,opt,name=combinedKey,proto3" json:"combinedKey,omitempty"` // EventTime represents the event time of the result. EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } @@ -254,13 +260,6 @@ func (x *GlobalReduceResponse) GetPartition() *Partition { return nil } -func (x *GlobalReduceResponse) GetCombinedKey() string { - if x != nil { - return x.CombinedKey - } - return "" -} - func (x *GlobalReduceResponse) GetEventTime() *timestamppb.Timestamp { if x != nil { return x.EventTime @@ -521,88 +520,87 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x22, 0x7f, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, - 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, - 0x6c, 0x6f, 0x74, 0x22, 0xa8, 0x04, 0x0a, 0x13, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x46, 0x0a, 0x07, 0x70, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x12, 0x52, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, - 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xc9, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, - 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x05, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x67, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3a, 0x0a, - 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, 0x0a, 0x05, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, - 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, - 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, - 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, - 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, - 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xbd, - 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, - 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, - 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, - 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, - 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x74, 0x6f, 0x22, 0x93, 0x01, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, - 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, - 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, - 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, - 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, - 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, - 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, - 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, - 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, - 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x07, 0x49, 0x73, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x67, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, - 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, + 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xa8, 0x04, 0x0a, 0x13, 0x47, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x46, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, + 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x52, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x67, 0x6c, + 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, + 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xc9, 0x01, 0x0a, + 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x50, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x3a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, + 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, + 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, + 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, + 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, + 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, + 0x61, 0x72, 0x6b, 0x22, 0x9b, 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, + 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x39, + 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, + 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, 0x6c, + 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, + 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x07, + 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, + 0x1e, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, + 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, + 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, + 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.proto b/pkg/apis/proto/globalreduce/v1/globalreduce.proto index 69c72317..405375fd 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.proto +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.proto @@ -21,6 +21,7 @@ message Partition { google.protobuf.Timestamp start = 1; google.protobuf.Timestamp end = 2; string slot = 3; + repeated string keys = 4; } /** @@ -68,9 +69,6 @@ message GlobalReduceResponse { // Partition represents a window partition to which the result belongs. Partition partition = 2; - // the key which was used for demultiplexing the request stream. - string combinedKey = 3; - // EventTime represents the event time of the result. google.protobuf.Timestamp event_time = 4; } diff --git a/pkg/apis/proto/sessionreduce/v1/mockgen.go b/pkg/apis/proto/sessionreduce/v1/mockgen.go index 7338cedd..53aa925b 100644 --- a/pkg/apis/proto/sessionreduce/v1/mockgen.go +++ b/pkg/apis/proto/sessionreduce/v1/mockgen.go @@ -1,3 +1,3 @@ package v1 -//go:generate mockgen -destination sessionreducemock/sessionreducemock -package sessionreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1 SessionReduceClient,SessionReduce_SessionReduceFnClient +//go:generate mockgen -destination sessionreducemock/sessionreducemock.go -package sessionreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1 SessionReduceClient,SessionReduce_SessionReduceFnClient diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index 4658e168..b4dd39ef 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -86,6 +86,7 @@ type Partition struct { Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` + Keys []string `protobuf:"bytes,4,rep,name=keys,proto3" json:"keys,omitempty"` } func (x *Partition) Reset() { @@ -141,6 +142,13 @@ func (x *Partition) GetSlot() string { return "" } +func (x *Partition) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + // * // SessionReduceRequest represents a request element. type SessionReduceRequest struct { @@ -208,8 +216,6 @@ type SessionReduceResponse struct { Results []*SessionReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - // the key which was used for demultiplexing the request stream. - CombinedKey string `protobuf:"bytes,3,opt,name=combinedKey,proto3" json:"combinedKey,omitempty"` // EventTime represents the event time of the result. EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } @@ -260,13 +266,6 @@ func (x *SessionReduceResponse) GetPartition() *Partition { return nil } -func (x *SessionReduceResponse) GetCombinedKey() string { - if x != nil { - return x.CombinedKey - } - return "" -} - func (x *SessionReduceResponse) GetEventTime() *timestamppb.Timestamp { if x != nil { return x.EventTime @@ -527,91 +526,90 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7f, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, - 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x22, 0xc7, 0x04, 0x0a, 0x14, 0x53, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x48, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x93, 0x01, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, + 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xc7, 0x04, 0x0a, 0x14, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x54, + 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x36, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xe3, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0a, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, + 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, + 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, + 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, + 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, + 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0x9f, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x48, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x73, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, - 0xe3, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, - 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, - 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, 0x44, 0x10, 0x02, 0x12, 0x09, - 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, - 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, - 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, - 0x22, 0xc1, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x07, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, + 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, + 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, + 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x12, 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4b, 0x65, - 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, - 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, 0x0a, 0x0d, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x66, 0x0a, - 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, - 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, - 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, - 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, + 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, + 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto index 0682e131..ad9f60bc 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -21,6 +21,7 @@ message Partition { google.protobuf.Timestamp start = 1; google.protobuf.Timestamp end = 2; string slot = 3; + repeated string keys = 4; } /** @@ -69,10 +70,7 @@ message SessionReduceResponse { // Partition represents a window partition to which the result belongs. Partition partition = 2; - - // the key which was used for demultiplexing the request stream. - string combinedKey = 3; - + // EventTime represents the event time of the result. google.protobuf.Timestamp event_time = 4; } diff --git a/pkg/globalreducer/service.go b/pkg/globalreducer/service.go index d570686c..63b6bb9c 100644 --- a/pkg/globalreducer/service.go +++ b/pkg/globalreducer/service.go @@ -34,7 +34,7 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*globalreducepb.Rea func (fs *Service) GlobalReduceFn(stream globalreducepb.GlobalReduce_GlobalReduceFnServer) error { ctx := stream.Context() - taskManager := newReduceTaskManager() + taskManager := newReduceTaskManager(fs.globalReducer) // err group for the go routine which reads from the output channel and sends to the stream var g errgroup.Group @@ -67,7 +67,7 @@ func (fs *Service) GlobalReduceFn(stream globalreducepb.GlobalReduce_GlobalReduc case globalreducepb.GlobalReduceRequest_WindowOperation_OPEN: // create a new task and start the global reduce operation // also append the datum to the task - err := taskManager.CreateTask(ctx, d, fs.globalReducer) + err := taskManager.CreateTask(ctx, d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr @@ -77,7 +77,7 @@ func (fs *Service) GlobalReduceFn(stream globalreducepb.GlobalReduce_GlobalReduc taskManager.CloseTask(d) case globalreducepb.GlobalReduceRequest_WindowOperation_APPEND: // append the datum to the task - err := taskManager.AppendToTask(d, fs.globalReducer) + err := taskManager.AppendToTask(d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go index b0749998..3905f771 100644 --- a/pkg/globalreducer/task_manager.go +++ b/pkg/globalreducer/task_manager.go @@ -15,13 +15,12 @@ import ( // globalReduceTask represents a global reduce task for a global reduce operation. type globalReduceTask struct { - combinedKey string partition *v1.Partition - reduceStreamer GlobalReducer + globalReducer GlobalReducer inputCh chan Datum outputCh chan Messages latestWatermark *atomic.Time - Done chan struct{} + doneCh chan struct{} } // buildGlobalReduceResponse builds the global reduce response from the messages. @@ -38,10 +37,9 @@ func (rt *globalReduceTask) buildGlobalReduceResponse(messages Messages) *v1.Glo // the event time is the latest watermark // since for global window, partition start and end time will be -1 response := &v1.GlobalReduceResponse{ - Results: results, - Partition: rt.partition, - CombinedKey: rt.combinedKey, - EventTime: timestamppb.New(rt.latestWatermark.Load()), + Results: results, + Partition: rt.partition, + EventTime: timestamppb.New(rt.latestWatermark.Load()), } return response @@ -52,48 +50,49 @@ func (rt *globalReduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", rt.partition.GetStart().AsTime().UnixMilli(), rt.partition.GetEnd().AsTime().UnixMilli(), - rt.combinedKey) + strings.Join(rt.partition.GetKeys(), delimiter)) } // globalReduceTaskManager manages the reduce tasks for a global reduce operation. type globalReduceTaskManager struct { - Tasks map[string]*globalReduceTask - Output chan *v1.GlobalReduceResponse - Mutex sync.RWMutex + globalReducer GlobalReducer + tasks map[string]*globalReduceTask + outputCh chan *v1.GlobalReduceResponse + rw sync.RWMutex } -func newReduceTaskManager() *globalReduceTaskManager { +func newReduceTaskManager(globalReducer GlobalReducer) *globalReduceTaskManager { return &globalReduceTaskManager{ - Tasks: make(map[string]*globalReduceTask), - Output: make(chan *v1.GlobalReduceResponse), + tasks: make(map[string]*globalReduceTask), + outputCh: make(chan *v1.GlobalReduceResponse), + globalReducer: globalReducer, } } // CreateTask creates a new reduce task and starts the global reduce operation. -func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1.GlobalReduceRequest, globalReducer GlobalReducer) error { - rtm.Mutex.Lock() +func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1.GlobalReduceRequest) error { + rtm.rw.Lock() if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } task := &globalReduceTask{ - combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), partition: request.Operation.Partitions[0], - reduceStreamer: globalReducer, + globalReducer: rtm.globalReducer, inputCh: make(chan Datum), outputCh: make(chan Messages), - Done: make(chan struct{}), + doneCh: make(chan struct{}), latestWatermark: atomic.NewTime(time.Time{}), } key := task.uniqueKey() - rtm.Tasks[key] = task + rtm.tasks[key] = task - rtm.Mutex.Unlock() + rtm.rw.Unlock() go func() { - defer close(task.Done) + defer close(task.doneCh) var wg sync.WaitGroup wg.Add(1) go func() { @@ -107,18 +106,18 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. if !ok { break readLoop } - rtm.Output <- task.buildGlobalReduceResponse(message) + rtm.outputCh <- task.buildGlobalReduceResponse(message) } } // send a done signal - close(task.Done) + close(task.doneCh) // delete the task from the tasks list - rtm.Mutex.Lock() - delete(rtm.Tasks, key) - rtm.Mutex.Unlock() + rtm.rw.Lock() + delete(rtm.tasks, key) + rtm.rw.Unlock() }() // start the global reduce operation - globalReducer.GlobalReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) + task.globalReducer.GlobalReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) // close the output channel after the global reduce operation is completed close(task.outputCh) // wait for the output channel reader to complete @@ -137,19 +136,20 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. // AppendToTask writes the message to the reduce task. // If the reduce task is not found, it will create a new reduce task and start the reduce operation. -func (rtm *globalReduceTaskManager) AppendToTask(request *v1.GlobalReduceRequest, globalReducer GlobalReducer) error { +func (rtm *globalReduceTaskManager) AppendToTask(request *v1.GlobalReduceRequest) error { if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } - rtm.Mutex.RLock() - task, ok := rtm.Tasks[generateKey(request.Operation.Partitions[0], request.Payload.Keys)] - rtm.Mutex.RUnlock() + rtm.rw.RLock() + task, ok := rtm.tasks[generateKey(request.Operation.Partitions[0])] + rtm.rw.RUnlock() if !ok { - return rtm.CreateTask(context.Background(), request, globalReducer) + return rtm.CreateTask(context.Background(), request) } + // send the datum to the task if the payload is not nil if request.Payload != nil { task.inputCh <- buildDatum(request) task.latestWatermark.Store(request.Payload.Watermark.AsTime()) @@ -160,17 +160,18 @@ func (rtm *globalReduceTaskManager) AppendToTask(request *v1.GlobalReduceRequest // CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. func (rtm *globalReduceTaskManager) CloseTask(request *v1.GlobalReduceRequest) { - rtm.Mutex.Lock() + rtm.rw.RLock() tasksToBeClosed := make([]*globalReduceTask, 0, len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - key := generateKey(partition, request.Payload.Keys) - task, ok := rtm.Tasks[key] + key := generateKey(partition) + task, ok := rtm.tasks[key] if ok { tasksToBeClosed = append(tasksToBeClosed, task) } } - rtm.Mutex.Unlock() + rtm.rw.RUnlock() + // close the input channel for all the tasks for _, task := range tasksToBeClosed { close(task.inputCh) } @@ -178,44 +179,44 @@ func (rtm *globalReduceTaskManager) CloseTask(request *v1.GlobalReduceRequest) { // OutputChannel returns the output channel for the reduce task manager to read the results. func (rtm *globalReduceTaskManager) OutputChannel() <-chan *v1.GlobalReduceResponse { - return rtm.Output + return rtm.outputCh } // WaitAll waits for all the reduce tasks to complete. func (rtm *globalReduceTaskManager) WaitAll() { - rtm.Mutex.RLock() - tasks := make([]*globalReduceTask, 0, len(rtm.Tasks)) - for _, task := range rtm.Tasks { + rtm.rw.RLock() + tasks := make([]*globalReduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.Mutex.RUnlock() + rtm.rw.RUnlock() for _, task := range tasks { - <-task.Done + <-task.doneCh } // after all the tasks are completed, close the output channel - close(rtm.Output) + close(rtm.outputCh) } // CloseAll closes all the reduce tasks. func (rtm *globalReduceTaskManager) CloseAll() { - rtm.Mutex.Lock() - tasks := make([]*globalReduceTask, 0, len(rtm.Tasks)) - for _, task := range rtm.Tasks { + rtm.rw.Lock() + tasks := make([]*globalReduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.Mutex.Unlock() + rtm.rw.Unlock() for _, task := range tasks { close(task.inputCh) } } -func generateKey(partition *v1.Partition, keys []string) string { +func generateKey(partition *v1.Partition) string { return fmt.Sprintf("%d:%d:%s", partition.GetStart().AsTime().UnixMilli(), partition.GetEnd().AsTime().UnixMilli(), - strings.Join(keys, delimiter)) + strings.Join(partition.GetKeys(), delimiter)) } func buildDatum(request *v1.GlobalReduceRequest) Datum { diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index 33380dc8..61eca66b 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -14,11 +14,11 @@ import ( // reduceTask represents a reduce task for a reduce operation. type reduceTask struct { - combinedKey string - partition *v1.Partition - reduceHandler Reducer - inputCh chan Datum - Done chan struct{} + combinedKey string + partition *v1.Partition + reducer Reducer + inputCh chan Datum + doneCh chan struct{} } // buildReduceResponse builds the reduce response from the messages. @@ -35,7 +35,7 @@ func (rt *reduceTask) buildReduceResponse(messages Messages) *v1.ReduceResponse response := &v1.ReduceResponse{ Results: results, Partition: rt.partition, - EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + EventTime: timestamppb.New(rt.partition.End.AsTime().Add(-1 * time.Millisecond)), } return response @@ -51,21 +51,21 @@ func (rt *reduceTask) uniqueKey() string { // reduceTaskManager manages the reduce tasks for a reduce operation. type reduceTaskManager struct { - Tasks map[string]*reduceTask - Output chan *v1.ReduceResponse - Mutex sync.RWMutex + tasks map[string]*reduceTask + outputCh chan *v1.ReduceResponse + rw sync.RWMutex } func newReduceTaskManager() *reduceTaskManager { return &reduceTaskManager{ - Tasks: make(map[string]*reduceTask), - Output: make(chan *v1.ReduceResponse), + tasks: make(map[string]*reduceTask), + outputCh: make(chan *v1.ReduceResponse), } } // CreateTask creates a new reduce task and starts the reduce operation. func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest, reducer Reducer) error { - rtm.Mutex.Lock() + rtm.rw.Lock() if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } @@ -77,20 +77,20 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), partition: request.Operation.Partitions[0], inputCh: make(chan Datum), - Done: make(chan struct{}), + doneCh: make(chan struct{}), } key := task.uniqueKey() - rtm.Tasks[key] = task + rtm.tasks[key] = task - rtm.Mutex.Unlock() + rtm.rw.Unlock() go func() { msgs := reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) // write the output to the output channel, service will forward it to downstream - rtm.Output <- task.buildReduceResponse(msgs) + rtm.outputCh <- task.buildReduceResponse(msgs) // send a done signal - close(task.Done) + close(task.doneCh) }() // write the first message to the input channel @@ -101,10 +101,10 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce // AppendToTask writes the message to the reduce task. // If the task is not found, it creates a new task and starts the reduce operation. func (rtm *reduceTaskManager) AppendToTask(request *v1.ReduceRequest, reducer Reducer) error { - rtm.Mutex.RLock() + rtm.rw.RLock() gKey := generateKey(request.Operation.Partitions[0], request.Payload.Keys) - task, ok := rtm.Tasks[gKey] - rtm.Mutex.RUnlock() + task, ok := rtm.tasks[gKey] + rtm.rw.RUnlock() // if the task is not found, create a new task if !ok { @@ -117,33 +117,33 @@ func (rtm *reduceTaskManager) AppendToTask(request *v1.ReduceRequest, reducer Re // OutputChannel returns the output channel for the reduce task manager to read the results. func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { - return rtm.Output + return rtm.outputCh } // WaitAll waits for all the reduce tasks to complete. func (rtm *reduceTaskManager) WaitAll() { - rtm.Mutex.RLock() - tasks := make([]*reduceTask, 0, len(rtm.Tasks)) - for _, task := range rtm.Tasks { + rtm.rw.RLock() + tasks := make([]*reduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.Mutex.RUnlock() + rtm.rw.RUnlock() for _, task := range tasks { - <-task.Done + <-task.doneCh } // after all the tasks are completed, close the output channel - close(rtm.Output) + close(rtm.outputCh) } // CloseAll closes all the reduce tasks. func (rtm *reduceTaskManager) CloseAll() { - rtm.Mutex.Lock() - tasks := make([]*reduceTask, 0, len(rtm.Tasks)) - for _, task := range rtm.Tasks { + rtm.rw.Lock() + tasks := make([]*reduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.Mutex.Unlock() + rtm.rw.Unlock() for _, task := range tasks { close(task.inputCh) diff --git a/pkg/sessionreducer/examples/counter/Dockerfile b/pkg/sessionreducer/examples/counter/Dockerfile new file mode 100644 index 00000000..7f1e3e5c --- /dev/null +++ b/pkg/sessionreducer/examples/counter/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/counter-example /bin/counter-example +RUN chmod +x /bin/counter-example + +#################################################################################################### +# counter +#################################################################################################### +FROM scratch as counter +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/counter-example /bin/counter-example +ENTRYPOINT [ "/bin/counter-example" ] diff --git a/pkg/sessionreducer/examples/counter/Makefile b/pkg/sessionreducer/examples/counter/Makefile new file mode 100644 index 00000000..aabb2a9f --- /dev/null +++ b/pkg/sessionreducer/examples/counter/Makefile @@ -0,0 +1,10 @@ +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go + +.PHONY: image +image: build + docker build -t "quay.io/yhl25/numaflow-go/session-counter:v0.5.4" --target counter . + +clean: + -rm -rf ./dist diff --git a/pkg/sessionreducer/examples/counter/README.md b/pkg/sessionreducer/examples/counter/README.md new file mode 100644 index 00000000..5a72f269 --- /dev/null +++ b/pkg/sessionreducer/examples/counter/README.md @@ -0,0 +1,3 @@ +# Counter + +An example User Defined Function that counts the number of events. diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go new file mode 100644 index 00000000..f6e127a2 --- /dev/null +++ b/pkg/sessionreducer/examples/counter/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "fmt" + "log" + + "go.uber.org/atomic" + + "github.com/numaproj/numaflow-go/pkg/sessionreducer" +) + +type Counter struct { + count atomic.Int64 +} + +func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan sessionreducer.Datum) sessionreducer.Messages { + log.Println("SessionReduce called") + for range input { + c.count.Inc() + } + log.Println("SessionReduce done") + return sessionreducer.MessagesBuilder().Append(sessionreducer.NewMessage([]byte(fmt.Sprintf("%d", c.count.Load())))) +} + +func (c *Counter) Accumulator(ctx context.Context) []byte { + log.Println("Accumulator called") + return []byte(fmt.Sprintf("%d", c.count.Load())) +} + +func (c *Counter) MergeAccumulator(ctx context.Context, accumulator []byte) { + log.Println("MergeAccumulator called") + c.count.Add(int64(len(accumulator))) +} + +func reduceCounter() sessionreducer.SessionReducer { + return &Counter{} +} + +func main() { + sessionreducer.NewServer(reduceCounter).Start(context.Background()) +} diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go index d99f2c5d..5f5ffa89 100644 --- a/pkg/sessionreducer/service.go +++ b/pkg/sessionreducer/service.go @@ -34,7 +34,7 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*sessionreducepb.Re func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionReduceFnServer) error { ctx := stream.Context() - taskManager := newReduceTaskManager() + taskManager := newReduceTaskManager(fs.createSessionReducer) // err group for the go routine which reads from the output channel and sends to the stream var g errgroup.Group @@ -67,7 +67,7 @@ func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionR case sessionreducepb.SessionReduceRequest_WindowOperation_OPEN: // create a new task and start the session reduce operation // also append the datum to the task - err := taskManager.CreateTask(ctx, d, fs.createSessionReducer()) + err := taskManager.CreateTask(ctx, d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr @@ -77,7 +77,7 @@ func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionR taskManager.CloseTask(d) case sessionreducepb.SessionReduceRequest_WindowOperation_APPEND: // append the datum to the task - err := taskManager.AppendToTask(d, fs.createSessionReducer()) + err := taskManager.AppendToTask(d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index 9f284c48..dba71b70 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -14,11 +14,10 @@ import ( // sessionReduceTask represents a session reduce task for a session reduce operation. type sessionReduceTask struct { - combinedKey string partition *v1.Partition - reduceStreamer SessionReducer + sessionReducer SessionReducer inputCh chan Datum - Done chan struct{} + doneCh chan struct{} } // buildSessionReduceResponse builds the session reduce response from the messages. @@ -33,10 +32,9 @@ func (rt *sessionReduceTask) buildSessionReduceResponse(messages Messages) *v1.S } response := &v1.SessionReduceResponse{ - Results: results, - Partition: rt.partition, - CombinedKey: rt.combinedKey, - EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + Results: results, + Partition: rt.partition, + EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), } return response @@ -47,58 +45,66 @@ func (rt *sessionReduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", rt.partition.GetStart().AsTime().UnixMilli(), rt.partition.GetEnd().AsTime().UnixMilli(), - rt.combinedKey) + strings.Join(rt.partition.GetKeys(), delimiter)) } // sessionReduceTaskManager manages the reduce tasks for a session reduce operation. type sessionReduceTaskManager struct { - Tasks map[string]*sessionReduceTask - Output chan *v1.SessionReduceResponse - Mutex sync.RWMutex + sessionReducerFactory CreateSessionReducer + tasks map[string]*sessionReduceTask + outputCh chan *v1.SessionReduceResponse + rw sync.RWMutex } -func newReduceTaskManager() *sessionReduceTaskManager { +func newReduceTaskManager(sessionReducerFactory CreateSessionReducer) *sessionReduceTaskManager { return &sessionReduceTaskManager{ - Tasks: make(map[string]*sessionReduceTask), - Output: make(chan *v1.SessionReduceResponse), + sessionReducerFactory: sessionReducerFactory, + tasks: make(map[string]*sessionReduceTask), + outputCh: make(chan *v1.SessionReduceResponse), } } // CreateTask creates a new reduce task and starts the session reduce operation. -func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest, sessionReducer SessionReducer) error { - rtm.Mutex.Lock() +func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest) error { + println("create task was invoked with partitions - ", len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + } + + rtm.rw.Lock() + + // for create operation, there should be exactly one partition if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } task := &sessionReduceTask{ - combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), partition: request.Operation.Partitions[0], - reduceStreamer: sessionReducer, + sessionReducer: rtm.sessionReducerFactory(), inputCh: make(chan Datum), - Done: make(chan struct{}), + doneCh: make(chan struct{}), } key := task.uniqueKey() - rtm.Tasks[key] = task + rtm.tasks[key] = task - rtm.Mutex.Unlock() + rtm.rw.Unlock() go func() { - msgs := sessionReducer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh) + msgs := task.sessionReducer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh) // write the output to the output channel, service will forward it to downstream - rtm.Output <- task.buildSessionReduceResponse(msgs) + rtm.outputCh <- task.buildSessionReduceResponse(msgs) // send a done signal - close(task.Done) + close(task.doneCh) // delete the task from the tasks list - rtm.Mutex.Lock() - delete(rtm.Tasks, key) - rtm.Mutex.Unlock() + rtm.rw.Lock() + delete(rtm.tasks, key) + rtm.rw.Unlock() }() // send the datum to the task if the payload is not nil if request.Payload != nil { - task.inputCh <- buildDatum(request) + task.inputCh <- buildDatum(request.Payload) } return nil @@ -106,35 +112,49 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 // AppendToTask writes the message to the reduce task. // If the reduce task is not found, it will create a new reduce task and start the reduce operation. -func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceRequest, sessionReducer SessionReducer) error { +func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) error { + println("append task was invoked with partitions - ", len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + } + + // for append operation, there should be exactly one partition if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } - rtm.Mutex.RLock() - task, ok := rtm.Tasks[generateKey(request.Operation.Partitions[0], request.Payload.Keys)] - rtm.Mutex.RUnlock() + rtm.rw.RLock() + task, ok := rtm.tasks[generateKey(request.Operation.Partitions[0])] + rtm.rw.RUnlock() + // if the task is not found, create a new task and start the reduce operation if !ok { - return rtm.CreateTask(context.Background(), request, sessionReducer) + return rtm.CreateTask(context.Background(), request) + } + // send the datum to the task if the payload is not nil + if request.Payload != nil { + task.inputCh <- buildDatum(request.Payload) } - - task.inputCh <- buildDatum(request) return nil } // CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { - rtm.Mutex.Lock() + println("close task was invoked with partitions - ", len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + } + + rtm.rw.RLock() tasksToBeClosed := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - key := generateKey(partition, request.Payload.Keys) - task, ok := rtm.Tasks[key] + key := generateKey(partition) + task, ok := rtm.tasks[key] if ok { tasksToBeClosed = append(tasksToBeClosed, task) } } - rtm.Mutex.Unlock() + rtm.rw.RUnlock() for _, task := range tasksToBeClosed { close(task.inputCh) @@ -144,37 +164,66 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) // MergeTasks merges the session reduce tasks. It will invoke close on all the tasks except the main task. The main task will // merge the aggregators from the other tasks. The main task will be first task in the list of tasks. func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { - rtm.Mutex.Lock() - tasks := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) + println("merge task was invoked with partitions - ", len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - key := generateKey(partition, request.Payload.Keys) - task, ok := rtm.Tasks[key] + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + } + + rtm.rw.Lock() + mergedPartition := request.Operation.Partitions[0] + + tasks := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) + task, ok := rtm.tasks[generateKey(mergedPartition)] + if !ok { + return fmt.Errorf("task not found") + } + + // send the datum to first task if the payload is not nil + if request.Payload != nil { + task.inputCh <- buildDatum(request.Payload) + } + + // merge the aggregators from the other tasks + for _, partition := range request.Operation.Partitions[1:] { + task, ok = rtm.tasks[generateKey(partition)] if !ok { - rtm.Mutex.Unlock() + rtm.rw.Unlock() return fmt.Errorf("task not found") } tasks = append(tasks, task) + + // mergedPartition will be the smallest window which contains all the windows + if partition.GetStart().AsTime().Before(mergedPartition.GetStart().AsTime()) { + mergedPartition.Start = partition.Start + } + + if partition.GetEnd().AsTime().After(mergedPartition.GetEnd().AsTime()) { + mergedPartition.End = partition.End + } } - rtm.Mutex.Unlock() - if len(tasks) == 0 { - return nil + // create a new task with the merged partition + mergedTask := &sessionReduceTask{ + partition: mergedPartition, + sessionReducer: rtm.sessionReducerFactory(), + inputCh: make(chan Datum), + doneCh: make(chan struct{}), } - mainTask := tasks[0] - aggregators := make([][]byte, 0, len(tasks)-1) + // add merged task to the tasks map + rtm.tasks[mergedTask.uniqueKey()] = mergedTask + rtm.rw.Unlock() - for _, task := range tasks[1:] { + accumulators := make([][]byte, 0, len(tasks)) + // close all the tasks and collect the accumulators + for _, task = range tasks { close(task.inputCh) - aggregators = append(aggregators, task.reduceStreamer.Accumulator(ctx)) + accumulators = append(accumulators, task.sessionReducer.Accumulator(ctx)) } - for _, aggregator := range aggregators { - mainTask.reduceStreamer.MergeAccumulator(ctx, aggregator) - } - - if request.Payload != nil { - mainTask.inputCh <- buildDatum(request) + // merge the accumulators using the merged task + for _, aggregator := range accumulators { + mergedTask.sessionReducer.MergeAccumulator(ctx, aggregator) } return nil @@ -185,66 +234,81 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 // expects request.Operation.Partitions to have exactly two partitions. The first partition is the old partition and the second // partition is the new partition. func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { + println("expand task was invoked with partitions - ", len(request.Operation.Partitions)) + for _, partition := range request.Operation.Partitions { + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + } + + // for expand operation, there should be exactly two partitions if len(request.Operation.Partitions) != 2 { return fmt.Errorf("expected exactly two partitions") } - rtm.Mutex.Lock() - key := generateKey(request.Operation.Partitions[0], request.Payload.Keys) - task, ok := rtm.Tasks[key] - task.partition = request.Operation.Partitions[1] - delete(rtm.Tasks, key) - rtm.Tasks[task.uniqueKey()] = task - rtm.Mutex.Unlock() + rtm.rw.Lock() + key := generateKey(request.Operation.Partitions[0]) + task, ok := rtm.tasks[key] if !ok { return fmt.Errorf("task not found") } + // assign the new partition to the task + task.partition = request.Operation.Partitions[1] + + // delete the old entry from the tasks map and add the new entry + delete(rtm.tasks, key) + rtm.tasks[task.uniqueKey()] = task + rtm.rw.Unlock() + + // send the datum to the task if the payload is not nil + if request.Payload != nil { + task.inputCh <- buildDatum(request.GetPayload()) + } + return nil } // OutputChannel returns the output channel for the reduce task manager to read the results. func (rtm *sessionReduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { - return rtm.Output + return rtm.outputCh } // WaitAll waits for all the pending reduce tasks to complete. func (rtm *sessionReduceTaskManager) WaitAll() { - rtm.Mutex.RLock() - tasks := make([]*sessionReduceTask, 0, len(rtm.Tasks)) - for _, task := range rtm.Tasks { + rtm.rw.RLock() + tasks := make([]*sessionReduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.Mutex.RUnlock() + rtm.rw.RUnlock() for _, task := range tasks { - <-task.Done + <-task.doneCh } // after all the tasks are completed, close the output channel - close(rtm.Output) + close(rtm.outputCh) } // CloseAll closes all the reduce tasks. func (rtm *sessionReduceTaskManager) CloseAll() { - rtm.Mutex.Lock() - tasks := make([]*sessionReduceTask, 0, len(rtm.Tasks)) - for _, task := range rtm.Tasks { + rtm.rw.Lock() + tasks := make([]*sessionReduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.Mutex.Unlock() + rtm.rw.Unlock() for _, task := range tasks { close(task.inputCh) } } -func generateKey(partition *v1.Partition, keys []string) string { +func generateKey(partition *v1.Partition) string { return fmt.Sprintf("%d:%d:%s", partition.GetStart().AsTime().UnixMilli(), partition.GetEnd().AsTime().UnixMilli(), - strings.Join(keys, delimiter)) + strings.Join(partition.GetKeys(), delimiter)) } -func buildDatum(request *v1.SessionReduceRequest) Datum { - return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +func buildDatum(payload *v1.SessionReduceRequest_Payload) Datum { + return NewHandlerDatum(payload.GetValue(), payload.EventTime.AsTime(), payload.Watermark.AsTime()) } From bf364c5635f166db2989698eee9a327811edc46b Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 21 Nov 2023 13:33:40 +0530 Subject: [PATCH 08/27] working version Signed-off-by: Yashash H L --- pkg/sessionreducer/examples/counter/main.go | 21 +++-- pkg/sessionreducer/task_manager.go | 96 ++++++++++++--------- 2 files changed, 72 insertions(+), 45 deletions(-) diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go index f6e127a2..32106f03 100644 --- a/pkg/sessionreducer/examples/counter/main.go +++ b/pkg/sessionreducer/examples/counter/main.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "strconv" "go.uber.org/atomic" @@ -11,30 +12,40 @@ import ( ) type Counter struct { - count atomic.Int64 + count *atomic.Int32 } func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan sessionreducer.Datum) sessionreducer.Messages { log.Println("SessionReduce called") for range input { + println("incrementing the counter") c.count.Inc() } log.Println("SessionReduce done") - return sessionreducer.MessagesBuilder().Append(sessionreducer.NewMessage([]byte(fmt.Sprintf("%d", c.count.Load())))) + return sessionreducer.MessagesBuilder().Append(sessionreducer.NewMessage([]byte(fmt.Sprintf("%d", c.count.Load()))).WithKeys(keys)) } func (c *Counter) Accumulator(ctx context.Context) []byte { log.Println("Accumulator called") - return []byte(fmt.Sprintf("%d", c.count.Load())) + println("returning the accumulator value - ", c.count.Load()) + return []byte(strconv.Itoa(int(c.count.Load()))) } func (c *Counter) MergeAccumulator(ctx context.Context, accumulator []byte) { log.Println("MergeAccumulator called") - c.count.Add(int64(len(accumulator))) + val, err := strconv.Atoi(string(accumulator)) + println("merging the accumulator with value - ", val) + if err != nil { + log.Println("unable to convert the accumulator value to int: ", err) + return + } + c.count.Add(int32(val)) } func reduceCounter() sessionreducer.SessionReducer { - return &Counter{} + return &Counter{ + count: atomic.NewInt32(0), + } } func main() { diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index dba71b70..58cd931c 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -7,17 +7,19 @@ import ( "sync" "time" + "go.uber.org/atomic" "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" ) -// sessionReduceTask represents a session reduce task for a session reduce operation. +// sessionReduceTask represents a task for a performing session reduce operation. type sessionReduceTask struct { partition *v1.Partition sessionReducer SessionReducer inputCh chan Datum doneCh chan struct{} + merged *atomic.Bool } // buildSessionReduceResponse builds the session reduce response from the messages. @@ -34,6 +36,7 @@ func (rt *sessionReduceTask) buildSessionReduceResponse(messages Messages) *v1.S response := &v1.SessionReduceResponse{ Results: results, Partition: rt.partition, + // event time is the end time of the window - 1 millisecond EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), } @@ -68,14 +71,14 @@ func newReduceTaskManager(sessionReducerFactory CreateSessionReducer) *sessionRe func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest) error { println("create task was invoked with partitions - ", len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) } rtm.rw.Lock() // for create operation, there should be exactly one partition if len(request.Operation.Partitions) != 1 { - return fmt.Errorf("invalid number of partitions") + return fmt.Errorf("create operation error: invalid number of partitions in the request - %d", len(request.Operation.Partitions)) } task := &sessionReduceTask{ @@ -83,17 +86,22 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 sessionReducer: rtm.sessionReducerFactory(), inputCh: make(chan Datum), doneCh: make(chan struct{}), + merged: atomic.NewBool(false), } + // add the task to the tasks list key := task.uniqueKey() rtm.tasks[key] = task rtm.rw.Unlock() go func() { - msgs := task.sessionReducer.SessionReduce(ctx, request.GetPayload().GetKeys(), task.inputCh) + msgs := task.sessionReducer.SessionReduce(ctx, task.partition.GetKeys(), task.inputCh) // write the output to the output channel, service will forward it to downstream - rtm.outputCh <- task.buildSessionReduceResponse(msgs) + // if the task is merged to another task, we don't need to send the response + if !task.merged.Load() { + rtm.outputCh <- task.buildSessionReduceResponse(msgs) + } // send a done signal close(task.doneCh) // delete the task from the tasks list @@ -115,12 +123,12 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) error { println("append task was invoked with partitions - ", len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) } // for append operation, there should be exactly one partition if len(request.Operation.Partitions) != 1 { - return fmt.Errorf("invalid number of partitions") + return fmt.Errorf("append operation error: invalid number of partitions in the request - %d", len(request.Operation.Partitions)) } rtm.rw.RLock() @@ -131,6 +139,7 @@ func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceReque if !ok { return rtm.CreateTask(context.Background(), request) } + // send the datum to the task if the payload is not nil if request.Payload != nil { task.inputCh <- buildDatum(request.Payload) @@ -138,11 +147,11 @@ func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceReque return nil } -// CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. +// CloseTask closes the input channel of the reduce tasks. func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { println("close task was invoked with partitions - ", len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) } rtm.rw.RLock() @@ -161,35 +170,29 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) } } -// MergeTasks merges the session reduce tasks. It will invoke close on all the tasks except the main task. The main task will -// merge the aggregators from the other tasks. The main task will be first task in the list of tasks. +// MergeTasks merges the session reduce tasks. It will create a new task with the merged window and +// merges the accumulators from the other tasks. func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { println("merge task was invoked with partitions - ", len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) } rtm.rw.Lock() mergedPartition := request.Operation.Partitions[0] tasks := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) - task, ok := rtm.tasks[generateKey(mergedPartition)] - if !ok { - return fmt.Errorf("task not found") - } - - // send the datum to first task if the payload is not nil - if request.Payload != nil { - task.inputCh <- buildDatum(request.Payload) - } // merge the aggregators from the other tasks - for _, partition := range request.Operation.Partitions[1:] { - task, ok = rtm.tasks[generateKey(partition)] + for _, partition := range request.Operation.Partitions { + key := generateKey(partition) + task, ok := rtm.tasks[key] if !ok { rtm.rw.Unlock() - return fmt.Errorf("task not found") + println("task not found for key inside merge - ", key) + return fmt.Errorf("merge operation error: task not found for %s", key) } + task.merged.Store(true) tasks = append(tasks, task) // mergedPartition will be the smallest window which contains all the windows @@ -202,25 +205,33 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 } } - // create a new task with the merged partition - mergedTask := &sessionReduceTask{ - partition: mergedPartition, - sessionReducer: rtm.sessionReducerFactory(), - inputCh: make(chan Datum), - doneCh: make(chan struct{}), - } - - // add merged task to the tasks map - rtm.tasks[mergedTask.uniqueKey()] = mergedTask rtm.rw.Unlock() accumulators := make([][]byte, 0, len(tasks)) // close all the tasks and collect the accumulators - for _, task = range tasks { + for _, task := range tasks { close(task.inputCh) + // wait for the task to complete + <-task.doneCh accumulators = append(accumulators, task.sessionReducer.Accumulator(ctx)) } + // create a new task with the merged partition + err := rtm.CreateTask(ctx, &v1.SessionReduceRequest{ + Payload: nil, + Operation: &v1.SessionReduceRequest_WindowOperation{ + Event: v1.SessionReduceRequest_WindowOperation_OPEN, + Partitions: []*v1.Partition{mergedPartition}, + }, + }) + if err != nil { + return err + } + + mergedTask, ok := rtm.tasks[generateKey(mergedPartition)] + if !ok { + return fmt.Errorf("merge operation error: merged task not found for key %s", mergedPartition.String()) + } // merge the accumulators using the merged task for _, aggregator := range accumulators { mergedTask.sessionReducer.MergeAccumulator(ctx, aggregator) @@ -229,26 +240,31 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 return nil } -// ExpandTask expands the reduce task. It will expand the window for the reduce task. -// deletes the old task and creates a new task with the new window. +// ExpandTask expands the reduce task. It will update the partition of the reduce task // expects request.Operation.Partitions to have exactly two partitions. The first partition is the old partition and the second // partition is the new partition. func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { println("expand task was invoked with partitions - ", len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli()) + println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) } // for expand operation, there should be exactly two partitions if len(request.Operation.Partitions) != 2 { - return fmt.Errorf("expected exactly two partitions") + return fmt.Errorf("expand operation error: expected exactly two partitions") } rtm.rw.Lock() key := generateKey(request.Operation.Partitions[0]) task, ok := rtm.tasks[key] if !ok { - return fmt.Errorf("task not found") + rtm.rw.Unlock() + println("task not found for key inside expand - ", key) + for k := range rtm.tasks { + println("task key - ", k) + } + panic("task not found") + //return fmt.Errorf("expand operation error: task not found for key - %s", key) } // assign the new partition to the task From 52ae842a332e605b97c609040674a9f92ae66aa0 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 21 Nov 2023 14:59:06 +0530 Subject: [PATCH 09/27] minor cleanup Signed-off-by: Yashash H L --- pkg/globalreducer/service.go | 2 +- pkg/globalreducer/task_manager.go | 4 +-- pkg/reducer/server.go | 2 +- pkg/reducer/service.go | 8 ++--- pkg/reducer/service_test.go | 2 +- pkg/reducer/task_manager.go | 12 ++++--- pkg/sessionreducer/examples/counter/main.go | 11 +++--- pkg/sessionreducer/service.go | 2 +- pkg/sessionreducer/task_manager.go | 37 +++------------------ 9 files changed, 25 insertions(+), 55 deletions(-) diff --git a/pkg/globalreducer/service.go b/pkg/globalreducer/service.go index 63b6bb9c..d7482b3a 100644 --- a/pkg/globalreducer/service.go +++ b/pkg/globalreducer/service.go @@ -77,7 +77,7 @@ func (fs *Service) GlobalReduceFn(stream globalreducepb.GlobalReduce_GlobalReduc taskManager.CloseTask(d) case globalreducepb.GlobalReduceRequest_WindowOperation_APPEND: // append the datum to the task - err := taskManager.AppendToTask(d) + err := taskManager.AppendToTask(ctx, d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go index 3905f771..28605ea1 100644 --- a/pkg/globalreducer/task_manager.go +++ b/pkg/globalreducer/task_manager.go @@ -136,7 +136,7 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. // AppendToTask writes the message to the reduce task. // If the reduce task is not found, it will create a new reduce task and start the reduce operation. -func (rtm *globalReduceTaskManager) AppendToTask(request *v1.GlobalReduceRequest) error { +func (rtm *globalReduceTaskManager) AppendToTask(ctx context.Context, request *v1.GlobalReduceRequest) error { if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") } @@ -146,7 +146,7 @@ func (rtm *globalReduceTaskManager) AppendToTask(request *v1.GlobalReduceRequest rtm.rw.RUnlock() if !ok { - return rtm.CreateTask(context.Background(), request) + return rtm.CreateTask(ctx, request) } // send the datum to the task if the payload is not nil diff --git a/pkg/reducer/server.go b/pkg/reducer/server.go index 81421481..0676ba08 100644 --- a/pkg/reducer/server.go +++ b/pkg/reducer/server.go @@ -25,7 +25,7 @@ func NewServer(r Reducer, inputOptions ...Option) numaflow.Server { } s := new(server) s.svc = new(Service) - s.svc.Reducer = r + s.svc.reducerHandle = r s.opts = opts return s } diff --git a/pkg/reducer/service.go b/pkg/reducer/service.go index e2b22abe..22ed6928 100644 --- a/pkg/reducer/service.go +++ b/pkg/reducer/service.go @@ -24,7 +24,7 @@ const ( // Service implements the proto gen server interface and contains the reduce operation handler. type Service struct { reducepb.UnimplementedReduceServer - Reducer Reducer + reducerHandle Reducer } // IsReady returns true to indicate the gRPC connection is ready. @@ -40,7 +40,7 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { g errgroup.Group ) - taskManager := newReduceTaskManager() + taskManager := newReduceTaskManager(fs.reducerHandle) // err group for the go routine which reads from the output channel and sends to the stream g.Go(func() error { @@ -74,14 +74,14 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { switch d.Operation.Event { case reducepb.ReduceRequest_WindowOperation_OPEN: // create a new reduce task and start the reduce operation - err = taskManager.CreateTask(ctx, d, fs.Reducer) + err = taskManager.CreateTask(ctx, d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr } case reducepb.ReduceRequest_WindowOperation_APPEND: // append the datum to the reduce task - err = taskManager.AppendToTask(d, fs.Reducer) + err = taskManager.AppendToTask(ctx, d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index 336025aa..4fa05148 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -453,7 +453,7 @@ func TestService_ReduceFn(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := &Service{ - Reducer: tt.handler, + reducerHandle: tt.handler, } // here's a trick for testing: // because we are not using gRPC, we directly set a new incoming ctx diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index 61eca66b..eacc3f19 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -51,20 +51,22 @@ func (rt *reduceTask) uniqueKey() string { // reduceTaskManager manages the reduce tasks for a reduce operation. type reduceTaskManager struct { + reducer Reducer tasks map[string]*reduceTask outputCh chan *v1.ReduceResponse rw sync.RWMutex } -func newReduceTaskManager() *reduceTaskManager { +func newReduceTaskManager(reducer Reducer) *reduceTaskManager { return &reduceTaskManager{ + reducer: reducer, tasks: make(map[string]*reduceTask), outputCh: make(chan *v1.ReduceResponse), } } // CreateTask creates a new reduce task and starts the reduce operation. -func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest, reducer Reducer) error { +func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest) error { rtm.rw.Lock() if len(request.Operation.Partitions) != 1 { return fmt.Errorf("invalid number of partitions") @@ -86,7 +88,7 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce rtm.rw.Unlock() go func() { - msgs := reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) + msgs := rtm.reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) // write the output to the output channel, service will forward it to downstream rtm.outputCh <- task.buildReduceResponse(msgs) // send a done signal @@ -100,7 +102,7 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce // AppendToTask writes the message to the reduce task. // If the task is not found, it creates a new task and starts the reduce operation. -func (rtm *reduceTaskManager) AppendToTask(request *v1.ReduceRequest, reducer Reducer) error { +func (rtm *reduceTaskManager) AppendToTask(ctx context.Context, request *v1.ReduceRequest) error { rtm.rw.RLock() gKey := generateKey(request.Operation.Partitions[0], request.Payload.Keys) task, ok := rtm.tasks[gKey] @@ -108,7 +110,7 @@ func (rtm *reduceTaskManager) AppendToTask(request *v1.ReduceRequest, reducer Re // if the task is not found, create a new task if !ok { - return rtm.CreateTask(context.Background(), request, reducer) + return rtm.CreateTask(ctx, request) } task.inputCh <- buildDatum(request) diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go index 32106f03..ca07210e 100644 --- a/pkg/sessionreducer/examples/counter/main.go +++ b/pkg/sessionreducer/examples/counter/main.go @@ -16,9 +16,8 @@ type Counter struct { } func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan sessionreducer.Datum) sessionreducer.Messages { - log.Println("SessionReduce called") + log.Println("SessionReduce invoked") for range input { - println("incrementing the counter") c.count.Inc() } log.Println("SessionReduce done") @@ -26,17 +25,15 @@ func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan } func (c *Counter) Accumulator(ctx context.Context) []byte { - log.Println("Accumulator called") - println("returning the accumulator value - ", c.count.Load()) + log.Println("Accumulator invoked") return []byte(strconv.Itoa(int(c.count.Load()))) } func (c *Counter) MergeAccumulator(ctx context.Context, accumulator []byte) { - log.Println("MergeAccumulator called") + log.Println("MergeAccumulator invoked") val, err := strconv.Atoi(string(accumulator)) - println("merging the accumulator with value - ", val) if err != nil { - log.Println("unable to convert the accumulator value to int: ", err) + log.Println("unable to convert the accumulator value to int: ", err.Error()) return } c.count.Add(int32(val)) diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go index 5f5ffa89..d7680a32 100644 --- a/pkg/sessionreducer/service.go +++ b/pkg/sessionreducer/service.go @@ -77,7 +77,7 @@ func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionR taskManager.CloseTask(d) case sessionreducepb.SessionReduceRequest_WindowOperation_APPEND: // append the datum to the task - err := taskManager.AppendToTask(d) + err := taskManager.AppendToTask(ctx, d) if err != nil { statusErr := status.Errorf(codes.Internal, err.Error()) return statusErr diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index 58cd931c..fd10f930 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -67,13 +67,8 @@ func newReduceTaskManager(sessionReducerFactory CreateSessionReducer) *sessionRe } } -// CreateTask creates a new reduce task and starts the session reduce operation. +// CreateTask creates a new task and starts the session reduce operation. func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest) error { - println("create task was invoked with partitions - ", len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) - } - rtm.rw.Lock() // for create operation, there should be exactly one partition @@ -120,11 +115,7 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 // AppendToTask writes the message to the reduce task. // If the reduce task is not found, it will create a new reduce task and start the reduce operation. -func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceRequest) error { - println("append task was invoked with partitions - ", len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) - } +func (rtm *sessionReduceTaskManager) AppendToTask(ctx context.Context, request *v1.SessionReduceRequest) error { // for append operation, there should be exactly one partition if len(request.Operation.Partitions) != 1 { @@ -137,7 +128,7 @@ func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceReque // if the task is not found, create a new task and start the reduce operation if !ok { - return rtm.CreateTask(context.Background(), request) + return rtm.CreateTask(ctx, request) } // send the datum to the task if the payload is not nil @@ -149,11 +140,6 @@ func (rtm *sessionReduceTaskManager) AppendToTask(request *v1.SessionReduceReque // CloseTask closes the input channel of the reduce tasks. func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { - println("close task was invoked with partitions - ", len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) - } - rtm.rw.RLock() tasksToBeClosed := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) for _, partition := range request.Operation.Partitions { @@ -173,11 +159,6 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) // MergeTasks merges the session reduce tasks. It will create a new task with the merged window and // merges the accumulators from the other tasks. func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { - println("merge task was invoked with partitions - ", len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) - } - rtm.rw.Lock() mergedPartition := request.Operation.Partitions[0] @@ -189,7 +170,6 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 task, ok := rtm.tasks[key] if !ok { rtm.rw.Unlock() - println("task not found for key inside merge - ", key) return fmt.Errorf("merge operation error: task not found for %s", key) } task.merged.Store(true) @@ -244,10 +224,6 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 // expects request.Operation.Partitions to have exactly two partitions. The first partition is the old partition and the second // partition is the new partition. func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { - println("expand task was invoked with partitions - ", len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - println("partition -", partition.GetStart().AsTime().UnixMilli(), " ", partition.GetEnd().AsTime().UnixMilli(), " ", strings.Join(partition.GetKeys(), ":")) - } // for expand operation, there should be exactly two partitions if len(request.Operation.Partitions) != 2 { @@ -259,12 +235,7 @@ func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest task, ok := rtm.tasks[key] if !ok { rtm.rw.Unlock() - println("task not found for key inside expand - ", key) - for k := range rtm.tasks { - println("task key - ", k) - } - panic("task not found") - //return fmt.Errorf("expand operation error: task not found for key - %s", key) + return fmt.Errorf("expand operation error: task not found for key - %s", key) } // assign the new partition to the task From fe61737eb7dd774e63cf8d413323f72f04a39a9c Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Thu, 23 Nov 2023 11:18:22 +0530 Subject: [PATCH 10/27] reduce streaming Signed-off-by: Yashash H L --- pkg/apis/proto/reduce/v1/reduce.pb.go | 107 +++++++------- pkg/apis/proto/reduce/v1/reduce.proto | 7 +- pkg/reducer/interface.go | 8 +- pkg/reducer/server_test.go | 4 +- pkg/reducer/service.go | 6 +- pkg/reducer/service_test.go | 149 +++++++++++--------- pkg/reducer/task_manager.go | 69 ++++++--- pkg/sessionreducer/examples/counter/main.go | 4 +- 8 files changed, 205 insertions(+), 149 deletions(-) diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index 4d597c4c..de5d78da 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -199,11 +199,11 @@ type ReduceResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Results []*ReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Result *ReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - // EventTime represents the event time of the result. - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + // EOF represents the end of the stream. + EOF bool `protobuf:"varint,4,opt,name=EOF,proto3" json:"EOF,omitempty"` } func (x *ReduceResponse) Reset() { @@ -238,9 +238,9 @@ func (*ReduceResponse) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{2} } -func (x *ReduceResponse) GetResults() []*ReduceResponse_Result { +func (x *ReduceResponse) GetResult() *ReduceResponse_Result { if x != nil { - return x.Results + return x.Result } return nil } @@ -252,11 +252,11 @@ func (x *ReduceResponse) GetPartition() *Partition { return nil } -func (x *ReduceResponse) GetEventTime() *timestamppb.Timestamp { +func (x *ReduceResponse) GetEOF() bool { if x != nil { - return x.EventTime + return x.EOF } - return nil + return false } // * @@ -443,9 +443,10 @@ type ReduceResponse_Result struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } func (x *ReduceResponse_Result) Reset() { @@ -501,6 +502,13 @@ func (x *ReduceResponse_Result) GetTags() []string { return nil } +func (x *ReduceResponse_Result) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + var File_pkg_apis_proto_reduce_v1_reduce_proto protoreflect.FileDescriptor var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ @@ -550,39 +558,40 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x22, 0x83, 0x02, 0x0a, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x22, 0x94, 0x02, 0x0a, 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3a, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x09, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, - 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, - 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, - 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, - 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, - 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, - 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x38, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x32, 0x0a, 0x09, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, + 0x03, 0x45, 0x4f, 0x46, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, + 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, + 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, + 0x6e, 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, + 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, + 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, + 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -616,13 +625,13 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs = []int32{ 5, // 1: reduce.v1.ReduceRequest.operation:type_name -> reduce.v1.ReduceRequest.WindowOperation 8, // 2: reduce.v1.Partition.start:type_name -> google.protobuf.Timestamp 8, // 3: reduce.v1.Partition.end:type_name -> google.protobuf.Timestamp - 7, // 4: reduce.v1.ReduceResponse.results:type_name -> reduce.v1.ReduceResponse.Result + 7, // 4: reduce.v1.ReduceResponse.result:type_name -> reduce.v1.ReduceResponse.Result 2, // 5: reduce.v1.ReduceResponse.partition:type_name -> reduce.v1.Partition - 8, // 6: reduce.v1.ReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Event - 2, // 8: reduce.v1.ReduceRequest.WindowOperation.partitions:type_name -> reduce.v1.Partition - 8, // 9: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp - 8, // 10: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 0, // 6: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Event + 2, // 7: reduce.v1.ReduceRequest.WindowOperation.partitions:type_name -> reduce.v1.Partition + 8, // 8: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 9: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 8, // 10: reduce.v1.ReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp 1, // 11: reduce.v1.Reduce.ReduceFn:input_type -> reduce.v1.ReduceRequest 9, // 12: reduce.v1.Reduce.IsReady:input_type -> google.protobuf.Empty 3, // 13: reduce.v1.Reduce.ReduceFn:output_type -> reduce.v1.ReduceResponse diff --git a/pkg/apis/proto/reduce/v1/reduce.proto b/pkg/apis/proto/reduce/v1/reduce.proto index 91b55b4e..4015ed2a 100644 --- a/pkg/apis/proto/reduce/v1/reduce.proto +++ b/pkg/apis/proto/reduce/v1/reduce.proto @@ -61,15 +61,16 @@ message ReduceResponse { repeated string keys = 1; bytes value = 2; repeated string tags = 3; + google.protobuf.Timestamp event_time = 4; } - repeated Result results = 1; + Result result = 1; // Partition represents a window partition to which the result belongs. Partition partition = 2; - // EventTime represents the event time of the result. - google.protobuf.Timestamp event_time = 3; + // EOF represents the end of the stream. + bool EOF = 4; } /** diff --git a/pkg/reducer/interface.go b/pkg/reducer/interface.go index 35b37bbb..62adc003 100644 --- a/pkg/reducer/interface.go +++ b/pkg/reducer/interface.go @@ -25,13 +25,13 @@ type IntervalWindow interface { // Reducer is the interface of reduce function implementation. type Reducer interface { - Reduce(ctx context.Context, keys []string, reduceCh <-chan Datum, md Metadata) Messages + Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) } // ReducerFunc is a utility type used to convert a Reduce function to a Reducer. -type ReducerFunc func(ctx context.Context, keys []string, reduceCh <-chan Datum, md Metadata) Messages +type ReducerFunc func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) // Reduce implements the function of reduce function. -func (rf ReducerFunc) Reduce(ctx context.Context, keys []string, reduceCh <-chan Datum, md Metadata) Messages { - return rf(ctx, keys, reduceCh, md) +func (rf ReducerFunc) Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { + rf(ctx, keys, inputCh, outputCh, md) } diff --git a/pkg/reducer/server_test.go b/pkg/reducer/server_test.go index 511bebb8..1a043693 100644 --- a/pkg/reducer/server_test.go +++ b/pkg/reducer/server_test.go @@ -21,13 +21,13 @@ func TestReduceServer_Start(t *testing.T) { _ = os.RemoveAll(serverInfoFile.Name()) }() - var reduceHandler = ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { + var reduceHandler = ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"})) + och <- NewMessage([]byte(strconv.Itoa(sum))) }) // note: using actual uds connection ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) diff --git a/pkg/reducer/service.go b/pkg/reducer/service.go index 22ed6928..f4dc53cc 100644 --- a/pkg/reducer/service.go +++ b/pkg/reducer/service.go @@ -45,9 +45,9 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { // err group for the go routine which reads from the output channel and sends to the stream g.Go(func() error { for output := range taskManager.OutputChannel() { - err := stream.Send(output) - if err != nil { - return err + sendErr := stream.Send(output) + if sendErr != nil { + return sendErr } } return nil diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index 4fa05148..404878c4 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -57,18 +57,18 @@ func TestService_ReduceFn(t *testing.T) { name string handler Reducer input []*reducepb.ReduceRequest - expected *reducepb.ReduceResponse + expected []*reducepb.ReduceResponse expectedErr bool }{ { name: "reduce_fn_forward_msg_same_keys", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"})) + och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) }), input: []*reducepb.ReduceRequest{ { @@ -126,31 +126,32 @@ func TestService_ReduceFn(t *testing.T) { }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { - Keys: []string{"client_test"}, - Value: []byte(strconv.Itoa(60)), + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), + EventTime: timestamppb.New(time.UnixMilli(119999)), }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, - Partition: &reducepb.Partition{ - Start: timestamppb.New(time.UnixMilli(60000)), - End: timestamppb.New(time.UnixMilli(120000)), - Slot: "slot-0", - }, - EventTime: timestamppb.New(time.UnixMilli(119999)), }, expectedErr: false, }, { name: "reduce_fn_forward_msg_multiple_keys", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"})) + och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) }), input: []*reducepb.ReduceRequest{ { @@ -262,39 +263,58 @@ func TestService_ReduceFn(t *testing.T) { }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { - Keys: []string{"client1_test"}, - Value: []byte(strconv.Itoa(20)), + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.UnixMilli(119999)), }, - { - Keys: []string{"client2_test"}, - Value: []byte(strconv.Itoa(40)), + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", }, - { - Keys: []string{"client3_test"}, - Value: []byte(strconv.Itoa(60)), + EOF: false, + }, + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), + EventTime: timestamppb.New(time.UnixMilli(119999)), }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, - Partition: &reducepb.Partition{ - Start: timestamppb.New(time.UnixMilli(60000)), - End: timestamppb.New(time.UnixMilli(120000)), - Slot: "slot-0", + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client3_test"}, + Value: []byte(strconv.Itoa(60)), + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, - EventTime: timestamppb.New(time.UnixMilli(119999)), }, expectedErr: false, }, { name: "reduce_fn_forward_msg_forward_to_all", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum)))) + och <- NewMessage([]byte(strconv.Itoa(sum))) }), input: []*reducepb.ReduceRequest{ { @@ -352,30 +372,30 @@ func TestService_ReduceFn(t *testing.T) { }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { - Value: []byte(strconv.Itoa(60)), + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Value: []byte(strconv.Itoa(60)), + EventTime: timestamppb.New(time.UnixMilli(119999)), }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, - Partition: &reducepb.Partition{ - Start: timestamppb.New(time.UnixMilli(60000)), - End: timestamppb.New(time.UnixMilli(120000)), - Slot: "slot-0", - }, - EventTime: timestamppb.New(time.UnixMilli(119999)), }, - expectedErr: false, }, { name: "reduce_fn_forward_msg_drop_msg", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - return MessagesBuilder().Append(MessageToDrop()) + och <- MessageToDrop() }), input: []*reducepb.ReduceRequest{ { @@ -433,19 +453,20 @@ func TestService_ReduceFn(t *testing.T) { }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { - Tags: []string{DROP}, - Value: []byte{}, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Tags: []string{DROP}, + Value: []byte{}, + EventTime: timestamppb.New(time.UnixMilli(119999)), }, + Partition: &reducepb.Partition{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, - Partition: &reducepb.Partition{ - Start: timestamppb.New(time.UnixMilli(60000)), - End: timestamppb.New(time.UnixMilli(120000)), - Slot: "slot-0", - }, - EventTime: timestamppb.New(time.UnixMilli(119999)), }, expectedErr: false, }, @@ -462,7 +483,7 @@ func TestService_ReduceFn(t *testing.T) { inputCh := make(chan *reducepb.ReduceRequest) outputCh := make(chan *reducepb.ReduceResponse) - result := &reducepb.ReduceResponse{} + result := make([]*reducepb.ReduceResponse, 0) udfReduceFnStream := NewReduceFnServerTest(ctx, inputCh, outputCh) @@ -480,9 +501,9 @@ func TestService_ReduceFn(t *testing.T) { go func() { defer wg.Done() for msg := range outputCh { - result.Results = append(result.Results, msg.Results...) - result.Partition = msg.Partition - result.EventTime = msg.EventTime + if !msg.EOF { + result = append(result, msg) + } } }() @@ -498,8 +519,8 @@ func TestService_ReduceFn(t *testing.T) { } //sort and compare, since order of the output doesn't matter - sort.Slice(result.Results, func(i, j int) bool { - return string(result.Results[i].Value) < string(result.Results[j].Value) + sort.Slice(result, func(i, j int) bool { + return string(result[i].Result.Value) < string(result[j].Result.Value) }) if !reflect.DeepEqual(result, tt.expected) { diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index eacc3f19..b9a72c68 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -18,24 +18,33 @@ type reduceTask struct { partition *v1.Partition reducer Reducer inputCh chan Datum + outputCh chan Message doneCh chan struct{} } // buildReduceResponse builds the reduce response from the messages. -func (rt *reduceTask) buildReduceResponse(messages Messages) *v1.ReduceResponse { - results := make([]*v1.ReduceResponse_Result, 0, len(messages)) - for _, message := range messages { - results = append(results, &v1.ReduceResponse_Result{ - Keys: message.Keys(), - Value: message.Value(), - Tags: message.Tags(), - }) +func (rt *reduceTask) buildReduceResponse(message Message) *v1.ReduceResponse { + + response := &v1.ReduceResponse{ + Result: &v1.ReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + EventTime: timestamppb.New(rt.partition.End.AsTime().Add(-1 * time.Millisecond)), + }, + Partition: rt.partition, } + return response +} + +func (rt *reduceTask) buildEOFResponse() *v1.ReduceResponse { response := &v1.ReduceResponse{ - Results: results, + Result: &v1.ReduceResponse_Result{ + EventTime: timestamppb.New(rt.partition.End.AsTime().Add(-1 * time.Millisecond)), + }, Partition: rt.partition, - EventTime: timestamppb.New(rt.partition.End.AsTime().Add(-1 * time.Millisecond)), + EOF: true, } return response @@ -51,17 +60,17 @@ func (rt *reduceTask) uniqueKey() string { // reduceTaskManager manages the reduce tasks for a reduce operation. type reduceTaskManager struct { - reducer Reducer - tasks map[string]*reduceTask - outputCh chan *v1.ReduceResponse - rw sync.RWMutex + reducer Reducer + tasks map[string]*reduceTask + responseCh chan *v1.ReduceResponse + rw sync.RWMutex } func newReduceTaskManager(reducer Reducer) *reduceTaskManager { return &reduceTaskManager{ - reducer: reducer, - tasks: make(map[string]*reduceTask), - outputCh: make(chan *v1.ReduceResponse), + reducer: reducer, + tasks: make(map[string]*reduceTask), + responseCh: make(chan *v1.ReduceResponse), } } @@ -79,6 +88,7 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), partition: request.Operation.Partitions[0], inputCh: make(chan Datum), + outputCh: make(chan Message), doneCh: make(chan struct{}), } @@ -88,9 +98,24 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce rtm.rw.Unlock() go func() { - msgs := rtm.reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) - // write the output to the output channel, service will forward it to downstream - rtm.outputCh <- task.buildReduceResponse(msgs) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for message := range task.outputCh { + // write the output to the output channel, service will forward it to downstream + rtm.responseCh <- task.buildReduceResponse(message) + } + // send EOF + rtm.responseCh <- task.buildEOFResponse() + }() + + // invoke the reduce function + rtm.reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh, md) + // close the output channel after the reduce function is done + close(task.outputCh) + // wait for the output to be forwarded + wg.Wait() // send a done signal close(task.doneCh) }() @@ -119,7 +144,7 @@ func (rtm *reduceTaskManager) AppendToTask(ctx context.Context, request *v1.Redu // OutputChannel returns the output channel for the reduce task manager to read the results. func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { - return rtm.outputCh + return rtm.responseCh } // WaitAll waits for all the reduce tasks to complete. @@ -135,7 +160,7 @@ func (rtm *reduceTaskManager) WaitAll() { <-task.doneCh } // after all the tasks are completed, close the output channel - close(rtm.outputCh) + close(rtm.responseCh) } // CloseAll closes all the reduce tasks. diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go index ca07210e..c7039fed 100644 --- a/pkg/sessionreducer/examples/counter/main.go +++ b/pkg/sessionreducer/examples/counter/main.go @@ -39,12 +39,12 @@ func (c *Counter) MergeAccumulator(ctx context.Context, accumulator []byte) { c.count.Add(int32(val)) } -func reduceCounter() sessionreducer.SessionReducer { +func NewSessionCounter() sessionreducer.SessionReducer { return &Counter{ count: atomic.NewInt32(0), } } func main() { - sessionreducer.NewServer(reduceCounter).Start(context.Background()) + sessionreducer.NewServer(NewSessionCounter).Start(context.Background()) } From e4e32df2d00e7390d5afe151aad939272d5537b8 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Thu, 23 Nov 2023 13:28:48 +0530 Subject: [PATCH 11/27] support partial response Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.pb.go | 112 +++++++++-------- .../proto/globalreduce/v1/globalreduce.proto | 9 +- pkg/apis/proto/reduce/v1/reduce.pb.go | 4 +- pkg/apis/proto/reduce/v1/reduce.proto | 2 +- .../sessionreduce/v1/sessionreduce.pb.go | 114 ++++++++++-------- .../sessionreduce/v1/sessionreduce.proto | 8 +- pkg/globalreducer/interface.go | 2 +- pkg/globalreducer/task_manager.go | 70 +++++------ pkg/globalreducer/types.go | 35 ------ pkg/reducer/examples/counter/main.go | 6 +- pkg/reducer/examples/sum/go.mod | 2 +- pkg/reducer/examples/sum/go.sum | 2 + pkg/reducer/examples/sum/main.go | 6 +- pkg/reducer/message.go | 18 --- pkg/sessionreducer/examples/counter/main.go | 4 +- pkg/sessionreducer/interface.go | 2 +- pkg/sessionreducer/message.go | 18 --- pkg/sessionreducer/task_manager.go | 61 +++++++--- 18 files changed, 227 insertions(+), 248 deletions(-) diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go index 48e40d54..3ac4f7ea 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go @@ -207,11 +207,11 @@ type GlobalReduceResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Results []*GlobalReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Result *GlobalReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - // EventTime represents the event time of the result. - EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + // EOF represents the end of the response for a partition. + EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` } func (x *GlobalReduceResponse) Reset() { @@ -246,9 +246,9 @@ func (*GlobalReduceResponse) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{2} } -func (x *GlobalReduceResponse) GetResults() []*GlobalReduceResponse_Result { +func (x *GlobalReduceResponse) GetResult() *GlobalReduceResponse_Result { if x != nil { - return x.Results + return x.Result } return nil } @@ -260,11 +260,11 @@ func (x *GlobalReduceResponse) GetPartition() *Partition { return nil } -func (x *GlobalReduceResponse) GetEventTime() *timestamppb.Timestamp { +func (x *GlobalReduceResponse) GetEOF() bool { if x != nil { - return x.EventTime + return x.EOF } - return nil + return false } // * @@ -454,6 +454,8 @@ type GlobalReduceResponse_Result struct { Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` + // EventTime represents the event time of the result. + EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } func (x *GlobalReduceResponse_Result) Reset() { @@ -509,6 +511,13 @@ func (x *GlobalReduceResponse_Result) GetTags() []string { return nil } +func (x *GlobalReduceResponse_Result) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + var File_pkg_apis_proto_globalreduce_v1_globalreduce_proto protoreflect.FileDescriptor var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ @@ -564,43 +573,44 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, - 0x61, 0x72, 0x6b, 0x22, 0x9b, 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, - 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, - 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x39, - 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, - 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x07, - 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, - 0x1e, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, - 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, - 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, - 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x72, 0x6b, 0x22, 0xac, 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x06, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, + 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, + 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, + 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, + 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, + 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x1a, 0x1e, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, + 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, + 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -634,13 +644,13 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = []int32{ 8, // 1: globalreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp 6, // 2: globalreduce.v1.GlobalReduceRequest.payload:type_name -> globalreduce.v1.GlobalReduceRequest.Payload 5, // 3: globalreduce.v1.GlobalReduceRequest.operation:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation - 7, // 4: globalreduce.v1.GlobalReduceResponse.results:type_name -> globalreduce.v1.GlobalReduceResponse.Result + 7, // 4: globalreduce.v1.GlobalReduceResponse.result:type_name -> globalreduce.v1.GlobalReduceResponse.Result 1, // 5: globalreduce.v1.GlobalReduceResponse.partition:type_name -> globalreduce.v1.Partition - 8, // 6: globalreduce.v1.GlobalReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Event - 1, // 8: globalreduce.v1.GlobalReduceRequest.WindowOperation.partitions:type_name -> globalreduce.v1.Partition - 8, // 9: globalreduce.v1.GlobalReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp - 8, // 10: globalreduce.v1.GlobalReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 0, // 6: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Event + 1, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.partitions:type_name -> globalreduce.v1.Partition + 8, // 8: globalreduce.v1.GlobalReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 9: globalreduce.v1.GlobalReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 8, // 10: globalreduce.v1.GlobalReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp 2, // 11: globalreduce.v1.GlobalReduce.GlobalReduceFn:input_type -> globalreduce.v1.GlobalReduceRequest 9, // 12: globalreduce.v1.GlobalReduce.IsReady:input_type -> google.protobuf.Empty 3, // 13: globalreduce.v1.GlobalReduce.GlobalReduceFn:output_type -> globalreduce.v1.GlobalReduceResponse diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.proto b/pkg/apis/proto/globalreduce/v1/globalreduce.proto index 405375fd..5424a484 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.proto +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.proto @@ -62,15 +62,18 @@ message GlobalReduceResponse { repeated string keys = 1; bytes value = 2; repeated string tags = 3; + // EventTime represents the event time of the result. + google.protobuf.Timestamp event_time = 4; } - repeated Result results = 1; + Result result = 1; // Partition represents a window partition to which the result belongs. Partition partition = 2; - // EventTime represents the event time of the result. - google.protobuf.Timestamp event_time = 4; + // EOF represents the end of the response for a partition. + bool EOF = 3; + } /** diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index de5d78da..355337aa 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -203,7 +203,7 @@ type ReduceResponse struct { // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` // EOF represents the end of the stream. - EOF bool `protobuf:"varint,4,opt,name=EOF,proto3" json:"EOF,omitempty"` + EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` } func (x *ReduceResponse) Reset() { @@ -567,7 +567,7 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, - 0x03, 0x45, 0x4f, 0x46, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, + 0x03, 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, diff --git a/pkg/apis/proto/reduce/v1/reduce.proto b/pkg/apis/proto/reduce/v1/reduce.proto index 4015ed2a..e4bdbed2 100644 --- a/pkg/apis/proto/reduce/v1/reduce.proto +++ b/pkg/apis/proto/reduce/v1/reduce.proto @@ -70,7 +70,7 @@ message ReduceResponse { Partition partition = 2; // EOF represents the end of the stream. - bool EOF = 4; + bool EOF = 3; } /** diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index b4dd39ef..f8fecf89 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -213,11 +213,10 @@ type SessionReduceResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Results []*SessionReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Result *SessionReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` // Partition represents a window partition to which the result belongs. Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - // EventTime represents the event time of the result. - EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` } func (x *SessionReduceResponse) Reset() { @@ -252,9 +251,9 @@ func (*SessionReduceResponse) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{2} } -func (x *SessionReduceResponse) GetResults() []*SessionReduceResponse_Result { +func (x *SessionReduceResponse) GetResult() *SessionReduceResponse_Result { if x != nil { - return x.Results + return x.Result } return nil } @@ -266,11 +265,11 @@ func (x *SessionReduceResponse) GetPartition() *Partition { return nil } -func (x *SessionReduceResponse) GetEventTime() *timestamppb.Timestamp { +func (x *SessionReduceResponse) GetEOF() bool { if x != nil { - return x.EventTime + return x.EOF } - return nil + return false } // * @@ -457,9 +456,10 @@ type SessionReduceResponse_Result struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` } func (x *SessionReduceResponse_Result) Reset() { @@ -515,6 +515,13 @@ func (x *SessionReduceResponse_Result) GetTags() []string { return nil } +func (x *SessionReduceResponse_Result) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + var File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto protoreflect.FileDescriptor var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ @@ -572,44 +579,45 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, - 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0x9f, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xb0, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x48, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, - 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, + 0x46, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, + 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, + 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, + 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, + 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, + 0xbb, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x12, 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x1a, - 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, - 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x12, 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, - 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, - 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, + 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, + 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, + 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -643,13 +651,13 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ 8, // 1: sessionreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp 6, // 2: sessionreduce.v1.SessionReduceRequest.payload:type_name -> sessionreduce.v1.SessionReduceRequest.Payload 5, // 3: sessionreduce.v1.SessionReduceRequest.operation:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation - 7, // 4: sessionreduce.v1.SessionReduceResponse.results:type_name -> sessionreduce.v1.SessionReduceResponse.Result + 7, // 4: sessionreduce.v1.SessionReduceResponse.result:type_name -> sessionreduce.v1.SessionReduceResponse.Result 1, // 5: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition - 8, // 6: sessionreduce.v1.SessionReduceResponse.event_time:type_name -> google.protobuf.Timestamp - 0, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event - 1, // 8: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition - 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp - 8, // 10: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 0, // 6: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + 1, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition + 8, // 8: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 8, // 10: sessionreduce.v1.SessionReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp 2, // 11: sessionreduce.v1.SessionReduce.SessionReduceFn:input_type -> sessionreduce.v1.SessionReduceRequest 9, // 12: sessionreduce.v1.SessionReduce.IsReady:input_type -> google.protobuf.Empty 3, // 13: sessionreduce.v1.SessionReduce.SessionReduceFn:output_type -> sessionreduce.v1.SessionReduceResponse diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto index ad9f60bc..06446fb2 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -64,15 +64,15 @@ message SessionReduceResponse { repeated string keys = 1; bytes value = 2; repeated string tags = 3; + google.protobuf.Timestamp event_time = 4; } - repeated Result results = 1; + Result result = 1; // Partition represents a window partition to which the result belongs. Partition partition = 2; - - // EventTime represents the event time of the result. - google.protobuf.Timestamp event_time = 4; + + bool EOF = 3; } /** diff --git a/pkg/globalreducer/interface.go b/pkg/globalreducer/interface.go index 9e213024..3950286f 100644 --- a/pkg/globalreducer/interface.go +++ b/pkg/globalreducer/interface.go @@ -14,5 +14,5 @@ type Datum interface { // GlobalReducer is the interface which can be used to implement a session reduce operation. type GlobalReducer interface { - GlobalReduce(ctx context.Context, keys []string, input <-chan Datum, output <-chan Messages) + GlobalReduce(ctx context.Context, keys []string, input <-chan Datum, output <-chan Message) } diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go index 28605ea1..d9de6da9 100644 --- a/pkg/globalreducer/task_manager.go +++ b/pkg/globalreducer/task_manager.go @@ -18,28 +18,36 @@ type globalReduceTask struct { partition *v1.Partition globalReducer GlobalReducer inputCh chan Datum - outputCh chan Messages + outputCh chan Message latestWatermark *atomic.Time doneCh chan struct{} } // buildGlobalReduceResponse builds the global reduce response from the messages. -func (rt *globalReduceTask) buildGlobalReduceResponse(messages Messages) *v1.GlobalReduceResponse { - results := make([]*v1.GlobalReduceResponse_Result, 0, len(messages)) - for _, message := range messages { - results = append(results, &v1.GlobalReduceResponse_Result{ - Keys: message.Keys(), - Value: message.Value(), - Tags: message.Tags(), - }) - } +func (rt *globalReduceTask) buildGlobalReduceResponse(message Message) *v1.GlobalReduceResponse { // the event time is the latest watermark // since for global window, partition start and end time will be -1 response := &v1.GlobalReduceResponse{ - Results: results, + Result: &v1.GlobalReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + EventTime: timestamppb.New(rt.latestWatermark.Load()), + }, Partition: rt.partition, - EventTime: timestamppb.New(rt.latestWatermark.Load()), + } + + return response +} + +func (rt *globalReduceTask) buildEOFResponse() *v1.GlobalReduceResponse { + response := &v1.GlobalReduceResponse{ + Result: &v1.GlobalReduceResponse_Result{ + EventTime: timestamppb.New(rt.latestWatermark.Load()), + }, + Partition: rt.partition, + EOF: true, } return response @@ -57,14 +65,14 @@ func (rt *globalReduceTask) uniqueKey() string { type globalReduceTaskManager struct { globalReducer GlobalReducer tasks map[string]*globalReduceTask - outputCh chan *v1.GlobalReduceResponse + responseCh chan *v1.GlobalReduceResponse rw sync.RWMutex } func newReduceTaskManager(globalReducer GlobalReducer) *globalReduceTaskManager { return &globalReduceTaskManager{ tasks: make(map[string]*globalReduceTask), - outputCh: make(chan *v1.GlobalReduceResponse), + responseCh: make(chan *v1.GlobalReduceResponse), globalReducer: globalReducer, } } @@ -81,7 +89,7 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. partition: request.Operation.Partitions[0], globalReducer: rtm.globalReducer, inputCh: make(chan Datum), - outputCh: make(chan Messages), + outputCh: make(chan Message), doneCh: make(chan struct{}), latestWatermark: atomic.NewTime(time.Time{}), } @@ -97,31 +105,25 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. wg.Add(1) go func() { defer wg.Done() - readLoop: - for { - select { - case <-ctx.Done(): - return - case message, ok := <-task.outputCh: - if !ok { - break readLoop - } - rtm.outputCh <- task.buildGlobalReduceResponse(message) - } + for msg := range task.outputCh { + rtm.responseCh <- task.buildGlobalReduceResponse(msg) } - // send a done signal - close(task.doneCh) - // delete the task from the tasks list - rtm.rw.Lock() - delete(rtm.tasks, key) - rtm.rw.Unlock() + // send the EOF response + rtm.responseCh <- task.buildEOFResponse() }() + // start the global reduce operation task.globalReducer.GlobalReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) // close the output channel after the global reduce operation is completed close(task.outputCh) // wait for the output channel reader to complete wg.Wait() + // send a done signal + close(task.doneCh) + // delete the task from the tasks list + rtm.rw.Lock() + delete(rtm.tasks, key) + rtm.rw.Unlock() }() // send the datum to the task if the payload is not nil @@ -179,7 +181,7 @@ func (rtm *globalReduceTaskManager) CloseTask(request *v1.GlobalReduceRequest) { // OutputChannel returns the output channel for the reduce task manager to read the results. func (rtm *globalReduceTaskManager) OutputChannel() <-chan *v1.GlobalReduceResponse { - return rtm.outputCh + return rtm.responseCh } // WaitAll waits for all the reduce tasks to complete. @@ -195,7 +197,7 @@ func (rtm *globalReduceTaskManager) WaitAll() { <-task.doneCh } // after all the tasks are completed, close the output channel - close(rtm.outputCh) + close(rtm.responseCh) } // CloseAll closes all the reduce tasks. diff --git a/pkg/globalreducer/types.go b/pkg/globalreducer/types.go index 69778e65..c49df3f1 100644 --- a/pkg/globalreducer/types.go +++ b/pkg/globalreducer/types.go @@ -28,38 +28,3 @@ func (h *handlerDatum) EventTime() time.Time { func (h *handlerDatum) Watermark() time.Time { return h.watermark } - -// intervalWindow implements IntervalWindow interface which will be passed as metadata -// to reduce handlers -type intervalWindow struct { - startTime time.Time - endTime time.Time -} - -func NewIntervalWindow(startTime time.Time, endTime time.Time) IntervalWindow { - return &intervalWindow{ - startTime: startTime, - endTime: endTime, - } -} - -func (i *intervalWindow) StartTime() time.Time { - return i.startTime -} - -func (i *intervalWindow) EndTime() time.Time { - return i.endTime -} - -// metadata implements Metadata interface which will be passed to reduce function. -type metadata struct { - intervalWindow IntervalWindow -} - -func NewMetadata(window IntervalWindow) Metadata { - return &metadata{intervalWindow: window} -} - -func (m *metadata) IntervalWindow() IntervalWindow { - return m.intervalWindow -} diff --git a/pkg/reducer/examples/counter/main.go b/pkg/reducer/examples/counter/main.go index a3868732..2bc950c6 100644 --- a/pkg/reducer/examples/counter/main.go +++ b/pkg/reducer/examples/counter/main.go @@ -7,16 +7,16 @@ import ( "github.com/numaproj/numaflow-go/pkg/reducer" ) -func reduceCounter(_ context.Context, keys []string, reduceCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { +func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducer.Datum, outputCh chan<- reducer.Message, md reducer.Metadata) { // count the incoming events var resultKeys = keys var resultVal []byte var counter = 0 - for range reduceCh { + for range inputCh { counter++ } resultVal = []byte(strconv.Itoa(counter)) - return reducer.MessagesBuilder().Append(reducer.NewMessage(resultVal).WithKeys(resultKeys)) + outputCh <- reducer.NewMessage(resultVal).WithKeys(resultKeys) } func main() { diff --git a/pkg/reducer/examples/sum/go.mod b/pkg/reducer/examples/sum/go.mod index 8b968e1e..d85516be 100644 --- a/pkg/reducer/examples/sum/go.mod +++ b/pkg/reducer/examples/sum/go.mod @@ -2,7 +2,7 @@ module even_odd go 1.20 -require github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe +require github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducer/examples/sum/go.sum b/pkg/reducer/examples/sum/go.sum index 18c03144..9e69119c 100644 --- a/pkg/reducer/examples/sum/go.sum +++ b/pkg/reducer/examples/sum/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe h1:nK/BGffgwQ4L9pyllwzSZttPxMf+OOqK3DOP97KZdRk= github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe/go.mod h1:zcJq1YAA/jnxCQLW7EFK4+HXWCd2QtW4tMOvRjHFa2g= +github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd h1:NuigpRGq/KpLxXhpw52hNamn80iSjuctQGMHJOqqOQM= +github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/reducer/examples/sum/main.go b/pkg/reducer/examples/sum/main.go index cd2dffa8..dd0b43f1 100644 --- a/pkg/reducer/examples/sum/main.go +++ b/pkg/reducer/examples/sum/main.go @@ -13,7 +13,7 @@ import ( type Sum struct { } -func (s *Sum) Reduce(ctx context.Context, keys []string, reduceCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { +func (s *Sum) Reduce(ctx context.Context, keys []string, inputCh <-chan reducer.Datum, outputCh chan<- reducer.Message, md reducer.Metadata) { // sum up values for the same keys intervalWindow := md.IntervalWindow() _ = intervalWindow @@ -21,7 +21,7 @@ func (s *Sum) Reduce(ctx context.Context, keys []string, reduceCh <-chan reducer var resultVal []byte var sum = 0 // sum up the values - for d := range reduceCh { + for d := range inputCh { val := d.Value() eventTime := d.EventTime() _ = eventTime @@ -36,7 +36,7 @@ func (s *Sum) Reduce(ctx context.Context, keys []string, reduceCh <-chan reducer sum += v } resultVal = []byte(strconv.Itoa(sum)) - return reducer.MessagesBuilder().Append(reducer.NewMessage(resultVal).WithKeys(resultKeys)) + outputCh <- reducer.NewMessage(resultVal).WithKeys(resultKeys) } func main() { diff --git a/pkg/reducer/message.go b/pkg/reducer/message.go index e7114887..716ac8da 100644 --- a/pkg/reducer/message.go +++ b/pkg/reducer/message.go @@ -50,21 +50,3 @@ func (m Message) Value() []byte { func (m Message) Tags() []string { return m.tags } - -type Messages []Message - -// MessagesBuilder returns an empty instance of Messages -func MessagesBuilder() Messages { - return Messages{} -} - -// Append appends a Message -func (m Messages) Append(msg Message) Messages { - m = append(m, msg) - return m -} - -// Items returns the message list -func (m Messages) Items() []Message { - return m -} diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go index c7039fed..83d547f3 100644 --- a/pkg/sessionreducer/examples/counter/main.go +++ b/pkg/sessionreducer/examples/counter/main.go @@ -15,13 +15,13 @@ type Counter struct { count *atomic.Int32 } -func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan sessionreducer.Datum) sessionreducer.Messages { +func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan sessionreducer.Datum, outputCh chan<- sessionreducer.Message) { log.Println("SessionReduce invoked") for range input { c.count.Inc() } log.Println("SessionReduce done") - return sessionreducer.MessagesBuilder().Append(sessionreducer.NewMessage([]byte(fmt.Sprintf("%d", c.count.Load()))).WithKeys(keys)) + outputCh <- sessionreducer.NewMessage([]byte(fmt.Sprintf("%d", c.count.Load()))).WithKeys(keys) } func (c *Counter) Accumulator(ctx context.Context) []byte { diff --git a/pkg/sessionreducer/interface.go b/pkg/sessionreducer/interface.go index ec2289f3..c356a5ae 100644 --- a/pkg/sessionreducer/interface.go +++ b/pkg/sessionreducer/interface.go @@ -14,7 +14,7 @@ type Datum interface { // SessionReducer is the interface which can be used to implement a session reduce operation. type SessionReducer interface { - SessionReduce(ctx context.Context, keys []string, input <-chan Datum) Messages + SessionReduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message) Accumulator(ctx context.Context) []byte MergeAccumulator(ctx context.Context, accumulator []byte) } diff --git a/pkg/sessionreducer/message.go b/pkg/sessionreducer/message.go index e00d8659..bfc69af0 100644 --- a/pkg/sessionreducer/message.go +++ b/pkg/sessionreducer/message.go @@ -50,21 +50,3 @@ func (m Message) Value() []byte { func (m Message) Tags() []string { return m.tags } - -type Messages []Message - -// MessagesBuilder returns an empty instance of Messages -func MessagesBuilder() Messages { - return Messages{} -} - -// Append appends a Message -func (m Messages) Append(msg Message) Messages { - m = append(m, msg) - return m -} - -// Items returns the message list -func (m Messages) Items() []Message { - return m -} diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index fd10f930..3c4a3c29 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -18,26 +18,37 @@ type sessionReduceTask struct { partition *v1.Partition sessionReducer SessionReducer inputCh chan Datum + outputCh chan Message doneCh chan struct{} merged *atomic.Bool } // buildSessionReduceResponse builds the session reduce response from the messages. -func (rt *sessionReduceTask) buildSessionReduceResponse(messages Messages) *v1.SessionReduceResponse { - results := make([]*v1.SessionReduceResponse_Result, 0, len(messages)) - for _, message := range messages { - results = append(results, &v1.SessionReduceResponse_Result{ +func (rt *sessionReduceTask) buildSessionReduceResponse(message Message) *v1.SessionReduceResponse { + + response := &v1.SessionReduceResponse{ + Result: &v1.SessionReduceResponse_Result{ Keys: message.Keys(), Value: message.Value(), Tags: message.Tags(), - }) + // event time is the end time of the window - 1 millisecond + EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + }, + Partition: rt.partition, } + return response +} + +// buildEOFResponse builds the EOF response for the session reduce task. +func (rt *sessionReduceTask) buildEOFResponse() *v1.SessionReduceResponse { response := &v1.SessionReduceResponse{ - Results: results, + Result: &v1.SessionReduceResponse_Result{ + // event time is the end time of the window - 1 millisecond + EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + }, Partition: rt.partition, - // event time is the end time of the window - 1 millisecond - EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + EOF: true, } return response @@ -55,7 +66,7 @@ func (rt *sessionReduceTask) uniqueKey() string { type sessionReduceTaskManager struct { sessionReducerFactory CreateSessionReducer tasks map[string]*sessionReduceTask - outputCh chan *v1.SessionReduceResponse + responseCh chan *v1.SessionReduceResponse rw sync.RWMutex } @@ -63,7 +74,7 @@ func newReduceTaskManager(sessionReducerFactory CreateSessionReducer) *sessionRe return &sessionReduceTaskManager{ sessionReducerFactory: sessionReducerFactory, tasks: make(map[string]*sessionReduceTask), - outputCh: make(chan *v1.SessionReduceResponse), + responseCh: make(chan *v1.SessionReduceResponse), } } @@ -80,6 +91,7 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 partition: request.Operation.Partitions[0], sessionReducer: rtm.sessionReducerFactory(), inputCh: make(chan Datum), + outputCh: make(chan Message), doneCh: make(chan struct{}), merged: atomic.NewBool(false), } @@ -91,12 +103,25 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 rtm.rw.Unlock() go func() { - msgs := task.sessionReducer.SessionReduce(ctx, task.partition.GetKeys(), task.inputCh) - // write the output to the output channel, service will forward it to downstream - // if the task is merged to another task, we don't need to send the response - if !task.merged.Load() { - rtm.outputCh <- task.buildSessionReduceResponse(msgs) - } + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for message := range task.outputCh { + if !task.merged.Load() { + // write the output to the output channel, service will forward it to downstream + // if the task is merged to another task, we don't need to send the response + rtm.responseCh <- task.buildSessionReduceResponse(message) + } + } + // send EOF + rtm.responseCh <- task.buildEOFResponse() + }() + + task.sessionReducer.SessionReduce(ctx, task.partition.GetKeys(), task.inputCh, task.outputCh) + // close the output channel and wait for the response to be forwarded + close(task.outputCh) + wg.Wait() // send a done signal close(task.doneCh) // delete the task from the tasks list @@ -256,7 +281,7 @@ func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest // OutputChannel returns the output channel for the reduce task manager to read the results. func (rtm *sessionReduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { - return rtm.outputCh + return rtm.responseCh } // WaitAll waits for all the pending reduce tasks to complete. @@ -272,7 +297,7 @@ func (rtm *sessionReduceTaskManager) WaitAll() { <-task.doneCh } // after all the tasks are completed, close the output channel - close(rtm.outputCh) + close(rtm.responseCh) } // CloseAll closes all the reduce tasks. From b5e4c92e9dc53d1a7371712edf69799e84a74da2 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Mon, 27 Nov 2023 13:29:56 +0530 Subject: [PATCH 12/27] fix naming Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.pb.go | 236 ++++++++--------- .../proto/globalreduce/v1/globalreduce.proto | 13 +- pkg/apis/proto/reduce/v1/reduce.pb.go | 172 ++++++------- pkg/apis/proto/reduce/v1/reduce.proto | 13 +- .../sessionreduce/v1/sessionreduce.pb.go | 241 +++++++++--------- .../sessionreduce/v1/sessionreduce.proto | 12 +- pkg/globalreducer/task_manager.go | 42 +-- pkg/reducer/service_test.go | 42 +-- pkg/reducer/task_manager.go | 60 +++-- pkg/sessionreducer/task_manager.go | 96 +++---- 10 files changed, 470 insertions(+), 457 deletions(-) diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go index 3ac4f7ea..1ef68d13 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go @@ -71,8 +71,9 @@ func (GlobalReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []int return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1, 0, 0} } -// Partition represents a window partition. -type Partition struct { +// KeyedWindow represents a window with keys. +// since the client track the keys, we use keyed window. +type KeyedWindow struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -83,8 +84,8 @@ type Partition struct { Keys []string `protobuf:"bytes,4,rep,name=keys,proto3" json:"keys,omitempty"` } -func (x *Partition) Reset() { - *x = Partition{} +func (x *KeyedWindow) Reset() { + *x = KeyedWindow{} if protoimpl.UnsafeEnabled { mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -92,13 +93,13 @@ func (x *Partition) Reset() { } } -func (x *Partition) String() string { +func (x *KeyedWindow) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Partition) ProtoMessage() {} +func (*KeyedWindow) ProtoMessage() {} -func (x *Partition) ProtoReflect() protoreflect.Message { +func (x *KeyedWindow) ProtoReflect() protoreflect.Message { mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -110,33 +111,33 @@ func (x *Partition) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Partition.ProtoReflect.Descriptor instead. -func (*Partition) Descriptor() ([]byte, []int) { +// Deprecated: Use KeyedWindow.ProtoReflect.Descriptor instead. +func (*KeyedWindow) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{0} } -func (x *Partition) GetStart() *timestamppb.Timestamp { +func (x *KeyedWindow) GetStart() *timestamppb.Timestamp { if x != nil { return x.Start } return nil } -func (x *Partition) GetEnd() *timestamppb.Timestamp { +func (x *KeyedWindow) GetEnd() *timestamppb.Timestamp { if x != nil { return x.End } return nil } -func (x *Partition) GetSlot() string { +func (x *KeyedWindow) GetSlot() string { if x != nil { return x.Slot } return "" } -func (x *Partition) GetKeys() []string { +func (x *KeyedWindow) GetKeys() []string { if x != nil { return x.Keys } @@ -208,9 +209,9 @@ type GlobalReduceResponse struct { unknownFields protoimpl.UnknownFields Result *GlobalReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - // Partition represents a window partition to which the result belongs. - Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - // EOF represents the end of the response for a partition. + // keyedWindow represents a window to which the result belongs. + KeyedWindow *KeyedWindow `protobuf:"bytes,2,opt,name=keyedWindow,proto3" json:"keyedWindow,omitempty"` + // EOF represents the end of the response for a window. EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` } @@ -253,9 +254,9 @@ func (x *GlobalReduceResponse) GetResult() *GlobalReduceResponse_Result { return nil } -func (x *GlobalReduceResponse) GetPartition() *Partition { +func (x *GlobalReduceResponse) GetKeyedWindow() *KeyedWindow { if x != nil { - return x.Partition + return x.KeyedWindow } return nil } @@ -323,8 +324,8 @@ type GlobalReduceRequest_WindowOperation struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Event GlobalReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=globalreduce.v1.GlobalReduceRequest_WindowOperation_Event" json:"event,omitempty"` - Partitions []*Partition `protobuf:"bytes,2,rep,name=partitions,proto3" json:"partitions,omitempty"` + Event GlobalReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=globalreduce.v1.GlobalReduceRequest_WindowOperation_Event" json:"event,omitempty"` + KeyedWindows []*KeyedWindow `protobuf:"bytes,2,rep,name=keyedWindows,proto3" json:"keyedWindows,omitempty"` } func (x *GlobalReduceRequest_WindowOperation) Reset() { @@ -366,9 +367,9 @@ func (x *GlobalReduceRequest_WindowOperation) GetEvent() GlobalReduceRequest_Win return GlobalReduceRequest_WindowOperation_OPEN } -func (x *GlobalReduceRequest_WindowOperation) GetPartitions() []*Partition { +func (x *GlobalReduceRequest_WindowOperation) GetKeyedWindows() []*KeyedWindow { if x != nil { - return x.Partitions + return x.KeyedWindows } return nil } @@ -529,88 +530,89 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x22, 0x93, 0x01, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xa8, 0x04, 0x0a, 0x13, 0x47, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x46, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x52, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x67, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xc9, 0x01, 0x0a, - 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x50, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x3a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, - 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, - 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, - 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, - 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, - 0x61, 0x72, 0x6b, 0x22, 0xac, 0x02, 0x0a, 0x14, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x06, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, - 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, - 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, + 0x74, 0x6f, 0x22, 0x95, 0x01, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, + 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xae, 0x04, 0x0a, 0x13, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, - 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x1a, 0x1e, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, - 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, - 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, - 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x74, 0x12, 0x46, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x52, 0x0a, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, + 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xcf, + 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x3a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, + 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x22, 0x28, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, + 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, + 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, + 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xb2, 0x02, 0x0a, 0x14, + 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x3e, 0x0a, 0x0b, 0x6b, 0x65, + 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x0b, 0x6b, + 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, + 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, + 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, + 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, 0x62, + 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, 0x6f, 0x62, + 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x07, 0x49, + 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, + 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, + 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, + 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, + 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -629,25 +631,25 @@ var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes = make([]pr var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes = []interface{}{ (GlobalReduceRequest_WindowOperation_Event)(0), // 0: globalreduce.v1.GlobalReduceRequest.WindowOperation.Event - (*Partition)(nil), // 1: globalreduce.v1.Partition - (*GlobalReduceRequest)(nil), // 2: globalreduce.v1.GlobalReduceRequest - (*GlobalReduceResponse)(nil), // 3: globalreduce.v1.GlobalReduceResponse - (*ReadyResponse)(nil), // 4: globalreduce.v1.ReadyResponse - (*GlobalReduceRequest_WindowOperation)(nil), // 5: globalreduce.v1.GlobalReduceRequest.WindowOperation - (*GlobalReduceRequest_Payload)(nil), // 6: globalreduce.v1.GlobalReduceRequest.Payload - (*GlobalReduceResponse_Result)(nil), // 7: globalreduce.v1.GlobalReduceResponse.Result - (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 9: google.protobuf.Empty + (*KeyedWindow)(nil), // 1: globalreduce.v1.KeyedWindow + (*GlobalReduceRequest)(nil), // 2: globalreduce.v1.GlobalReduceRequest + (*GlobalReduceResponse)(nil), // 3: globalreduce.v1.GlobalReduceResponse + (*ReadyResponse)(nil), // 4: globalreduce.v1.ReadyResponse + (*GlobalReduceRequest_WindowOperation)(nil), // 5: globalreduce.v1.GlobalReduceRequest.WindowOperation + (*GlobalReduceRequest_Payload)(nil), // 6: globalreduce.v1.GlobalReduceRequest.Payload + (*GlobalReduceResponse_Result)(nil), // 7: globalreduce.v1.GlobalReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = []int32{ - 8, // 0: globalreduce.v1.Partition.start:type_name -> google.protobuf.Timestamp - 8, // 1: globalreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp + 8, // 0: globalreduce.v1.KeyedWindow.start:type_name -> google.protobuf.Timestamp + 8, // 1: globalreduce.v1.KeyedWindow.end:type_name -> google.protobuf.Timestamp 6, // 2: globalreduce.v1.GlobalReduceRequest.payload:type_name -> globalreduce.v1.GlobalReduceRequest.Payload 5, // 3: globalreduce.v1.GlobalReduceRequest.operation:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation 7, // 4: globalreduce.v1.GlobalReduceResponse.result:type_name -> globalreduce.v1.GlobalReduceResponse.Result - 1, // 5: globalreduce.v1.GlobalReduceResponse.partition:type_name -> globalreduce.v1.Partition + 1, // 5: globalreduce.v1.GlobalReduceResponse.keyedWindow:type_name -> globalreduce.v1.KeyedWindow 0, // 6: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Event - 1, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.partitions:type_name -> globalreduce.v1.Partition + 1, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.keyedWindows:type_name -> globalreduce.v1.KeyedWindow 8, // 8: globalreduce.v1.GlobalReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 9: globalreduce.v1.GlobalReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp 8, // 10: globalreduce.v1.GlobalReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp @@ -669,7 +671,7 @@ func file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_init() { } if !protoimpl.UnsafeEnabled { file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Partition); i { + switch v := v.(*KeyedWindow); i { case 0: return &v.state case 1: diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.proto b/pkg/apis/proto/globalreduce/v1/globalreduce.proto index 5424a484..3d38dfb7 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.proto +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.proto @@ -16,8 +16,9 @@ service GlobalReduce { rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); } -// Partition represents a window partition. -message Partition { +// KeyedWindow represents a window with keys. +// since the client track the keys, we use keyed window. +message KeyedWindow { google.protobuf.Timestamp start = 1; google.protobuf.Timestamp end = 2; string slot = 3; @@ -38,7 +39,7 @@ message GlobalReduceRequest { } Event event = 1; - repeated Partition partitions = 2; + repeated KeyedWindow keyedWindows = 2; } // Payload represents a payload element. @@ -68,10 +69,10 @@ message GlobalReduceResponse { Result result = 1; - // Partition represents a window partition to which the result belongs. - Partition partition = 2; + // keyedWindow represents a window to which the result belongs. + KeyedWindow keyedWindow = 2; - // EOF represents the end of the response for a partition. + // EOF represents the end of the response for a window. bool EOF = 3; } diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index 355337aa..ff5f7255 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -128,8 +128,9 @@ func (x *ReduceRequest) GetOperation() *ReduceRequest_WindowOperation { return nil } -// Partition represents a window partition. -type Partition struct { +// Window represents a window. +// Since the client doesn't track keys, window doesn't have a keys field. +type Window struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -139,8 +140,8 @@ type Partition struct { Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` } -func (x *Partition) Reset() { - *x = Partition{} +func (x *Window) Reset() { + *x = Window{} if protoimpl.UnsafeEnabled { mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -148,13 +149,13 @@ func (x *Partition) Reset() { } } -func (x *Partition) String() string { +func (x *Window) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Partition) ProtoMessage() {} +func (*Window) ProtoMessage() {} -func (x *Partition) ProtoReflect() protoreflect.Message { +func (x *Window) ProtoReflect() protoreflect.Message { mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -166,26 +167,26 @@ func (x *Partition) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Partition.ProtoReflect.Descriptor instead. -func (*Partition) Descriptor() ([]byte, []int) { +// Deprecated: Use Window.ProtoReflect.Descriptor instead. +func (*Window) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{1} } -func (x *Partition) GetStart() *timestamppb.Timestamp { +func (x *Window) GetStart() *timestamppb.Timestamp { if x != nil { return x.Start } return nil } -func (x *Partition) GetEnd() *timestamppb.Timestamp { +func (x *Window) GetEnd() *timestamppb.Timestamp { if x != nil { return x.End } return nil } -func (x *Partition) GetSlot() string { +func (x *Window) GetSlot() string { if x != nil { return x.Slot } @@ -200,9 +201,9 @@ type ReduceResponse struct { unknownFields protoimpl.UnknownFields Result *ReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - // Partition represents a window partition to which the result belongs. - Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - // EOF represents the end of the stream. + // window represents a window to which the result belongs. + Window *Window `protobuf:"bytes,2,opt,name=window,proto3" json:"window,omitempty"` + // EOF represents the end of the response for a window. EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` } @@ -245,9 +246,9 @@ func (x *ReduceResponse) GetResult() *ReduceResponse_Result { return nil } -func (x *ReduceResponse) GetPartition() *Partition { +func (x *ReduceResponse) GetWindow() *Window { if x != nil { - return x.Partition + return x.Window } return nil } @@ -315,8 +316,8 @@ type ReduceRequest_WindowOperation struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Event ReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=reduce.v1.ReduceRequest_WindowOperation_Event" json:"event,omitempty"` - Partitions []*Partition `protobuf:"bytes,2,rep,name=partitions,proto3" json:"partitions,omitempty"` + Event ReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=reduce.v1.ReduceRequest_WindowOperation_Event" json:"event,omitempty"` + Windows []*Window `protobuf:"bytes,2,rep,name=windows,proto3" json:"windows,omitempty"` } func (x *ReduceRequest_WindowOperation) Reset() { @@ -358,9 +359,9 @@ func (x *ReduceRequest_WindowOperation) GetEvent() ReduceRequest_WindowOperation return ReduceRequest_WindowOperation_OPEN } -func (x *ReduceRequest_WindowOperation) GetPartitions() []*Partition { +func (x *ReduceRequest_WindowOperation) GetWindows() []*Window { if x != nil { - return x.Partitions + return x.Windows } return nil } @@ -519,7 +520,7 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0xf8, 0x03, 0x0a, 0x0d, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x22, 0xef, 0x03, 0x0a, 0x0d, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, @@ -528,70 +529,69 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x0b, 0x32, 0x28, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xb7, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xae, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x12, 0x34, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, - 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, - 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, - 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, - 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0x7f, 0x0a, 0x09, 0x50, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x12, 0x2b, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x22, 0x28, 0x0a, + 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, + 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, + 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, + 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, + 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, + 0x72, 0x6b, 0x22, 0x7c, 0x0a, 0x06, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x30, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, + 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, + 0x22, 0x8b, 0x02, 0x0a, 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x29, 0x0a, + 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, + 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x22, 0x94, 0x02, 0x0a, - 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x38, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x32, 0x0a, 0x09, 0x70, 0x61, 0x72, - 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, - 0x03, 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, - 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, - 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, - 0x69, 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, - 0x6e, 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, - 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, - 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, - 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, - 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x25, + 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x18, 0x2e, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, + 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, + 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -611,7 +611,7 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.Messa var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []interface{}{ (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Event (*ReduceRequest)(nil), // 1: reduce.v1.ReduceRequest - (*Partition)(nil), // 2: reduce.v1.Partition + (*Window)(nil), // 2: reduce.v1.Window (*ReduceResponse)(nil), // 3: reduce.v1.ReduceResponse (*ReadyResponse)(nil), // 4: reduce.v1.ReadyResponse (*ReduceRequest_WindowOperation)(nil), // 5: reduce.v1.ReduceRequest.WindowOperation @@ -623,12 +623,12 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []interface{}{ var file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs = []int32{ 6, // 0: reduce.v1.ReduceRequest.payload:type_name -> reduce.v1.ReduceRequest.Payload 5, // 1: reduce.v1.ReduceRequest.operation:type_name -> reduce.v1.ReduceRequest.WindowOperation - 8, // 2: reduce.v1.Partition.start:type_name -> google.protobuf.Timestamp - 8, // 3: reduce.v1.Partition.end:type_name -> google.protobuf.Timestamp + 8, // 2: reduce.v1.Window.start:type_name -> google.protobuf.Timestamp + 8, // 3: reduce.v1.Window.end:type_name -> google.protobuf.Timestamp 7, // 4: reduce.v1.ReduceResponse.result:type_name -> reduce.v1.ReduceResponse.Result - 2, // 5: reduce.v1.ReduceResponse.partition:type_name -> reduce.v1.Partition + 2, // 5: reduce.v1.ReduceResponse.window:type_name -> reduce.v1.Window 0, // 6: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Event - 2, // 7: reduce.v1.ReduceRequest.WindowOperation.partitions:type_name -> reduce.v1.Partition + 2, // 7: reduce.v1.ReduceRequest.WindowOperation.windows:type_name -> reduce.v1.Window 8, // 8: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 9: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp 8, // 10: reduce.v1.ReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp @@ -662,7 +662,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { } } file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Partition); i { + switch v := v.(*Window); i { case 0: return &v.state case 1: diff --git a/pkg/apis/proto/reduce/v1/reduce.proto b/pkg/apis/proto/reduce/v1/reduce.proto index e4bdbed2..5506bc65 100644 --- a/pkg/apis/proto/reduce/v1/reduce.proto +++ b/pkg/apis/proto/reduce/v1/reduce.proto @@ -30,7 +30,7 @@ message ReduceRequest { } Event event = 1; - repeated Partition partitions = 2; + repeated Window windows = 2; } // Payload represents a payload element. @@ -45,8 +45,9 @@ message ReduceRequest { WindowOperation operation = 2; } -// Partition represents a window partition. -message Partition { +// Window represents a window. +// Since the client doesn't track keys, window doesn't have a keys field. +message Window { google.protobuf.Timestamp start = 1; google.protobuf.Timestamp end = 2; string slot = 3; @@ -66,10 +67,10 @@ message ReduceResponse { Result result = 1; - // Partition represents a window partition to which the result belongs. - Partition partition = 2; + // window represents a window to which the result belongs. + Window window = 2; - // EOF represents the end of the stream. + // EOF represents the end of the response for a window. bool EOF = 3; } diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index f8fecf89..7c7a24fb 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -77,8 +77,9 @@ func (SessionReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []in return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0, 0} } -// Partition represents a window partition. -type Partition struct { +// KeyedWindow represents a window with keys. +// since the client track the keys, we use keyed window. +type KeyedWindow struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -89,8 +90,8 @@ type Partition struct { Keys []string `protobuf:"bytes,4,rep,name=keys,proto3" json:"keys,omitempty"` } -func (x *Partition) Reset() { - *x = Partition{} +func (x *KeyedWindow) Reset() { + *x = KeyedWindow{} if protoimpl.UnsafeEnabled { mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -98,13 +99,13 @@ func (x *Partition) Reset() { } } -func (x *Partition) String() string { +func (x *KeyedWindow) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Partition) ProtoMessage() {} +func (*KeyedWindow) ProtoMessage() {} -func (x *Partition) ProtoReflect() protoreflect.Message { +func (x *KeyedWindow) ProtoReflect() protoreflect.Message { mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -116,33 +117,33 @@ func (x *Partition) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Partition.ProtoReflect.Descriptor instead. -func (*Partition) Descriptor() ([]byte, []int) { +// Deprecated: Use KeyedWindow.ProtoReflect.Descriptor instead. +func (*KeyedWindow) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{0} } -func (x *Partition) GetStart() *timestamppb.Timestamp { +func (x *KeyedWindow) GetStart() *timestamppb.Timestamp { if x != nil { return x.Start } return nil } -func (x *Partition) GetEnd() *timestamppb.Timestamp { +func (x *KeyedWindow) GetEnd() *timestamppb.Timestamp { if x != nil { return x.End } return nil } -func (x *Partition) GetSlot() string { +func (x *KeyedWindow) GetSlot() string { if x != nil { return x.Slot } return "" } -func (x *Partition) GetKeys() []string { +func (x *KeyedWindow) GetKeys() []string { if x != nil { return x.Keys } @@ -214,9 +215,10 @@ type SessionReduceResponse struct { unknownFields protoimpl.UnknownFields Result *SessionReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - // Partition represents a window partition to which the result belongs. - Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` - EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` + // keyedWindow represents a window to which the result belongs. + KeyedWindow *KeyedWindow `protobuf:"bytes,2,opt,name=keyedWindow,proto3" json:"keyedWindow,omitempty"` + // EOF represents the end of the response for a window. + EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` } func (x *SessionReduceResponse) Reset() { @@ -258,9 +260,9 @@ func (x *SessionReduceResponse) GetResult() *SessionReduceResponse_Result { return nil } -func (x *SessionReduceResponse) GetPartition() *Partition { +func (x *SessionReduceResponse) GetKeyedWindow() *KeyedWindow { if x != nil { - return x.Partition + return x.KeyedWindow } return nil } @@ -328,8 +330,8 @@ type SessionReduceRequest_WindowOperation struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Event SessionReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=sessionreduce.v1.SessionReduceRequest_WindowOperation_Event" json:"event,omitempty"` - Partitions []*Partition `protobuf:"bytes,2,rep,name=partitions,proto3" json:"partitions,omitempty"` + Event SessionReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=sessionreduce.v1.SessionReduceRequest_WindowOperation_Event" json:"event,omitempty"` + KeyedWindows []*KeyedWindow `protobuf:"bytes,2,rep,name=keyedWindows,proto3" json:"keyedWindows,omitempty"` } func (x *SessionReduceRequest_WindowOperation) Reset() { @@ -371,9 +373,9 @@ func (x *SessionReduceRequest_WindowOperation) GetEvent() SessionReduceRequest_W return SessionReduceRequest_WindowOperation_OPEN } -func (x *SessionReduceRequest_WindowOperation) GetPartitions() []*Partition { +func (x *SessionReduceRequest_WindowOperation) GetKeyedWindows() []*KeyedWindow { if x != nil { - return x.Partitions + return x.KeyedWindows } return nil } @@ -533,91 +535,92 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x93, 0x01, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, - 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xc7, 0x04, 0x0a, 0x14, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x54, - 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x36, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, - 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xe3, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0a, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, - 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, - 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, - 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x95, 0x01, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, - 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, - 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xb0, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x46, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, - 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x39, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, - 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, - 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, - 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, - 0xbb, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x12, 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, - 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, - 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, - 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, - 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xcd, 0x04, + 0x0a, 0x14, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x12, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xe9, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x41, + 0x0a, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x52, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, + 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, + 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, + 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, + 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, + 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xb6, 0x02, + 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x3f, 0x0a, 0x0b, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x52, 0x0b, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, + 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, + 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, + 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, + 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, + 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, + 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -636,25 +639,25 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([] var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []interface{}{ (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event - (*Partition)(nil), // 1: sessionreduce.v1.Partition - (*SessionReduceRequest)(nil), // 2: sessionreduce.v1.SessionReduceRequest - (*SessionReduceResponse)(nil), // 3: sessionreduce.v1.SessionReduceResponse - (*ReadyResponse)(nil), // 4: sessionreduce.v1.ReadyResponse - (*SessionReduceRequest_WindowOperation)(nil), // 5: sessionreduce.v1.SessionReduceRequest.WindowOperation - (*SessionReduceRequest_Payload)(nil), // 6: sessionreduce.v1.SessionReduceRequest.Payload - (*SessionReduceResponse_Result)(nil), // 7: sessionreduce.v1.SessionReduceResponse.Result - (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 9: google.protobuf.Empty + (*KeyedWindow)(nil), // 1: sessionreduce.v1.KeyedWindow + (*SessionReduceRequest)(nil), // 2: sessionreduce.v1.SessionReduceRequest + (*SessionReduceResponse)(nil), // 3: sessionreduce.v1.SessionReduceResponse + (*ReadyResponse)(nil), // 4: sessionreduce.v1.ReadyResponse + (*SessionReduceRequest_WindowOperation)(nil), // 5: sessionreduce.v1.SessionReduceRequest.WindowOperation + (*SessionReduceRequest_Payload)(nil), // 6: sessionreduce.v1.SessionReduceRequest.Payload + (*SessionReduceResponse_Result)(nil), // 7: sessionreduce.v1.SessionReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ - 8, // 0: sessionreduce.v1.Partition.start:type_name -> google.protobuf.Timestamp - 8, // 1: sessionreduce.v1.Partition.end:type_name -> google.protobuf.Timestamp + 8, // 0: sessionreduce.v1.KeyedWindow.start:type_name -> google.protobuf.Timestamp + 8, // 1: sessionreduce.v1.KeyedWindow.end:type_name -> google.protobuf.Timestamp 6, // 2: sessionreduce.v1.SessionReduceRequest.payload:type_name -> sessionreduce.v1.SessionReduceRequest.Payload 5, // 3: sessionreduce.v1.SessionReduceRequest.operation:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation 7, // 4: sessionreduce.v1.SessionReduceResponse.result:type_name -> sessionreduce.v1.SessionReduceResponse.Result - 1, // 5: sessionreduce.v1.SessionReduceResponse.partition:type_name -> sessionreduce.v1.Partition + 1, // 5: sessionreduce.v1.SessionReduceResponse.keyedWindow:type_name -> sessionreduce.v1.KeyedWindow 0, // 6: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event - 1, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.partitions:type_name -> sessionreduce.v1.Partition + 1, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.keyedWindows:type_name -> sessionreduce.v1.KeyedWindow 8, // 8: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp 8, // 10: sessionreduce.v1.SessionReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp @@ -676,7 +679,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { } if !protoimpl.UnsafeEnabled { file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Partition); i { + switch v := v.(*KeyedWindow); i { case 0: return &v.state case 1: diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto index 06446fb2..b09407d8 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -16,8 +16,9 @@ service SessionReduce { rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); } -// Partition represents a window partition. -message Partition { +// KeyedWindow represents a window with keys. +// since the client track the keys, we use keyed window. +message KeyedWindow { google.protobuf.Timestamp start = 1; google.protobuf.Timestamp end = 2; string slot = 3; @@ -40,7 +41,7 @@ message SessionReduceRequest { } Event event = 1; - repeated Partition partitions = 2; + repeated KeyedWindow keyedWindows = 2; } // Payload represents a payload element. @@ -69,9 +70,10 @@ message SessionReduceResponse { Result result = 1; - // Partition represents a window partition to which the result belongs. - Partition partition = 2; + // keyedWindow represents a window to which the result belongs. + KeyedWindow keyedWindow = 2; + // EOF represents the end of the response for a window. bool EOF = 3; } diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go index d9de6da9..c71494d9 100644 --- a/pkg/globalreducer/task_manager.go +++ b/pkg/globalreducer/task_manager.go @@ -15,7 +15,7 @@ import ( // globalReduceTask represents a global reduce task for a global reduce operation. type globalReduceTask struct { - partition *v1.Partition + keyedWindow *v1.KeyedWindow globalReducer GlobalReducer inputCh chan Datum outputCh chan Message @@ -27,7 +27,7 @@ type globalReduceTask struct { func (rt *globalReduceTask) buildGlobalReduceResponse(message Message) *v1.GlobalReduceResponse { // the event time is the latest watermark - // since for global window, partition start and end time will be -1 + // since for global window, keyedWindow start and end time will be -1 response := &v1.GlobalReduceResponse{ Result: &v1.GlobalReduceResponse_Result{ Keys: message.Keys(), @@ -35,7 +35,7 @@ func (rt *globalReduceTask) buildGlobalReduceResponse(message Message) *v1.Globa Tags: message.Tags(), EventTime: timestamppb.New(rt.latestWatermark.Load()), }, - Partition: rt.partition, + KeyedWindow: rt.keyedWindow, } return response @@ -46,8 +46,8 @@ func (rt *globalReduceTask) buildEOFResponse() *v1.GlobalReduceResponse { Result: &v1.GlobalReduceResponse_Result{ EventTime: timestamppb.New(rt.latestWatermark.Load()), }, - Partition: rt.partition, - EOF: true, + KeyedWindow: rt.keyedWindow, + EOF: true, } return response @@ -56,9 +56,9 @@ func (rt *globalReduceTask) buildEOFResponse() *v1.GlobalReduceResponse { // uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. func (rt *globalReduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", - rt.partition.GetStart().AsTime().UnixMilli(), - rt.partition.GetEnd().AsTime().UnixMilli(), - strings.Join(rt.partition.GetKeys(), delimiter)) + rt.keyedWindow.GetStart().AsTime().UnixMilli(), + rt.keyedWindow.GetEnd().AsTime().UnixMilli(), + strings.Join(rt.keyedWindow.GetKeys(), delimiter)) } // globalReduceTaskManager manages the reduce tasks for a global reduce operation. @@ -81,12 +81,12 @@ func newReduceTaskManager(globalReducer GlobalReducer) *globalReduceTaskManager func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1.GlobalReduceRequest) error { rtm.rw.Lock() - if len(request.Operation.Partitions) != 1 { - return fmt.Errorf("invalid number of partitions") + if len(request.Operation.KeyedWindows) != 1 { + return fmt.Errorf("create operation error: invalid number of windows") } task := &globalReduceTask{ - partition: request.Operation.Partitions[0], + keyedWindow: request.Operation.KeyedWindows[0], globalReducer: rtm.globalReducer, inputCh: make(chan Datum), outputCh: make(chan Message), @@ -139,12 +139,12 @@ func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1. // AppendToTask writes the message to the reduce task. // If the reduce task is not found, it will create a new reduce task and start the reduce operation. func (rtm *globalReduceTaskManager) AppendToTask(ctx context.Context, request *v1.GlobalReduceRequest) error { - if len(request.Operation.Partitions) != 1 { - return fmt.Errorf("invalid number of partitions") + if len(request.Operation.KeyedWindows) != 1 { + return fmt.Errorf("append operation error: invalid number of windows") } rtm.rw.RLock() - task, ok := rtm.tasks[generateKey(request.Operation.Partitions[0])] + task, ok := rtm.tasks[generateKey(request.Operation.KeyedWindows[0])] rtm.rw.RUnlock() if !ok { @@ -163,9 +163,9 @@ func (rtm *globalReduceTaskManager) AppendToTask(ctx context.Context, request *v // CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. func (rtm *globalReduceTaskManager) CloseTask(request *v1.GlobalReduceRequest) { rtm.rw.RLock() - tasksToBeClosed := make([]*globalReduceTask, 0, len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - key := generateKey(partition) + tasksToBeClosed := make([]*globalReduceTask, 0, len(request.Operation.KeyedWindows)) + for _, window := range request.Operation.KeyedWindows { + key := generateKey(window) task, ok := rtm.tasks[key] if ok { tasksToBeClosed = append(tasksToBeClosed, task) @@ -214,11 +214,11 @@ func (rtm *globalReduceTaskManager) CloseAll() { } } -func generateKey(partition *v1.Partition) string { +func generateKey(keyedWindow *v1.KeyedWindow) string { return fmt.Sprintf("%d:%d:%s", - partition.GetStart().AsTime().UnixMilli(), - partition.GetEnd().AsTime().UnixMilli(), - strings.Join(partition.GetKeys(), delimiter)) + keyedWindow.GetStart().AsTime().UnixMilli(), + keyedWindow.GetEnd().AsTime().UnixMilli(), + strings.Join(keyedWindow.GetKeys(), delimiter)) } func buildDatum(request *v1.GlobalReduceRequest) Datum { diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index 404878c4..a601c832 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -80,7 +80,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_OPEN, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -98,7 +98,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -116,7 +116,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -133,7 +133,7 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(60)), EventTime: timestamppb.New(time.UnixMilli(119999)), }, - Partition: &reducepb.Partition{ + Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), Slot: "slot-0", @@ -163,7 +163,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_OPEN, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -181,7 +181,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_OPEN, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -199,7 +199,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -217,7 +217,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -235,7 +235,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -253,7 +253,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -270,7 +270,7 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(20)), EventTime: timestamppb.New(time.UnixMilli(119999)), }, - Partition: &reducepb.Partition{ + Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), Slot: "slot-0", @@ -283,7 +283,7 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(40)), EventTime: timestamppb.New(time.UnixMilli(119999)), }, - Partition: &reducepb.Partition{ + Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), Slot: "slot-0", @@ -296,7 +296,7 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(60)), EventTime: timestamppb.New(time.UnixMilli(119999)), }, - Partition: &reducepb.Partition{ + Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), Slot: "slot-0", @@ -326,7 +326,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_OPEN, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -344,7 +344,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -362,7 +362,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -378,7 +378,7 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte(strconv.Itoa(60)), EventTime: timestamppb.New(time.UnixMilli(119999)), }, - Partition: &reducepb.Partition{ + Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), Slot: "slot-0", @@ -407,7 +407,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_OPEN, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -425,7 +425,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_APPEND, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -443,7 +443,7 @@ func TestService_ReduceFn(t *testing.T) { }, Operation: &reducepb.ReduceRequest_WindowOperation{ Event: reducepb.ReduceRequest_WindowOperation_OPEN, - Partitions: []*reducepb.Partition{ + Windows: []*reducepb.Window{ { Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), @@ -460,7 +460,7 @@ func TestService_ReduceFn(t *testing.T) { Value: []byte{}, EventTime: timestamppb.New(time.UnixMilli(119999)), }, - Partition: &reducepb.Partition{ + Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), End: timestamppb.New(time.UnixMilli(120000)), Slot: "slot-0", diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index b9a72c68..3a759998 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -14,12 +14,12 @@ import ( // reduceTask represents a reduce task for a reduce operation. type reduceTask struct { - combinedKey string - partition *v1.Partition - reducer Reducer - inputCh chan Datum - outputCh chan Message - doneCh chan struct{} + keys []string + window *v1.Window + reducer Reducer + inputCh chan Datum + outputCh chan Message + doneCh chan struct{} } // buildReduceResponse builds the reduce response from the messages. @@ -30,9 +30,9 @@ func (rt *reduceTask) buildReduceResponse(message Message) *v1.ReduceResponse { Keys: message.Keys(), Value: message.Value(), Tags: message.Tags(), - EventTime: timestamppb.New(rt.partition.End.AsTime().Add(-1 * time.Millisecond)), + EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), }, - Partition: rt.partition, + Window: rt.window, } return response @@ -41,10 +41,10 @@ func (rt *reduceTask) buildReduceResponse(message Message) *v1.ReduceResponse { func (rt *reduceTask) buildEOFResponse() *v1.ReduceResponse { response := &v1.ReduceResponse{ Result: &v1.ReduceResponse_Result{ - EventTime: timestamppb.New(rt.partition.End.AsTime().Add(-1 * time.Millisecond)), + EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), }, - Partition: rt.partition, - EOF: true, + Window: rt.window, + EOF: true, } return response @@ -53,9 +53,9 @@ func (rt *reduceTask) buildEOFResponse() *v1.ReduceResponse { // uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. func (rt *reduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", - rt.partition.GetStart().AsTime().UnixMilli(), - rt.partition.GetEnd().AsTime().UnixMilli(), - rt.combinedKey) + rt.window.GetStart().AsTime().UnixMilli(), + rt.window.GetEnd().AsTime().UnixMilli(), + strings.Join(rt.keys, delimiter)) } // reduceTaskManager manages the reduce tasks for a reduce operation. @@ -76,20 +76,20 @@ func newReduceTaskManager(reducer Reducer) *reduceTaskManager { // CreateTask creates a new reduce task and starts the reduce operation. func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest) error { - rtm.rw.Lock() - if len(request.Operation.Partitions) != 1 { - return fmt.Errorf("invalid number of partitions") + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("create operation error: invalid number of windows") } - md := NewMetadata(NewIntervalWindow(request.Operation.Partitions[0].GetStart().AsTime(), - request.Operation.Partitions[0].GetEnd().AsTime())) + rtm.rw.Lock() + md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), + request.Operation.Windows[0].GetEnd().AsTime())) task := &reduceTask{ - combinedKey: strings.Join(request.GetPayload().GetKeys(), delimiter), - partition: request.Operation.Partitions[0], - inputCh: make(chan Datum), - outputCh: make(chan Message), - doneCh: make(chan struct{}), + keys: request.GetPayload().GetKeys(), + window: request.Operation.Windows[0], + inputCh: make(chan Datum), + outputCh: make(chan Message), + doneCh: make(chan struct{}), } key := task.uniqueKey() @@ -128,8 +128,12 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce // AppendToTask writes the message to the reduce task. // If the task is not found, it creates a new task and starts the reduce operation. func (rtm *reduceTaskManager) AppendToTask(ctx context.Context, request *v1.ReduceRequest) error { + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("append operation error: invalid number of windows") + } + rtm.rw.RLock() - gKey := generateKey(request.Operation.Partitions[0], request.Payload.Keys) + gKey := generateKey(request.Operation.Windows[0], request.Payload.Keys) task, ok := rtm.tasks[gKey] rtm.rw.RUnlock() @@ -177,10 +181,10 @@ func (rtm *reduceTaskManager) CloseAll() { } } -func generateKey(partition *v1.Partition, keys []string) string { +func generateKey(window *v1.Window, keys []string) string { return fmt.Sprintf("%d:%d:%s", - partition.GetStart().AsTime().UnixMilli(), - partition.GetEnd().AsTime().UnixMilli(), + window.GetStart().AsTime().UnixMilli(), + window.GetEnd().AsTime().UnixMilli(), strings.Join(keys, delimiter)) } diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index 3c4a3c29..2aaf7cd4 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -15,7 +15,7 @@ import ( // sessionReduceTask represents a task for a performing session reduce operation. type sessionReduceTask struct { - partition *v1.Partition + keyedWindow *v1.KeyedWindow sessionReducer SessionReducer inputCh chan Datum outputCh chan Message @@ -32,9 +32,9 @@ func (rt *sessionReduceTask) buildSessionReduceResponse(message Message) *v1.Ses Value: message.Value(), Tags: message.Tags(), // event time is the end time of the window - 1 millisecond - EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + EventTime: timestamppb.New(rt.keyedWindow.GetEnd().AsTime().Add(-1 * time.Millisecond)), }, - Partition: rt.partition, + KeyedWindow: rt.keyedWindow, } return response @@ -45,10 +45,10 @@ func (rt *sessionReduceTask) buildEOFResponse() *v1.SessionReduceResponse { response := &v1.SessionReduceResponse{ Result: &v1.SessionReduceResponse_Result{ // event time is the end time of the window - 1 millisecond - EventTime: timestamppb.New(rt.partition.GetEnd().AsTime().Add(-1 * time.Millisecond)), + EventTime: timestamppb.New(rt.keyedWindow.GetEnd().AsTime().Add(-1 * time.Millisecond)), }, - Partition: rt.partition, - EOF: true, + KeyedWindow: rt.keyedWindow, + EOF: true, } return response @@ -57,9 +57,9 @@ func (rt *sessionReduceTask) buildEOFResponse() *v1.SessionReduceResponse { // uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. func (rt *sessionReduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", - rt.partition.GetStart().AsTime().UnixMilli(), - rt.partition.GetEnd().AsTime().UnixMilli(), - strings.Join(rt.partition.GetKeys(), delimiter)) + rt.keyedWindow.GetStart().AsTime().UnixMilli(), + rt.keyedWindow.GetEnd().AsTime().UnixMilli(), + strings.Join(rt.keyedWindow.GetKeys(), delimiter)) } // sessionReduceTaskManager manages the reduce tasks for a session reduce operation. @@ -82,13 +82,13 @@ func newReduceTaskManager(sessionReducerFactory CreateSessionReducer) *sessionRe func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest) error { rtm.rw.Lock() - // for create operation, there should be exactly one partition - if len(request.Operation.Partitions) != 1 { - return fmt.Errorf("create operation error: invalid number of partitions in the request - %d", len(request.Operation.Partitions)) + // for create operation, there should be exactly one keyedWindow + if len(request.Operation.KeyedWindows) != 1 { + return fmt.Errorf("create operation error: invalid number of windows in the request - %d", len(request.Operation.KeyedWindows)) } task := &sessionReduceTask{ - partition: request.Operation.Partitions[0], + keyedWindow: request.Operation.KeyedWindows[0], sessionReducer: rtm.sessionReducerFactory(), inputCh: make(chan Datum), outputCh: make(chan Message), @@ -118,7 +118,7 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 rtm.responseCh <- task.buildEOFResponse() }() - task.sessionReducer.SessionReduce(ctx, task.partition.GetKeys(), task.inputCh, task.outputCh) + task.sessionReducer.SessionReduce(ctx, task.keyedWindow.GetKeys(), task.inputCh, task.outputCh) // close the output channel and wait for the response to be forwarded close(task.outputCh) wg.Wait() @@ -142,13 +142,13 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 // If the reduce task is not found, it will create a new reduce task and start the reduce operation. func (rtm *sessionReduceTaskManager) AppendToTask(ctx context.Context, request *v1.SessionReduceRequest) error { - // for append operation, there should be exactly one partition - if len(request.Operation.Partitions) != 1 { - return fmt.Errorf("append operation error: invalid number of partitions in the request - %d", len(request.Operation.Partitions)) + // for append operation, there should be exactly one keyedWindow + if len(request.Operation.KeyedWindows) != 1 { + return fmt.Errorf("append operation error: invalid number of windows in the request - %d", len(request.Operation.KeyedWindows)) } rtm.rw.RLock() - task, ok := rtm.tasks[generateKey(request.Operation.Partitions[0])] + task, ok := rtm.tasks[generateKey(request.Operation.KeyedWindows[0])] rtm.rw.RUnlock() // if the task is not found, create a new task and start the reduce operation @@ -166,9 +166,9 @@ func (rtm *sessionReduceTaskManager) AppendToTask(ctx context.Context, request * // CloseTask closes the input channel of the reduce tasks. func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { rtm.rw.RLock() - tasksToBeClosed := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) - for _, partition := range request.Operation.Partitions { - key := generateKey(partition) + tasksToBeClosed := make([]*sessionReduceTask, 0, len(request.Operation.KeyedWindows)) + for _, window := range request.Operation.KeyedWindows { + key := generateKey(window) task, ok := rtm.tasks[key] if ok { tasksToBeClosed = append(tasksToBeClosed, task) @@ -185,13 +185,13 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) // merges the accumulators from the other tasks. func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { rtm.rw.Lock() - mergedPartition := request.Operation.Partitions[0] + mergedWindow := request.Operation.KeyedWindows[0] - tasks := make([]*sessionReduceTask, 0, len(request.Operation.Partitions)) + tasks := make([]*sessionReduceTask, 0, len(request.Operation.KeyedWindows)) // merge the aggregators from the other tasks - for _, partition := range request.Operation.Partitions { - key := generateKey(partition) + for _, window := range request.Operation.KeyedWindows { + key := generateKey(window) task, ok := rtm.tasks[key] if !ok { rtm.rw.Unlock() @@ -200,13 +200,13 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 task.merged.Store(true) tasks = append(tasks, task) - // mergedPartition will be the smallest window which contains all the windows - if partition.GetStart().AsTime().Before(mergedPartition.GetStart().AsTime()) { - mergedPartition.Start = partition.Start + // mergedWindow will be the largest window which contains all the windows + if window.GetStart().AsTime().Before(mergedWindow.GetStart().AsTime()) { + mergedWindow.Start = window.Start } - if partition.GetEnd().AsTime().After(mergedPartition.GetEnd().AsTime()) { - mergedPartition.End = partition.End + if window.GetEnd().AsTime().After(mergedWindow.GetEnd().AsTime()) { + mergedWindow.End = window.End } } @@ -221,21 +221,21 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 accumulators = append(accumulators, task.sessionReducer.Accumulator(ctx)) } - // create a new task with the merged partition + // create a new task with the merged keyedWindow err := rtm.CreateTask(ctx, &v1.SessionReduceRequest{ Payload: nil, Operation: &v1.SessionReduceRequest_WindowOperation{ - Event: v1.SessionReduceRequest_WindowOperation_OPEN, - Partitions: []*v1.Partition{mergedPartition}, + Event: v1.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*v1.KeyedWindow{mergedWindow}, }, }) if err != nil { return err } - mergedTask, ok := rtm.tasks[generateKey(mergedPartition)] + mergedTask, ok := rtm.tasks[generateKey(mergedWindow)] if !ok { - return fmt.Errorf("merge operation error: merged task not found for key %s", mergedPartition.String()) + return fmt.Errorf("merge operation error: merged task not found for key %s", mergedWindow.String()) } // merge the accumulators using the merged task for _, aggregator := range accumulators { @@ -245,26 +245,26 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 return nil } -// ExpandTask expands the reduce task. It will update the partition of the reduce task -// expects request.Operation.Partitions to have exactly two partitions. The first partition is the old partition and the second -// partition is the new partition. +// ExpandTask expands the reduce task. It will update the keyedWindow of the reduce task +// expects request.Operation.KeyedWindows to have exactly two windows. The first is the old window and the second +// is the new window. func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { - // for expand operation, there should be exactly two partitions - if len(request.Operation.Partitions) != 2 { - return fmt.Errorf("expand operation error: expected exactly two partitions") + // for expand operation, there should be exactly two windows + if len(request.Operation.KeyedWindows) != 2 { + return fmt.Errorf("expand operation error: expected exactly two windows") } rtm.rw.Lock() - key := generateKey(request.Operation.Partitions[0]) + key := generateKey(request.Operation.KeyedWindows[0]) task, ok := rtm.tasks[key] if !ok { rtm.rw.Unlock() return fmt.Errorf("expand operation error: task not found for key - %s", key) } - // assign the new partition to the task - task.partition = request.Operation.Partitions[1] + // assign the new keyedWindow to the task + task.keyedWindow = request.Operation.KeyedWindows[1] // delete the old entry from the tasks map and add the new entry delete(rtm.tasks, key) @@ -314,11 +314,11 @@ func (rtm *sessionReduceTaskManager) CloseAll() { } } -func generateKey(partition *v1.Partition) string { +func generateKey(keyedWindows *v1.KeyedWindow) string { return fmt.Sprintf("%d:%d:%s", - partition.GetStart().AsTime().UnixMilli(), - partition.GetEnd().AsTime().UnixMilli(), - strings.Join(partition.GetKeys(), delimiter)) + keyedWindows.GetStart().AsTime().UnixMilli(), + keyedWindows.GetEnd().AsTime().UnixMilli(), + strings.Join(keyedWindows.GetKeys(), delimiter)) } func buildDatum(payload *v1.SessionReduceRequest_Payload) Datum { From d92882380acbd2f71693cdfb809b158b592cc77d Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Mon, 27 Nov 2023 16:55:29 +0530 Subject: [PATCH 13/27] tests for session reduce Signed-off-by: Yashash H L --- pkg/sessionreducer/service_test.go | 817 +++++++++++++++++++++++++++++ pkg/sessionreducer/task_manager.go | 8 +- 2 files changed, 824 insertions(+), 1 deletion(-) create mode 100644 pkg/sessionreducer/service_test.go diff --git a/pkg/sessionreducer/service_test.go b/pkg/sessionreducer/service_test.go new file mode 100644 index 00000000..14328cf2 --- /dev/null +++ b/pkg/sessionreducer/service_test.go @@ -0,0 +1,817 @@ +package sessionreducer + +import ( + "context" + "io" + "log" + "reflect" + "sort" + "strconv" + "sync" + "testing" + "time" + + "go.uber.org/atomic" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" + + sessionreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" +) + +type SessionReduceFnServerTest struct { + ctx context.Context + inputCh chan *sessionreducepb.SessionReduceRequest + outputCh chan *sessionreducepb.SessionReduceResponse + grpc.ServerStream +} + +func NewReduceFnServerTest(ctx context.Context, + inputCh chan *sessionreducepb.SessionReduceRequest, + outputCh chan *sessionreducepb.SessionReduceResponse) *SessionReduceFnServerTest { + return &SessionReduceFnServerTest{ + ctx: ctx, + inputCh: inputCh, + outputCh: outputCh, + } +} + +func (u *SessionReduceFnServerTest) Send(list *sessionreducepb.SessionReduceResponse) error { + u.outputCh <- list + return nil +} + +func (u *SessionReduceFnServerTest) Recv() (*sessionreducepb.SessionReduceRequest, error) { + val, ok := <-u.inputCh + if !ok { + return val, io.EOF + } + return val, nil +} + +func (u *SessionReduceFnServerTest) Context() context.Context { + return u.ctx +} + +type SessionSum struct { + sum *atomic.Int32 +} + +func (s *SessionSum) SessionReduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message) { + for val := range inputCh { + msgVal, _ := strconv.Atoi(string(val.Value())) + s.sum.Add(int32(msgVal)) + } + outputCh <- NewMessage([]byte(strconv.Itoa(int(s.sum.Load())))).WithKeys([]string{keys[0] + "_test"}) +} + +func (s *SessionSum) Accumulator(ctx context.Context) []byte { + return []byte(strconv.Itoa(int(s.sum.Load()))) +} + +func (s *SessionSum) MergeAccumulator(ctx context.Context, accumulator []byte) { + val, err := strconv.Atoi(string(accumulator)) + if err != nil { + log.Println("unable to convert the accumulator value to int: ", err.Error()) + return + } + s.sum.Add(int32(val)) +} + +func NewSessionSum() SessionReducer { + return &SessionSum{ + sum: atomic.NewInt32(0), + } +} + +func TestService_SessionReduceFn(t *testing.T) { + + tests := []struct { + name string + handler CreateSessionReducer + input []*sessionreducepb.SessionReduceRequest + expected []*sessionreducepb.SessionReduceResponse + expectedErr bool + }{ + { + name: "open_append_close", + handler: NewSessionSum, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "open_expand_close", + handler: NewSessionSum, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(75000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(79000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(75000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(79000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.UnixMilli(74999)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(75000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + EOF: false, + }, + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), + EventTime: timestamppb.New(time.UnixMilli(78999)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(79000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "open_merge_close", + handler: NewSessionSum, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.UnixMilli(84999)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + EOF: false, + }, + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), + EventTime: timestamppb.New(time.UnixMilli(87999)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "open_expand_append_merge_close", + handler: NewSessionSum, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(40)), + EventTime: timestamppb.New(time.UnixMilli(94999)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + EOF: false, + }, + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(80)), + EventTime: timestamppb.New(time.UnixMilli(97999)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := &Service{ + createSessionReducer: tt.handler, + } + // here's a trick for testing: + // because we are not using gRPC, we directly set a new incoming ctx + // instead of the regular outgoing context in the real gRPC connection. + + inputCh := make(chan *sessionreducepb.SessionReduceRequest) + outputCh := make(chan *sessionreducepb.SessionReduceResponse) + result := make([]*sessionreducepb.SessionReduceResponse, 0) + + udfReduceFnStream := NewReduceFnServerTest(context.Background(), inputCh, outputCh) + + var wg sync.WaitGroup + var err error + + wg.Add(1) + go func() { + defer wg.Done() + err = fs.SessionReduceFn(udfReduceFnStream) + close(outputCh) + }() + + wg.Add(1) + go func() { + defer wg.Done() + for msg := range outputCh { + if !msg.EOF { + result = append(result, msg) + } + } + }() + + for _, val := range tt.input { + udfReduceFnStream.inputCh <- val + } + close(udfReduceFnStream.inputCh) + wg.Wait() + + if (err != nil) != tt.expectedErr { + t.Errorf("ReduceFn() error = %v, wantErr %v", err, tt.expectedErr) + return + } + + //sort and compare, since order of the output doesn't matter + sort.Slice(result, func(i, j int) bool { + return string(result[i].Result.Value) < string(result[j].Result.Value) + }) + + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("SessionReduceFn() got = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index 2aaf7cd4..a0fb5efa 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -21,6 +21,7 @@ type sessionReduceTask struct { outputCh chan Message doneCh chan struct{} merged *atomic.Bool + closed *atomic.Bool } // buildSessionReduceResponse builds the session reduce response from the messages. @@ -94,6 +95,7 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 outputCh: make(chan Message), doneCh: make(chan struct{}), merged: atomic.NewBool(false), + closed: atomic.NewBool(false), } // add the task to the tasks list @@ -177,6 +179,7 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) rtm.rw.RUnlock() for _, task := range tasksToBeClosed { + task.closed.Store(true) close(task.inputCh) } } @@ -249,7 +252,6 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 // expects request.Operation.KeyedWindows to have exactly two windows. The first is the old window and the second // is the new window. func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { - // for expand operation, there should be exactly two windows if len(request.Operation.KeyedWindows) != 2 { return fmt.Errorf("expand operation error: expected exactly two windows") @@ -310,6 +312,10 @@ func (rtm *sessionReduceTaskManager) CloseAll() { rtm.rw.Unlock() for _, task := range tasks { + if task.closed.Load() || task.merged.Load() { + continue + } + task.closed.Store(true) close(task.inputCh) } } From dd8004948c8cea3bf17d11159466f27e2b425c3d Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Thu, 30 Nov 2023 21:31:56 +0530 Subject: [PATCH 14/27] avoid sending eof for merged windows Signed-off-by: Yashash H L --- pkg/sessionreducer/task_manager.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index a0fb5efa..bfd3b279 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -116,8 +116,10 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 rtm.responseCh <- task.buildSessionReduceResponse(message) } } - // send EOF - rtm.responseCh <- task.buildEOFResponse() + if !task.merged.Load() { + // send EOF + rtm.responseCh <- task.buildEOFResponse() + } }() task.sessionReducer.SessionReduce(ctx, task.keyedWindow.GetKeys(), task.inputCh, task.outputCh) @@ -236,7 +238,9 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 return err } + rtm.rw.RLock() mergedTask, ok := rtm.tasks[generateKey(mergedWindow)] + rtm.rw.RUnlock() if !ok { return fmt.Errorf("merge operation error: merged task not found for key %s", mergedWindow.String()) } From f833d63f4238a2c25c9e877f51de470eabd66a85 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Wed, 6 Dec 2023 16:33:23 +0530 Subject: [PATCH 15/27] updating images Signed-off-by: Yashash H L --- pkg/reducer/examples/counter/Makefile | 2 +- pkg/reducer/examples/sum/Makefile | 2 +- pkg/reducer/examples/sum/go.mod | 2 +- pkg/reducer/examples/sum/go.sum | 2 ++ pkg/sessionreducer/examples/counter/Makefile | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/reducer/examples/counter/Makefile b/pkg/reducer/examples/counter/Makefile index 072cd6ab..a9ccc9d8 100644 --- a/pkg/reducer/examples/counter/Makefile +++ b/pkg/reducer/examples/counter/Makefile @@ -4,7 +4,7 @@ build: .PHONY: image image: build - docker build -t "quay.io/yhl25/numaflow-go/reduce-counter:v0.5.4" --target counter . + docker build -t "quay.io/numaio/numaflow-go/reduce-counter:v0.5.3" --target counter . clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/sum/Makefile b/pkg/reducer/examples/sum/Makefile index dc21aa79..9b5c4e7b 100644 --- a/pkg/reducer/examples/sum/Makefile +++ b/pkg/reducer/examples/sum/Makefile @@ -4,7 +4,7 @@ build: .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-sum:v0.5.0" --target sum . + docker build -t "quay.io/numaio/numaflow-go/reduce-sum:v0.5.3" --target sum . clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/sum/go.mod b/pkg/reducer/examples/sum/go.mod index d85516be..aa1a1dcb 100644 --- a/pkg/reducer/examples/sum/go.mod +++ b/pkg/reducer/examples/sum/go.mod @@ -2,7 +2,7 @@ module even_odd go 1.20 -require github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd +require github.com/numaproj/numaflow-go v0.5.3-0.20231130160156-dd8004948c8c require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducer/examples/sum/go.sum b/pkg/reducer/examples/sum/go.sum index 9e69119c..94c5c6ce 100644 --- a/pkg/reducer/examples/sum/go.sum +++ b/pkg/reducer/examples/sum/go.sum @@ -8,6 +8,8 @@ github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe h1:nK/BGffg github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe/go.mod h1:zcJq1YAA/jnxCQLW7EFK4+HXWCd2QtW4tMOvRjHFa2g= github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd h1:NuigpRGq/KpLxXhpw52hNamn80iSjuctQGMHJOqqOQM= github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/numaproj/numaflow-go v0.5.3-0.20231130160156-dd8004948c8c h1:RFA+jBXC7Ert+vXplpI84PQynLKsstMNah9fCSHoTX8= +github.com/numaproj/numaflow-go v0.5.3-0.20231130160156-dd8004948c8c/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/sessionreducer/examples/counter/Makefile b/pkg/sessionreducer/examples/counter/Makefile index aabb2a9f..cb151461 100644 --- a/pkg/sessionreducer/examples/counter/Makefile +++ b/pkg/sessionreducer/examples/counter/Makefile @@ -4,7 +4,7 @@ build: .PHONY: image image: build - docker build -t "quay.io/yhl25/numaflow-go/session-counter:v0.5.4" --target counter . + docker build -t "quay.io/numaio/numaflow-go/session-counter:v0.5.3" --target counter . clean: -rm -rf ./dist From 2dafecd7e29574d607b1e15cee42936a510352b3 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Wed, 6 Dec 2023 20:15:31 +0530 Subject: [PATCH 16/27] avoid breaking change, by introducing reduceStreamer Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.pb.go | 2 +- .../globalreduce/v1/globalreduce_grpc.pb.go | 2 +- pkg/apis/proto/map/v1/map.pb.go | 2 +- pkg/apis/proto/map/v1/map_grpc.pb.go | 2 +- pkg/apis/proto/mapstream/v1/mapstream.pb.go | 2 +- .../proto/mapstream/v1/mapstream_grpc.pb.go | 2 +- pkg/apis/proto/reduce/v1/reduce.pb.go | 2 +- pkg/apis/proto/reduce/v1/reduce_grpc.pb.go | 2 +- .../sessionreduce/v1/sessionreduce.pb.go | 2 +- .../sessionreduce/v1/sessionreduce_grpc.pb.go | 2 +- pkg/apis/proto/sideinput/v1/sideinput.pb.go | 2 +- .../proto/sideinput/v1/sideinput_grpc.pb.go | 2 +- pkg/apis/proto/sink/v1/sink.pb.go | 2 +- pkg/apis/proto/sink/v1/sink_grpc.pb.go | 2 +- pkg/apis/proto/source/v1/source.pb.go | 2 +- pkg/apis/proto/source/v1/source_grpc.pb.go | 2 +- .../proto/sourcetransform/v1/transform.pb.go | 2 +- .../sourcetransform/v1/transform_grpc.pb.go | 2 +- pkg/reduceStreamer/doc.go | 5 + .../examples/counter/Dockerfile | 20 + pkg/reduceStreamer/examples/counter/Makefile | 10 + pkg/reduceStreamer/examples/counter/README.md | 3 + pkg/reduceStreamer/examples/counter/main.go | 29 + pkg/reduceStreamer/examples/sum/Dockerfile | 20 + pkg/reduceStreamer/examples/sum/Makefile | 10 + pkg/reduceStreamer/examples/sum/README.md | 3 + pkg/reduceStreamer/examples/sum/main.go | 53 ++ pkg/reduceStreamer/interface.go | 37 ++ pkg/reduceStreamer/message.go | 52 ++ pkg/reduceStreamer/options.go | 43 ++ pkg/reduceStreamer/options_test.go | 18 + pkg/reduceStreamer/server.go | 56 ++ pkg/reduceStreamer/server_test.go | 37 ++ pkg/reduceStreamer/service.go | 101 ++++ pkg/reduceStreamer/service_test.go | 531 ++++++++++++++++++ pkg/reduceStreamer/task_manager.go | 193 +++++++ pkg/reduceStreamer/types.go | 65 +++ pkg/reducer/examples/counter/main.go | 13 +- pkg/reducer/examples/sum/go.mod | 16 - pkg/reducer/examples/sum/go.sum | 32 -- pkg/reducer/examples/sum/main.go | 16 +- pkg/reducer/interface.go | 8 +- pkg/reducer/message.go | 20 +- pkg/reducer/server_test.go | 4 +- pkg/reducer/service_test.go | 16 +- pkg/reducer/task_manager.go | 43 +- 46 files changed, 1375 insertions(+), 115 deletions(-) create mode 100644 pkg/reduceStreamer/doc.go create mode 100644 pkg/reduceStreamer/examples/counter/Dockerfile create mode 100644 pkg/reduceStreamer/examples/counter/Makefile create mode 100644 pkg/reduceStreamer/examples/counter/README.md create mode 100644 pkg/reduceStreamer/examples/counter/main.go create mode 100644 pkg/reduceStreamer/examples/sum/Dockerfile create mode 100644 pkg/reduceStreamer/examples/sum/Makefile create mode 100644 pkg/reduceStreamer/examples/sum/README.md create mode 100644 pkg/reduceStreamer/examples/sum/main.go create mode 100644 pkg/reduceStreamer/interface.go create mode 100644 pkg/reduceStreamer/message.go create mode 100644 pkg/reduceStreamer/options.go create mode 100644 pkg/reduceStreamer/options_test.go create mode 100644 pkg/reduceStreamer/server.go create mode 100644 pkg/reduceStreamer/server_test.go create mode 100644 pkg/reduceStreamer/service.go create mode 100644 pkg/reduceStreamer/service_test.go create mode 100644 pkg/reduceStreamer/task_manager.go create mode 100644 pkg/reduceStreamer/types.go delete mode 100644 pkg/reducer/examples/sum/go.mod delete mode 100644 pkg/reducer/examples/sum/go.sum diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go index 1ef68d13..38770574 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go +++ b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/globalreduce/v1/globalreduce.proto package v1 diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go index 67ce9cb8..6b1dc292 100644 --- a/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go +++ b/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/globalreduce/v1/globalreduce.proto package v1 diff --git a/pkg/apis/proto/map/v1/map.pb.go b/pkg/apis/proto/map/v1/map.pb.go index 03bc0967..29d0b37d 100644 --- a/pkg/apis/proto/map/v1/map.pb.go +++ b/pkg/apis/proto/map/v1/map.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/map/v1/map.proto package v1 diff --git a/pkg/apis/proto/map/v1/map_grpc.pb.go b/pkg/apis/proto/map/v1/map_grpc.pb.go index 39b713bb..d3844348 100644 --- a/pkg/apis/proto/map/v1/map_grpc.pb.go +++ b/pkg/apis/proto/map/v1/map_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/map/v1/map.proto package v1 diff --git a/pkg/apis/proto/mapstream/v1/mapstream.pb.go b/pkg/apis/proto/mapstream/v1/mapstream.pb.go index 889b7648..23112488 100644 --- a/pkg/apis/proto/mapstream/v1/mapstream.pb.go +++ b/pkg/apis/proto/mapstream/v1/mapstream.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/mapstream/v1/mapstream.proto package v1 diff --git a/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go b/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go index 1ad13424..e8927dc5 100644 --- a/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go +++ b/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/mapstream/v1/mapstream.proto package v1 diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index ff5f7255..6455b93a 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/reduce/v1/reduce.proto package v1 diff --git a/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go b/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go index e93cca9c..afe493d1 100644 --- a/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/reduce/v1/reduce.proto package v1 diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index 7c7a24fb..85b185a2 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto package v1 diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go index c56ea476..51b7bd6c 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto package v1 diff --git a/pkg/apis/proto/sideinput/v1/sideinput.pb.go b/pkg/apis/proto/sideinput/v1/sideinput.pb.go index 95a32865..5fd5b7cd 100644 --- a/pkg/apis/proto/sideinput/v1/sideinput.pb.go +++ b/pkg/apis/proto/sideinput/v1/sideinput.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/sideinput/v1/sideinput.proto package v1 diff --git a/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go b/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go index 3f966308..2562314d 100644 --- a/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go +++ b/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/sideinput/v1/sideinput.proto package v1 diff --git a/pkg/apis/proto/sink/v1/sink.pb.go b/pkg/apis/proto/sink/v1/sink.pb.go index e746cedd..3da1634b 100644 --- a/pkg/apis/proto/sink/v1/sink.pb.go +++ b/pkg/apis/proto/sink/v1/sink.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/sink/v1/sink.proto package v1 diff --git a/pkg/apis/proto/sink/v1/sink_grpc.pb.go b/pkg/apis/proto/sink/v1/sink_grpc.pb.go index eddd9ef8..bc50da8e 100644 --- a/pkg/apis/proto/sink/v1/sink_grpc.pb.go +++ b/pkg/apis/proto/sink/v1/sink_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/sink/v1/sink.proto package v1 diff --git a/pkg/apis/proto/source/v1/source.pb.go b/pkg/apis/proto/source/v1/source.pb.go index e13121b2..749ec9e4 100644 --- a/pkg/apis/proto/source/v1/source.pb.go +++ b/pkg/apis/proto/source/v1/source.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/source/v1/source.proto package v1 diff --git a/pkg/apis/proto/source/v1/source_grpc.pb.go b/pkg/apis/proto/source/v1/source_grpc.pb.go index 1dce880a..9f1d5282 100644 --- a/pkg/apis/proto/source/v1/source_grpc.pb.go +++ b/pkg/apis/proto/source/v1/source_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/source/v1/source.proto package v1 diff --git a/pkg/apis/proto/sourcetransform/v1/transform.pb.go b/pkg/apis/proto/sourcetransform/v1/transform.pb.go index e38b6247..6aff9fb6 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform.pb.go +++ b/pkg/apis/proto/sourcetransform/v1/transform.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v3.21.9 +// protoc v4.25.1 // source: pkg/apis/proto/sourcetransform/v1/transform.proto package v1 diff --git a/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go b/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go index 2bd5561b..a3f4c206 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go +++ b/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.9 +// - protoc v4.25.1 // source: pkg/apis/proto/sourcetransform/v1/transform.proto package v1 diff --git a/pkg/reduceStreamer/doc.go b/pkg/reduceStreamer/doc.go new file mode 100644 index 00000000..dde78fc0 --- /dev/null +++ b/pkg/reduceStreamer/doc.go @@ -0,0 +1,5 @@ +// Package reduceStreamer implements the server code for reduce operation. + +// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/ + +package reduceStreamer diff --git a/pkg/reduceStreamer/examples/counter/Dockerfile b/pkg/reduceStreamer/examples/counter/Dockerfile new file mode 100644 index 00000000..7f1e3e5c --- /dev/null +++ b/pkg/reduceStreamer/examples/counter/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/counter-example /bin/counter-example +RUN chmod +x /bin/counter-example + +#################################################################################################### +# counter +#################################################################################################### +FROM scratch as counter +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/counter-example /bin/counter-example +ENTRYPOINT [ "/bin/counter-example" ] diff --git a/pkg/reduceStreamer/examples/counter/Makefile b/pkg/reduceStreamer/examples/counter/Makefile new file mode 100644 index 00000000..a9ccc9d8 --- /dev/null +++ b/pkg/reduceStreamer/examples/counter/Makefile @@ -0,0 +1,10 @@ +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-counter:v0.5.3" --target counter . + +clean: + -rm -rf ./dist diff --git a/pkg/reduceStreamer/examples/counter/README.md b/pkg/reduceStreamer/examples/counter/README.md new file mode 100644 index 00000000..5a72f269 --- /dev/null +++ b/pkg/reduceStreamer/examples/counter/README.md @@ -0,0 +1,3 @@ +# Counter + +An example User Defined Function that counts the number of events. diff --git a/pkg/reduceStreamer/examples/counter/main.go b/pkg/reduceStreamer/examples/counter/main.go new file mode 100644 index 00000000..3abade1b --- /dev/null +++ b/pkg/reduceStreamer/examples/counter/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + "strconv" + + "github.com/numaproj/numaflow-go/pkg/reduceStreamer" +) + +func reduceCounter(_ context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { + // count the incoming events + var resultKeys = keys + var resultVal []byte + var counter = 0 + for range inputCh { + counter++ + if counter >= 10 { + resultVal = []byte(strconv.Itoa(counter)) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + counter = 0 + } + } + resultVal = []byte(strconv.Itoa(counter)) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) +} + +func main() { + reduceStreamer.NewServer(reduceStreamer.ReducerFunc(reduceCounter)).Start(context.Background()) +} diff --git a/pkg/reduceStreamer/examples/sum/Dockerfile b/pkg/reduceStreamer/examples/sum/Dockerfile new file mode 100644 index 00000000..4b237f86 --- /dev/null +++ b/pkg/reduceStreamer/examples/sum/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/sum-example /bin/sum-example +RUN chmod +x /bin/sum-example + +#################################################################################################### +# sum +#################################################################################################### +FROM scratch as sum +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/sum-example /bin/sum-example +ENTRYPOINT [ "/bin/sum-example" ] diff --git a/pkg/reduceStreamer/examples/sum/Makefile b/pkg/reduceStreamer/examples/sum/Makefile new file mode 100644 index 00000000..9b5c4e7b --- /dev/null +++ b/pkg/reduceStreamer/examples/sum/Makefile @@ -0,0 +1,10 @@ +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-sum:v0.5.3" --target sum . + +clean: + -rm -rf ./dist diff --git a/pkg/reduceStreamer/examples/sum/README.md b/pkg/reduceStreamer/examples/sum/README.md new file mode 100644 index 00000000..62af7a17 --- /dev/null +++ b/pkg/reduceStreamer/examples/sum/README.md @@ -0,0 +1,3 @@ +# Sum + +This is a User Defined Function example which receives a stream of numbers and returns the sum of the numbers. diff --git a/pkg/reduceStreamer/examples/sum/main.go b/pkg/reduceStreamer/examples/sum/main.go new file mode 100644 index 00000000..34e20b5d --- /dev/null +++ b/pkg/reduceStreamer/examples/sum/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "fmt" + "log" + "strconv" + + "github.com/numaproj/numaflow-go/pkg/reduceStreamer" +) + +// Sum is a reduceStreamer that sum up the values for the given keys +type Sum struct { +} + +func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { + // sum up values for the same keys + intervalWindow := md.IntervalWindow() + _ = intervalWindow + var resultKeys = keys + var resultVal []byte + var sum = 0 + // sum up the values + for d := range inputCh { + val := d.Value() + eventTime := d.EventTime() + _ = eventTime + watermark := d.Watermark() + _ = watermark + + v, err := strconv.Atoi(string(val)) + if err != nil { + fmt.Printf("unable to convert the value to int: %v\n", err) + continue + } + sum += v + + if sum >= 100 { + resultVal = []byte(strconv.Itoa(sum)) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + sum = 0 + } + } + resultVal = []byte(strconv.Itoa(sum)) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) +} + +func main() { + err := reduceStreamer.NewServer(&Sum{}).Start(context.Background()) + if err != nil { + log.Panic("unable to start the server due to: ", err) + } +} diff --git a/pkg/reduceStreamer/interface.go b/pkg/reduceStreamer/interface.go new file mode 100644 index 00000000..d5d8dac9 --- /dev/null +++ b/pkg/reduceStreamer/interface.go @@ -0,0 +1,37 @@ +package reduceStreamer + +import ( + "context" + "time" +) + +// Datum contains methods to get the payload information. +type Datum interface { + Value() []byte + EventTime() time.Time + Watermark() time.Time +} + +// Metadata contains methods to get the metadata for the reduce operation. +type Metadata interface { + IntervalWindow() IntervalWindow +} + +// IntervalWindow contains methods to get the information for a given interval window. +type IntervalWindow interface { + StartTime() time.Time + EndTime() time.Time +} + +// ReduceStreamer is the interface of reduceStream function implementation. +type ReduceStreamer interface { + ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) +} + +// ReducerFunc is a utility type used to convert a ReduceStream function to a ReduceStreamer. +type ReducerFunc func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) + +// ReduceStream implements the function of reduce function. +func (rf ReducerFunc) ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { + rf(ctx, keys, inputCh, outputCh, md) +} diff --git a/pkg/reduceStreamer/message.go b/pkg/reduceStreamer/message.go new file mode 100644 index 00000000..766f7a09 --- /dev/null +++ b/pkg/reduceStreamer/message.go @@ -0,0 +1,52 @@ +package reduceStreamer + +import "fmt" + +var ( + DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ +) + +// Message is used to wrap the data return by reduceStream function +type Message struct { + value []byte + keys []string + tags []string +} + +// NewMessage creates a Message with value +func NewMessage(value []byte) Message { + return Message{value: value} +} + +// MessageToDrop creates a Message to be dropped +func MessageToDrop() Message { + return Message{value: []byte{}, tags: []string{DROP}} +} + +// WithKeys is used to assign the keys to the message +func (m Message) WithKeys(keys []string) Message { + m.keys = keys + return m +} + +// WithTags is used to assign the tags to the message +// tags will be used for conditional forwarding +func (m Message) WithTags(tags []string) Message { + m.tags = tags + return m +} + +// Keys returns message keys +func (m Message) Keys() []string { + return m.keys +} + +// Value returns message value +func (m Message) Value() []byte { + return m.value +} + +// Tags returns message tags +func (m Message) Tags() []string { + return m.tags +} diff --git a/pkg/reduceStreamer/options.go b/pkg/reduceStreamer/options.go new file mode 100644 index 00000000..2158a39e --- /dev/null +++ b/pkg/reduceStreamer/options.go @@ -0,0 +1,43 @@ +package reduceStreamer + +import ( + "github.com/numaproj/numaflow-go/pkg/info" +) + +type options struct { + sockAddr string + maxMessageSize int + serverInfoFilePath string +} + +// Option is the interface to apply options. +type Option func(*options) + +func DefaultOptions() *options { + return &options{ + sockAddr: address, + maxMessageSize: defaultMaxMessageSize, + serverInfoFilePath: info.ServerInfoFilePath, + } +} + +// WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. +func WithMaxMessageSize(size int) Option { + return func(opts *options) { + opts.maxMessageSize = size + } +} + +// WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. +func WithSockAddr(addr string) Option { + return func(opts *options) { + opts.sockAddr = addr + } +} + +// WithServerInfoFilePath sets the server info file path to the given path. +func WithServerInfoFilePath(f string) Option { + return func(opts *options) { + opts.serverInfoFilePath = f + } +} diff --git a/pkg/reduceStreamer/options_test.go b/pkg/reduceStreamer/options_test.go new file mode 100644 index 00000000..e3eda2df --- /dev/null +++ b/pkg/reduceStreamer/options_test.go @@ -0,0 +1,18 @@ +package reduceStreamer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithMaxMessageSize(t *testing.T) { + var ( + size = 1024 * 1024 * 10 + opts = &options{ + maxMessageSize: defaultMaxMessageSize, + } + ) + WithMaxMessageSize(1024 * 1024 * 10)(opts) + assert.Equal(t, size, opts.maxMessageSize) +} diff --git a/pkg/reduceStreamer/server.go b/pkg/reduceStreamer/server.go new file mode 100644 index 00000000..0a1e3a8c --- /dev/null +++ b/pkg/reduceStreamer/server.go @@ -0,0 +1,56 @@ +package reduceStreamer + +import ( + "context" + "fmt" + "os/signal" + "syscall" + + "github.com/numaproj/numaflow-go/pkg" + reducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" + "github.com/numaproj/numaflow-go/pkg/shared" +) + +// server is a reduce gRPC server. +type server struct { + svc *Service + opts *options +} + +// NewServer creates a new reduceStream server. +func NewServer(r ReduceStreamer, inputOptions ...Option) numaflow.Server { + opts := DefaultOptions() + for _, inputOption := range inputOptions { + inputOption(opts) + } + s := new(server) + s.svc = new(Service) + s.svc.reducerHandle = r + s.opts = opts + return s +} + +// Start starts the reduce gRPC server. +func (r *server) Start(ctx context.Context) error { + ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // write server info to the file + // start listening on unix domain socket + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath) + if err != nil { + return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) + } + // close the listener + defer func() { _ = lis.Close() }() + + // create a grpc server + grpcServer := shared.CreateGRPCServer(r.opts.maxMessageSize) + defer grpcServer.GracefulStop() + + // register the reduce service + reducepb.RegisterReduceServer(grpcServer, r.svc) + + // start the grpc server + return shared.StartGRPCServer(ctxWithSignal, grpcServer, lis) +} diff --git a/pkg/reduceStreamer/server_test.go b/pkg/reduceStreamer/server_test.go new file mode 100644 index 00000000..497c86ac --- /dev/null +++ b/pkg/reduceStreamer/server_test.go @@ -0,0 +1,37 @@ +package reduceStreamer + +import ( + "context" + "os" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestReduceServer_Start(t *testing.T) { + socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") + defer func() { + _ = os.RemoveAll(socketFile.Name()) + }() + + serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") + defer func() { + _ = os.RemoveAll(serverInfoFile.Name()) + }() + + var reduceHandler = ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))) + }) + // note: using actual uds connection + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + err := NewServer(reduceHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) + assert.NoError(t, err) +} diff --git a/pkg/reduceStreamer/service.go b/pkg/reduceStreamer/service.go new file mode 100644 index 00000000..252aab01 --- /dev/null +++ b/pkg/reduceStreamer/service.go @@ -0,0 +1,101 @@ +package reduceStreamer + +import ( + "context" + "io" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + reducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +const ( + uds = "unix" + defaultMaxMessageSize = 1024 * 1024 * 64 + address = "/var/run/numaflow/reducestream.sock" + winStartTime = "x-numaflow-win-start-time" + winEndTime = "x-numaflow-win-end-time" + delimiter = ":" +) + +// Service implements the proto gen server interface and contains the reduce operation handler. +type Service struct { + reducepb.UnimplementedReduceServer + reducerHandle ReduceStreamer +} + +// IsReady returns true to indicate the gRPC connection is ready. +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*reducepb.ReadyResponse, error) { + return &reducepb.ReadyResponse{Ready: true}, nil +} + +// ReduceFn applies a reduce function to a request stream and returns a list of results. +func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { + var ( + err error + ctx = stream.Context() + g errgroup.Group + ) + + taskManager := newReduceTaskManager(fs.reducerHandle) + + // err group for the go routine which reads from the output channel and sends to the stream + g.Go(func() error { + for output := range taskManager.OutputChannel() { + sendErr := stream.Send(output) + if sendErr != nil { + return sendErr + } + } + return nil + }) + + // read messages from the stream and write the messages to corresponding channels + // if the channel is not created, create the channel and invoke the reduceFn + for { + d, recvErr := stream.Recv() + // if EOF, close all the channels + if recvErr == io.EOF { + taskManager.CloseAll() + break + } + if recvErr != nil { + // the error here is returned by stream.Recv() + // it's already a gRPC error + return recvErr + } + + // for fixed and sliding, its just open or append operation + // close signal will be sent to all the reducers when grpc + // input stream gets EOF. + switch d.Operation.Event { + case reducepb.ReduceRequest_WindowOperation_OPEN: + // create a new reduce task and start the reduce operation + err = taskManager.CreateTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case reducepb.ReduceRequest_WindowOperation_APPEND: + // append the datum to the reduce task + err = taskManager.AppendToTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + } + } + + taskManager.WaitAll() + // wait for the go routine which reads from the output channel and sends to the stream to return + err = g.Wait() + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + + return nil +} diff --git a/pkg/reduceStreamer/service_test.go b/pkg/reduceStreamer/service_test.go new file mode 100644 index 00000000..8b73727d --- /dev/null +++ b/pkg/reduceStreamer/service_test.go @@ -0,0 +1,531 @@ +package reduceStreamer + +import ( + "context" + "io" + "reflect" + "sort" + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + grpcmd "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/types/known/timestamppb" + + reducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +type ReduceFnServerTest struct { + ctx context.Context + inputCh chan *reducepb.ReduceRequest + outputCh chan *reducepb.ReduceResponse + grpc.ServerStream +} + +func NewReduceFnServerTest(ctx context.Context, + inputCh chan *reducepb.ReduceRequest, + outputCh chan *reducepb.ReduceResponse) *ReduceFnServerTest { + return &ReduceFnServerTest{ + ctx: ctx, + inputCh: inputCh, + outputCh: outputCh, + } +} + +func (u *ReduceFnServerTest) Send(list *reducepb.ReduceResponse) error { + u.outputCh <- list + return nil +} + +func (u *ReduceFnServerTest) Recv() (*reducepb.ReduceRequest, error) { + val, ok := <-u.inputCh + if !ok { + return val, io.EOF + } + return val, nil +} + +func (u *ReduceFnServerTest) Context() context.Context { + return u.ctx +} + +func TestService_ReduceFn(t *testing.T) { + + tests := []struct { + name string + handler ReduceStreamer + input []*reducepb.ReduceRequest + expected []*reducepb.ReduceResponse + expectedErr bool + }{ + { + name: "reduce_fn_forward_msg_same_keys", + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) + }), + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "reduce_fn_forward_msg_multiple_keys", + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) + }), + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client3_test"}, + Value: []byte(strconv.Itoa(60)), + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "reduce_fn_forward_msg_forward_to_all", + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))) + }), + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Value: []byte(strconv.Itoa(60)), + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + }, + { + name: "reduce_fn_forward_msg_drop_msg", + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- MessageToDrop() + }), + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Tags: []string{DROP}, + Value: []byte{}, + EventTime: timestamppb.New(time.UnixMilli(119999)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + expectedErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := &Service{ + reducerHandle: tt.handler, + } + // here's a trick for testing: + // because we are not using gRPC, we directly set a new incoming ctx + // instead of the regular outgoing context in the real gRPC connection. + ctx := grpcmd.NewIncomingContext(context.Background(), grpcmd.New(map[string]string{winStartTime: "60000", winEndTime: "120000"})) + + inputCh := make(chan *reducepb.ReduceRequest) + outputCh := make(chan *reducepb.ReduceResponse) + result := make([]*reducepb.ReduceResponse, 0) + + udfReduceFnStream := NewReduceFnServerTest(ctx, inputCh, outputCh) + + var wg sync.WaitGroup + var err error + + wg.Add(1) + go func() { + defer wg.Done() + err = fs.ReduceFn(udfReduceFnStream) + close(outputCh) + }() + + wg.Add(1) + go func() { + defer wg.Done() + for msg := range outputCh { + if !msg.EOF { + result = append(result, msg) + } + } + }() + + for _, val := range tt.input { + udfReduceFnStream.inputCh <- val + } + close(udfReduceFnStream.inputCh) + wg.Wait() + + if (err != nil) != tt.expectedErr { + t.Errorf("ReduceFn() error = %v, wantErr %v", err, tt.expectedErr) + return + } + + //sort and compare, since order of the output doesn't matter + sort.Slice(result, func(i, j int) bool { + return string(result[i].Result.Value) < string(result[j].Result.Value) + }) + + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("ReduceFn() got = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/reduceStreamer/task_manager.go b/pkg/reduceStreamer/task_manager.go new file mode 100644 index 00000000..3dca3190 --- /dev/null +++ b/pkg/reduceStreamer/task_manager.go @@ -0,0 +1,193 @@ +package reduceStreamer + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" + + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +// reduceStreamTask represents a task for a performing reduceStream operation. +type reduceStreamTask struct { + keys []string + window *v1.Window + reduceStreamer ReduceStreamer + inputCh chan Datum + outputCh chan Message + doneCh chan struct{} +} + +// buildReduceResponse builds the reduce response from the messages. +func (rt *reduceStreamTask) buildReduceResponse(message Message) *v1.ReduceResponse { + + response := &v1.ReduceResponse{ + Result: &v1.ReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), + }, + Window: rt.window, + } + + return response +} + +func (rt *reduceStreamTask) buildEOFResponse() *v1.ReduceResponse { + response := &v1.ReduceResponse{ + Result: &v1.ReduceResponse_Result{ + EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), + }, + Window: rt.window, + EOF: true, + } + + return response +} + +// uniqueKey returns the unique key for the reduceStream task to be used in the task manager to identify the task. +func (rt *reduceStreamTask) uniqueKey() string { + return fmt.Sprintf("%d:%d:%s", + rt.window.GetStart().AsTime().UnixMilli(), + rt.window.GetEnd().AsTime().UnixMilli(), + strings.Join(rt.keys, delimiter)) +} + +// reduceStreamTaskManager manages the reduceStream tasks. +type reduceStreamTaskManager struct { + reduceStreamer ReduceStreamer + tasks map[string]*reduceStreamTask + responseCh chan *v1.ReduceResponse + rw sync.RWMutex +} + +func newReduceTaskManager(reduceStreamer ReduceStreamer) *reduceStreamTaskManager { + return &reduceStreamTaskManager{ + reduceStreamer: reduceStreamer, + tasks: make(map[string]*reduceStreamTask), + responseCh: make(chan *v1.ReduceResponse), + } +} + +// CreateTask creates a new reduceStream task and starts the reduceStream operation. +func (rtm *reduceStreamTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest) error { + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("create operation error: invalid number of windows") + } + + rtm.rw.Lock() + md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), + request.Operation.Windows[0].GetEnd().AsTime())) + + task := &reduceStreamTask{ + keys: request.GetPayload().GetKeys(), + window: request.Operation.Windows[0], + inputCh: make(chan Datum), + outputCh: make(chan Message), + doneCh: make(chan struct{}), + } + + key := task.uniqueKey() + rtm.tasks[key] = task + + rtm.rw.Unlock() + + go func() { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for message := range task.outputCh { + // write the output to the output channel, service will forward it to downstream + rtm.responseCh <- task.buildReduceResponse(message) + } + // send EOF + rtm.responseCh <- task.buildEOFResponse() + }() + + // invoke the reduceStream function + rtm.reduceStreamer.ReduceStream(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh, md) + // close the output channel after the reduceStream function is done + close(task.outputCh) + // wait for the output to be forwarded + wg.Wait() + // send a done signal + close(task.doneCh) + }() + + // write the first message to the input channel + task.inputCh <- buildDatum(request) + return nil +} + +// AppendToTask writes the message to the reduceStream task. +// If the task is not found, it creates a new task and starts the reduceStream operation. +func (rtm *reduceStreamTaskManager) AppendToTask(ctx context.Context, request *v1.ReduceRequest) error { + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("append operation error: invalid number of windows") + } + + rtm.rw.RLock() + gKey := generateKey(request.Operation.Windows[0], request.Payload.Keys) + task, ok := rtm.tasks[gKey] + rtm.rw.RUnlock() + + // if the task is not found, create a new task + if !ok { + return rtm.CreateTask(ctx, request) + } + + task.inputCh <- buildDatum(request) + return nil +} + +// OutputChannel returns the output channel for the reduceStream task manager to read the results. +func (rtm *reduceStreamTaskManager) OutputChannel() <-chan *v1.ReduceResponse { + return rtm.responseCh +} + +// WaitAll waits for all the reduceStream tasks to complete. +func (rtm *reduceStreamTaskManager) WaitAll() { + rtm.rw.RLock() + tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { + tasks = append(tasks, task) + } + rtm.rw.RUnlock() + + for _, task := range tasks { + <-task.doneCh + } + // after all the tasks are completed, close the output channel + close(rtm.responseCh) +} + +// CloseAll closes all the reduceStream tasks. +func (rtm *reduceStreamTaskManager) CloseAll() { + rtm.rw.Lock() + tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { + tasks = append(tasks, task) + } + rtm.rw.Unlock() + + for _, task := range tasks { + close(task.inputCh) + } +} + +func generateKey(window *v1.Window, keys []string) string { + return fmt.Sprintf("%d:%d:%s", + window.GetStart().AsTime().UnixMilli(), + window.GetEnd().AsTime().UnixMilli(), + strings.Join(keys, delimiter)) +} + +func buildDatum(request *v1.ReduceRequest) Datum { + return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +} diff --git a/pkg/reduceStreamer/types.go b/pkg/reduceStreamer/types.go new file mode 100644 index 00000000..8d2125fb --- /dev/null +++ b/pkg/reduceStreamer/types.go @@ -0,0 +1,65 @@ +package reduceStreamer + +import "time" + +// handlerDatum implements the Datum interface and is used in the reduce functions. +type handlerDatum struct { + value []byte + eventTime time.Time + watermark time.Time +} + +func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time) Datum { + return &handlerDatum{ + value: value, + eventTime: eventTime, + watermark: watermark, + } +} + +func (h *handlerDatum) Value() []byte { + return h.value +} + +func (h *handlerDatum) EventTime() time.Time { + return h.eventTime +} + +func (h *handlerDatum) Watermark() time.Time { + return h.watermark +} + +// intervalWindow implements IntervalWindow interface which will be passed as metadata +// to reduce handlers +type intervalWindow struct { + startTime time.Time + endTime time.Time +} + +func NewIntervalWindow(startTime time.Time, endTime time.Time) IntervalWindow { + return &intervalWindow{ + startTime: startTime, + endTime: endTime, + } +} + +func (i *intervalWindow) StartTime() time.Time { + return i.startTime +} + +func (i *intervalWindow) EndTime() time.Time { + return i.endTime +} + +// metadata implements Metadata interface which will be passed to reduce function. +type metadata struct { + intervalWindow IntervalWindow +} + +func NewMetadata(window IntervalWindow) Metadata { + return &metadata{intervalWindow: window} +} + +func (m *metadata) IntervalWindow() IntervalWindow { + return m.intervalWindow +} diff --git a/pkg/reducer/examples/counter/main.go b/pkg/reducer/examples/counter/main.go index 2bc950c6..3abade1b 100644 --- a/pkg/reducer/examples/counter/main.go +++ b/pkg/reducer/examples/counter/main.go @@ -4,21 +4,26 @@ import ( "context" "strconv" - "github.com/numaproj/numaflow-go/pkg/reducer" + "github.com/numaproj/numaflow-go/pkg/reduceStreamer" ) -func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducer.Datum, outputCh chan<- reducer.Message, md reducer.Metadata) { +func reduceCounter(_ context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { // count the incoming events var resultKeys = keys var resultVal []byte var counter = 0 for range inputCh { counter++ + if counter >= 10 { + resultVal = []byte(strconv.Itoa(counter)) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + counter = 0 + } } resultVal = []byte(strconv.Itoa(counter)) - outputCh <- reducer.NewMessage(resultVal).WithKeys(resultKeys) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) } func main() { - reducer.NewServer(reducer.ReducerFunc(reduceCounter)).Start(context.Background()) + reduceStreamer.NewServer(reduceStreamer.ReducerFunc(reduceCounter)).Start(context.Background()) } diff --git a/pkg/reducer/examples/sum/go.mod b/pkg/reducer/examples/sum/go.mod deleted file mode 100644 index aa1a1dcb..00000000 --- a/pkg/reducer/examples/sum/go.mod +++ /dev/null @@ -1,16 +0,0 @@ -module even_odd - -go 1.20 - -require github.com/numaproj/numaflow-go v0.5.3-0.20231130160156-dd8004948c8c - -require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect -) diff --git a/pkg/reducer/examples/sum/go.sum b/pkg/reducer/examples/sum/go.sum deleted file mode 100644 index 94c5c6ce..00000000 --- a/pkg/reducer/examples/sum/go.sum +++ /dev/null @@ -1,32 +0,0 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe h1:nK/BGffgwQ4L9pyllwzSZttPxMf+OOqK3DOP97KZdRk= -github.com/numaproj/numaflow-go v0.4.6-0.20230828035951-6f79b632ecfe/go.mod h1:zcJq1YAA/jnxCQLW7EFK4+HXWCd2QtW4tMOvRjHFa2g= -github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd h1:NuigpRGq/KpLxXhpw52hNamn80iSjuctQGMHJOqqOQM= -github.com/numaproj/numaflow-go v0.5.3-0.20231123054822-fe61737eb7dd/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= -github.com/numaproj/numaflow-go v0.5.3-0.20231130160156-dd8004948c8c h1:RFA+jBXC7Ert+vXplpI84PQynLKsstMNah9fCSHoTX8= -github.com/numaproj/numaflow-go v0.5.3-0.20231130160156-dd8004948c8c/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/reducer/examples/sum/main.go b/pkg/reducer/examples/sum/main.go index dd0b43f1..34e20b5d 100644 --- a/pkg/reducer/examples/sum/main.go +++ b/pkg/reducer/examples/sum/main.go @@ -6,14 +6,14 @@ import ( "log" "strconv" - "github.com/numaproj/numaflow-go/pkg/reducer" + "github.com/numaproj/numaflow-go/pkg/reduceStreamer" ) -// Sum is a reducer that sum up the values for the given keys +// Sum is a reduceStreamer that sum up the values for the given keys type Sum struct { } -func (s *Sum) Reduce(ctx context.Context, keys []string, inputCh <-chan reducer.Datum, outputCh chan<- reducer.Message, md reducer.Metadata) { +func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { // sum up values for the same keys intervalWindow := md.IntervalWindow() _ = intervalWindow @@ -34,13 +34,19 @@ func (s *Sum) Reduce(ctx context.Context, keys []string, inputCh <-chan reducer. continue } sum += v + + if sum >= 100 { + resultVal = []byte(strconv.Itoa(sum)) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + sum = 0 + } } resultVal = []byte(strconv.Itoa(sum)) - outputCh <- reducer.NewMessage(resultVal).WithKeys(resultKeys) + outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) } func main() { - err := reducer.NewServer(&Sum{}).Start(context.Background()) + err := reduceStreamer.NewServer(&Sum{}).Start(context.Background()) if err != nil { log.Panic("unable to start the server due to: ", err) } diff --git a/pkg/reducer/interface.go b/pkg/reducer/interface.go index 62adc003..4e1e73d1 100644 --- a/pkg/reducer/interface.go +++ b/pkg/reducer/interface.go @@ -25,13 +25,13 @@ type IntervalWindow interface { // Reducer is the interface of reduce function implementation. type Reducer interface { - Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) + Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, md Metadata) Messages } // ReducerFunc is a utility type used to convert a Reduce function to a Reducer. -type ReducerFunc func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) +type ReducerFunc func(ctx context.Context, keys []string, inputCh <-chan Datum, md Metadata) Messages // Reduce implements the function of reduce function. -func (rf ReducerFunc) Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { - rf(ctx, keys, inputCh, outputCh, md) +func (rf ReducerFunc) Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, md Metadata) Messages { + return rf(ctx, keys, inputCh, md) } diff --git a/pkg/reducer/message.go b/pkg/reducer/message.go index 716ac8da..44ec0c2e 100644 --- a/pkg/reducer/message.go +++ b/pkg/reducer/message.go @@ -6,7 +6,7 @@ var ( DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ ) -// Message is used to wrap the data return by reduce functions +// Message is used to wrap the data return by reduceStream function type Message struct { value []byte keys []string @@ -50,3 +50,21 @@ func (m Message) Value() []byte { func (m Message) Tags() []string { return m.tags } + +type Messages []Message + +// MessagesBuilder returns an empty instance of Messages +func MessagesBuilder() Messages { + return Messages{} +} + +// Append appends a Message +func (m Messages) Append(msg Message) Messages { + m = append(m, msg) + return m +} + +// Items returns the message list +func (m Messages) Items() []Message { + return m +} diff --git a/pkg/reducer/server_test.go b/pkg/reducer/server_test.go index 1a043693..abd4fa84 100644 --- a/pkg/reducer/server_test.go +++ b/pkg/reducer/server_test.go @@ -21,13 +21,13 @@ func TestReduceServer_Start(t *testing.T) { _ = os.RemoveAll(serverInfoFile.Name()) }() - var reduceHandler = ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + var reduceHandler = ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - och <- NewMessage([]byte(strconv.Itoa(sum))) + return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum)))) }) // note: using actual uds connection ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index a601c832..c057bea3 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -62,13 +62,13 @@ func TestService_ReduceFn(t *testing.T) { }{ { name: "reduce_fn_forward_msg_same_keys", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) + return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"})) }), input: []*reducepb.ReduceRequest{ { @@ -145,13 +145,13 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_multiple_keys", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) + return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"})) }), input: []*reducepb.ReduceRequest{ { @@ -308,13 +308,13 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_forward_to_all", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - och <- NewMessage([]byte(strconv.Itoa(sum))) + return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum)))) }), input: []*reducepb.ReduceRequest{ { @@ -389,13 +389,13 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_drop_msg", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } - och <- MessageToDrop() + return MessagesBuilder().Append(MessageToDrop()) }), input: []*reducepb.ReduceRequest{ { diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index 3a759998..0a685c3e 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -12,8 +12,8 @@ import ( v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" ) -// reduceTask represents a reduce task for a reduce operation. -type reduceTask struct { +// reduceStreamTask represents a task for a performing reduceStream operation. +type reduceStreamTask struct { keys []string window *v1.Window reducer Reducer @@ -23,7 +23,7 @@ type reduceTask struct { } // buildReduceResponse builds the reduce response from the messages. -func (rt *reduceTask) buildReduceResponse(message Message) *v1.ReduceResponse { +func (rt *reduceStreamTask) buildReduceResponse(message Message) *v1.ReduceResponse { response := &v1.ReduceResponse{ Result: &v1.ReduceResponse_Result{ @@ -38,7 +38,7 @@ func (rt *reduceTask) buildReduceResponse(message Message) *v1.ReduceResponse { return response } -func (rt *reduceTask) buildEOFResponse() *v1.ReduceResponse { +func (rt *reduceStreamTask) buildEOFResponse() *v1.ReduceResponse { response := &v1.ReduceResponse{ Result: &v1.ReduceResponse_Result{ EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), @@ -51,7 +51,7 @@ func (rt *reduceTask) buildEOFResponse() *v1.ReduceResponse { } // uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. -func (rt *reduceTask) uniqueKey() string { +func (rt *reduceStreamTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", rt.window.GetStart().AsTime().UnixMilli(), rt.window.GetEnd().AsTime().UnixMilli(), @@ -61,7 +61,7 @@ func (rt *reduceTask) uniqueKey() string { // reduceTaskManager manages the reduce tasks for a reduce operation. type reduceTaskManager struct { reducer Reducer - tasks map[string]*reduceTask + tasks map[string]*reduceStreamTask responseCh chan *v1.ReduceResponse rw sync.RWMutex } @@ -69,7 +69,7 @@ type reduceTaskManager struct { func newReduceTaskManager(reducer Reducer) *reduceTaskManager { return &reduceTaskManager{ reducer: reducer, - tasks: make(map[string]*reduceTask), + tasks: make(map[string]*reduceStreamTask), responseCh: make(chan *v1.ReduceResponse), } } @@ -84,7 +84,7 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), request.Operation.Windows[0].GetEnd().AsTime())) - task := &reduceTask{ + task := &reduceStreamTask{ keys: request.GetPayload().GetKeys(), window: request.Operation.Windows[0], inputCh: make(chan Datum), @@ -98,24 +98,17 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce rtm.rw.Unlock() go func() { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for message := range task.outputCh { - // write the output to the output channel, service will forward it to downstream - rtm.responseCh <- task.buildReduceResponse(message) - } - // send EOF - rtm.responseCh <- task.buildEOFResponse() - }() - // invoke the reduce function - rtm.reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh, md) + messages := rtm.reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) + + for _, message := range messages { + // write the output to the output channel, service will forward it to downstream + rtm.responseCh <- task.buildReduceResponse(message) + } + // send EOF + rtm.responseCh <- task.buildEOFResponse() // close the output channel after the reduce function is done close(task.outputCh) - // wait for the output to be forwarded - wg.Wait() // send a done signal close(task.doneCh) }() @@ -154,7 +147,7 @@ func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { // WaitAll waits for all the reduce tasks to complete. func (rtm *reduceTaskManager) WaitAll() { rtm.rw.RLock() - tasks := make([]*reduceTask, 0, len(rtm.tasks)) + tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } @@ -170,7 +163,7 @@ func (rtm *reduceTaskManager) WaitAll() { // CloseAll closes all the reduce tasks. func (rtm *reduceTaskManager) CloseAll() { rtm.rw.Lock() - tasks := make([]*reduceTask, 0, len(rtm.tasks)) + tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } From f9530b28f7231a412b293a03a6e45ea8234add4b Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Wed, 6 Dec 2023 20:18:31 +0530 Subject: [PATCH 17/27] updating examples Signed-off-by: Yashash H L --- pkg/reduceStreamer/examples/counter/Makefile | 2 +- pkg/reduceStreamer/examples/sum/Makefile | 2 +- pkg/reducer/examples/counter/main.go | 13 ++++--------- pkg/reducer/examples/sum/main.go | 16 +++++----------- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/pkg/reduceStreamer/examples/counter/Makefile b/pkg/reduceStreamer/examples/counter/Makefile index a9ccc9d8..ffe3036c 100644 --- a/pkg/reduceStreamer/examples/counter/Makefile +++ b/pkg/reduceStreamer/examples/counter/Makefile @@ -4,7 +4,7 @@ build: .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-counter:v0.5.3" --target counter . + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:v0.5.3" --target counter . clean: -rm -rf ./dist diff --git a/pkg/reduceStreamer/examples/sum/Makefile b/pkg/reduceStreamer/examples/sum/Makefile index 9b5c4e7b..ec0a4e6a 100644 --- a/pkg/reduceStreamer/examples/sum/Makefile +++ b/pkg/reduceStreamer/examples/sum/Makefile @@ -4,7 +4,7 @@ build: .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-sum:v0.5.3" --target sum . + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:v0.5.3" --target sum . clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/counter/main.go b/pkg/reducer/examples/counter/main.go index 3abade1b..0b89e51c 100644 --- a/pkg/reducer/examples/counter/main.go +++ b/pkg/reducer/examples/counter/main.go @@ -4,26 +4,21 @@ import ( "context" "strconv" - "github.com/numaproj/numaflow-go/pkg/reduceStreamer" + "github.com/numaproj/numaflow-go/pkg/reducer" ) -func reduceCounter(_ context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { +func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { // count the incoming events var resultKeys = keys var resultVal []byte var counter = 0 for range inputCh { counter++ - if counter >= 10 { - resultVal = []byte(strconv.Itoa(counter)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) - counter = 0 - } } resultVal = []byte(strconv.Itoa(counter)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + return reducer.MessagesBuilder().Append(reducer.NewMessage(resultVal).WithKeys(resultKeys)) } func main() { - reduceStreamer.NewServer(reduceStreamer.ReducerFunc(reduceCounter)).Start(context.Background()) + reducer.NewServer(reducer.ReducerFunc(reduceCounter)).Start(context.Background()) } diff --git a/pkg/reducer/examples/sum/main.go b/pkg/reducer/examples/sum/main.go index 34e20b5d..c47f73ed 100644 --- a/pkg/reducer/examples/sum/main.go +++ b/pkg/reducer/examples/sum/main.go @@ -6,14 +6,14 @@ import ( "log" "strconv" - "github.com/numaproj/numaflow-go/pkg/reduceStreamer" + "github.com/numaproj/numaflow-go/pkg/reducer" ) -// Sum is a reduceStreamer that sum up the values for the given keys +// Sum is a reducer that sum up the values for the given keys type Sum struct { } -func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { +func (s *Sum) Reduce(ctx context.Context, keys []string, inputCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { // sum up values for the same keys intervalWindow := md.IntervalWindow() _ = intervalWindow @@ -34,19 +34,13 @@ func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan re continue } sum += v - - if sum >= 100 { - resultVal = []byte(strconv.Itoa(sum)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) - sum = 0 - } } resultVal = []byte(strconv.Itoa(sum)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + return reducer.MessagesBuilder().Append(reducer.NewMessage(resultVal).WithKeys(resultKeys)) } func main() { - err := reduceStreamer.NewServer(&Sum{}).Start(context.Background()) + err := reducer.NewServer(&Sum{}).Start(context.Background()) if err != nil { log.Panic("unable to start the server due to: ", err) } From 191cf47835ac6ec757c7e1af8f8f160a3e1a10a7 Mon Sep 17 00:00:00 2001 From: Vigith Maurice Date: Thu, 7 Dec 2023 19:00:39 -0800 Subject: [PATCH 18/27] chore: rename Signed-off-by: Vigith Maurice --- pkg/reduceStreamer/doc.go | 2 +- pkg/reduceStreamer/examples/counter/main.go | 10 +++++----- pkg/reduceStreamer/examples/sum/main.go | 12 ++++++------ pkg/reduceStreamer/interface.go | 2 +- pkg/reduceStreamer/message.go | 2 +- pkg/reduceStreamer/options.go | 2 +- pkg/reduceStreamer/options_test.go | 2 +- pkg/reduceStreamer/server.go | 2 +- pkg/reduceStreamer/server_test.go | 2 +- pkg/reduceStreamer/service.go | 2 +- pkg/reduceStreamer/service_test.go | 2 +- pkg/reduceStreamer/task_manager.go | 2 +- pkg/reduceStreamer/types.go | 2 +- 13 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/reduceStreamer/doc.go b/pkg/reduceStreamer/doc.go index dde78fc0..c7160045 100644 --- a/pkg/reduceStreamer/doc.go +++ b/pkg/reduceStreamer/doc.go @@ -2,4 +2,4 @@ // Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/ -package reduceStreamer +package reducestreamer diff --git a/pkg/reduceStreamer/examples/counter/main.go b/pkg/reduceStreamer/examples/counter/main.go index 3abade1b..22de7e55 100644 --- a/pkg/reduceStreamer/examples/counter/main.go +++ b/pkg/reduceStreamer/examples/counter/main.go @@ -4,10 +4,10 @@ import ( "context" "strconv" - "github.com/numaproj/numaflow-go/pkg/reduceStreamer" + "github.com/numaproj/numaflow-go/pkg/reducestreamer" ) -func reduceCounter(_ context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { +func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducestreamer.Datum, outputCh chan<- reducestreamer.Message, md reducestreamer.Metadata) { // count the incoming events var resultKeys = keys var resultVal []byte @@ -16,14 +16,14 @@ func reduceCounter(_ context.Context, keys []string, inputCh <-chan reduceStream counter++ if counter >= 10 { resultVal = []byte(strconv.Itoa(counter)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) counter = 0 } } resultVal = []byte(strconv.Itoa(counter)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) } func main() { - reduceStreamer.NewServer(reduceStreamer.ReducerFunc(reduceCounter)).Start(context.Background()) + reducestreamer.NewServer(reducestreamer.ReducerFunc(reduceCounter)).Start(context.Background()) } diff --git a/pkg/reduceStreamer/examples/sum/main.go b/pkg/reduceStreamer/examples/sum/main.go index 34e20b5d..5c5e3038 100644 --- a/pkg/reduceStreamer/examples/sum/main.go +++ b/pkg/reduceStreamer/examples/sum/main.go @@ -6,14 +6,14 @@ import ( "log" "strconv" - "github.com/numaproj/numaflow-go/pkg/reduceStreamer" + "github.com/numaproj/numaflow-go/pkg/reducestreamer" ) -// Sum is a reduceStreamer that sum up the values for the given keys +// Sum is a reducestreamer that sum up the values for the given keys type Sum struct { } -func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan reduceStreamer.Datum, outputCh chan<- reduceStreamer.Message, md reduceStreamer.Metadata) { +func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan reducestreamer.Datum, outputCh chan<- reducestreamer.Message, md reducestreamer.Metadata) { // sum up values for the same keys intervalWindow := md.IntervalWindow() _ = intervalWindow @@ -37,16 +37,16 @@ func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan re if sum >= 100 { resultVal = []byte(strconv.Itoa(sum)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) sum = 0 } } resultVal = []byte(strconv.Itoa(sum)) - outputCh <- reduceStreamer.NewMessage(resultVal).WithKeys(resultKeys) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) } func main() { - err := reduceStreamer.NewServer(&Sum{}).Start(context.Background()) + err := reducestreamer.NewServer(&Sum{}).Start(context.Background()) if err != nil { log.Panic("unable to start the server due to: ", err) } diff --git a/pkg/reduceStreamer/interface.go b/pkg/reduceStreamer/interface.go index d5d8dac9..8d0a01c8 100644 --- a/pkg/reduceStreamer/interface.go +++ b/pkg/reduceStreamer/interface.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "context" diff --git a/pkg/reduceStreamer/message.go b/pkg/reduceStreamer/message.go index 766f7a09..2fdf5c64 100644 --- a/pkg/reduceStreamer/message.go +++ b/pkg/reduceStreamer/message.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import "fmt" diff --git a/pkg/reduceStreamer/options.go b/pkg/reduceStreamer/options.go index 2158a39e..7862fb03 100644 --- a/pkg/reduceStreamer/options.go +++ b/pkg/reduceStreamer/options.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "github.com/numaproj/numaflow-go/pkg/info" diff --git a/pkg/reduceStreamer/options_test.go b/pkg/reduceStreamer/options_test.go index e3eda2df..947d80e0 100644 --- a/pkg/reduceStreamer/options_test.go +++ b/pkg/reduceStreamer/options_test.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "testing" diff --git a/pkg/reduceStreamer/server.go b/pkg/reduceStreamer/server.go index 0a1e3a8c..af85ce9c 100644 --- a/pkg/reduceStreamer/server.go +++ b/pkg/reduceStreamer/server.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "context" diff --git a/pkg/reduceStreamer/server_test.go b/pkg/reduceStreamer/server_test.go index 497c86ac..06d0ddb4 100644 --- a/pkg/reduceStreamer/server_test.go +++ b/pkg/reduceStreamer/server_test.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "context" diff --git a/pkg/reduceStreamer/service.go b/pkg/reduceStreamer/service.go index 252aab01..ccd5bb79 100644 --- a/pkg/reduceStreamer/service.go +++ b/pkg/reduceStreamer/service.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "context" diff --git a/pkg/reduceStreamer/service_test.go b/pkg/reduceStreamer/service_test.go index 8b73727d..c8bafc6b 100644 --- a/pkg/reduceStreamer/service_test.go +++ b/pkg/reduceStreamer/service_test.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "context" diff --git a/pkg/reduceStreamer/task_manager.go b/pkg/reduceStreamer/task_manager.go index 3dca3190..cf5a44fc 100644 --- a/pkg/reduceStreamer/task_manager.go +++ b/pkg/reduceStreamer/task_manager.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import ( "context" diff --git a/pkg/reduceStreamer/types.go b/pkg/reduceStreamer/types.go index 8d2125fb..fac5b63a 100644 --- a/pkg/reduceStreamer/types.go +++ b/pkg/reduceStreamer/types.go @@ -1,4 +1,4 @@ -package reduceStreamer +package reducestreamer import "time" From e5b9f4da85463f73be34679294c812903c564133 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Fri, 8 Dec 2023 12:09:18 +0530 Subject: [PATCH 19/27] remove global reduce Signed-off-by: Yashash H L --- .../proto/globalreduce/v1/globalreduce.pb.go | 777 ------------------ .../proto/globalreduce/v1/globalreduce.proto | 85 -- .../globalreduce/v1/globalreduce_grpc.pb.go | 179 ---- .../v1/globalreducemock/globalreducemock.go | 216 ----- pkg/apis/proto/globalreduce/v1/mockgen.go | 3 - pkg/globalreducer/doc.go | 5 - pkg/globalreducer/interface.go | 18 - pkg/globalreducer/message.go | 70 -- pkg/globalreducer/options.go | 43 - pkg/globalreducer/options_test.go | 18 - pkg/globalreducer/server.go | 56 -- pkg/globalreducer/service.go | 100 --- pkg/globalreducer/task_manager.go | 226 ----- pkg/globalreducer/types.go | 30 - pkg/{reduceStreamer => reducestreamer}/doc.go | 0 .../examples/counter/Dockerfile | 0 .../examples/counter/Makefile | 0 .../examples/counter/README.md | 0 .../examples/counter/main.go | 0 .../examples/sum/Dockerfile | 0 .../examples/sum/Makefile | 0 .../examples/sum/README.md | 0 .../examples/sum/main.go | 0 .../interface.go | 0 .../message.go | 0 .../options.go | 0 .../options_test.go | 0 .../server.go | 0 .../server_test.go | 0 .../service.go | 0 .../service_test.go | 0 .../task_manager.go | 0 .../types.go | 0 33 files changed, 1826 deletions(-) delete mode 100644 pkg/apis/proto/globalreduce/v1/globalreduce.pb.go delete mode 100644 pkg/apis/proto/globalreduce/v1/globalreduce.proto delete mode 100644 pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go delete mode 100644 pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go delete mode 100644 pkg/apis/proto/globalreduce/v1/mockgen.go delete mode 100644 pkg/globalreducer/doc.go delete mode 100644 pkg/globalreducer/interface.go delete mode 100644 pkg/globalreducer/message.go delete mode 100644 pkg/globalreducer/options.go delete mode 100644 pkg/globalreducer/options_test.go delete mode 100644 pkg/globalreducer/server.go delete mode 100644 pkg/globalreducer/service.go delete mode 100644 pkg/globalreducer/task_manager.go delete mode 100644 pkg/globalreducer/types.go rename pkg/{reduceStreamer => reducestreamer}/doc.go (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/counter/Dockerfile (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/counter/Makefile (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/counter/README.md (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/counter/main.go (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/sum/Dockerfile (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/sum/Makefile (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/sum/README.md (100%) rename pkg/{reduceStreamer => reducestreamer}/examples/sum/main.go (100%) rename pkg/{reduceStreamer => reducestreamer}/interface.go (100%) rename pkg/{reduceStreamer => reducestreamer}/message.go (100%) rename pkg/{reduceStreamer => reducestreamer}/options.go (100%) rename pkg/{reduceStreamer => reducestreamer}/options_test.go (100%) rename pkg/{reduceStreamer => reducestreamer}/server.go (100%) rename pkg/{reduceStreamer => reducestreamer}/server_test.go (100%) rename pkg/{reduceStreamer => reducestreamer}/service.go (100%) rename pkg/{reduceStreamer => reducestreamer}/service_test.go (100%) rename pkg/{reduceStreamer => reducestreamer}/task_manager.go (100%) rename pkg/{reduceStreamer => reducestreamer}/types.go (100%) diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go deleted file mode 100644 index 38770574..00000000 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.pb.go +++ /dev/null @@ -1,777 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.1 -// protoc v4.25.1 -// source: pkg/apis/proto/globalreduce/v1/globalreduce.proto - -package v1 - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - emptypb "google.golang.org/protobuf/types/known/emptypb" - timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type GlobalReduceRequest_WindowOperation_Event int32 - -const ( - GlobalReduceRequest_WindowOperation_OPEN GlobalReduceRequest_WindowOperation_Event = 0 - GlobalReduceRequest_WindowOperation_CLOSE GlobalReduceRequest_WindowOperation_Event = 1 - GlobalReduceRequest_WindowOperation_APPEND GlobalReduceRequest_WindowOperation_Event = 4 -) - -// Enum value maps for GlobalReduceRequest_WindowOperation_Event. -var ( - GlobalReduceRequest_WindowOperation_Event_name = map[int32]string{ - 0: "OPEN", - 1: "CLOSE", - 4: "APPEND", - } - GlobalReduceRequest_WindowOperation_Event_value = map[string]int32{ - "OPEN": 0, - "CLOSE": 1, - "APPEND": 4, - } -) - -func (x GlobalReduceRequest_WindowOperation_Event) Enum() *GlobalReduceRequest_WindowOperation_Event { - p := new(GlobalReduceRequest_WindowOperation_Event) - *p = x - return p -} - -func (x GlobalReduceRequest_WindowOperation_Event) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (GlobalReduceRequest_WindowOperation_Event) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes[0].Descriptor() -} - -func (GlobalReduceRequest_WindowOperation_Event) Type() protoreflect.EnumType { - return &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes[0] -} - -func (x GlobalReduceRequest_WindowOperation_Event) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use GlobalReduceRequest_WindowOperation_Event.Descriptor instead. -func (GlobalReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1, 0, 0} -} - -// KeyedWindow represents a window with keys. -// since the client track the keys, we use keyed window. -type KeyedWindow struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` - End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` - Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` - Keys []string `protobuf:"bytes,4,rep,name=keys,proto3" json:"keys,omitempty"` -} - -func (x *KeyedWindow) Reset() { - *x = KeyedWindow{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *KeyedWindow) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*KeyedWindow) ProtoMessage() {} - -func (x *KeyedWindow) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use KeyedWindow.ProtoReflect.Descriptor instead. -func (*KeyedWindow) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{0} -} - -func (x *KeyedWindow) GetStart() *timestamppb.Timestamp { - if x != nil { - return x.Start - } - return nil -} - -func (x *KeyedWindow) GetEnd() *timestamppb.Timestamp { - if x != nil { - return x.End - } - return nil -} - -func (x *KeyedWindow) GetSlot() string { - if x != nil { - return x.Slot - } - return "" -} - -func (x *KeyedWindow) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -// * -// GlobalReduceRequest represents a request element. -type GlobalReduceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Payload *GlobalReduceRequest_Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - Operation *GlobalReduceRequest_WindowOperation `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` -} - -func (x *GlobalReduceRequest) Reset() { - *x = GlobalReduceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GlobalReduceRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GlobalReduceRequest) ProtoMessage() {} - -func (x *GlobalReduceRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GlobalReduceRequest.ProtoReflect.Descriptor instead. -func (*GlobalReduceRequest) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1} -} - -func (x *GlobalReduceRequest) GetPayload() *GlobalReduceRequest_Payload { - if x != nil { - return x.Payload - } - return nil -} - -func (x *GlobalReduceRequest) GetOperation() *GlobalReduceRequest_WindowOperation { - if x != nil { - return x.Operation - } - return nil -} - -// * -// GlobalReduceResponse represents a response element. -type GlobalReduceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Result *GlobalReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - // keyedWindow represents a window to which the result belongs. - KeyedWindow *KeyedWindow `protobuf:"bytes,2,opt,name=keyedWindow,proto3" json:"keyedWindow,omitempty"` - // EOF represents the end of the response for a window. - EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` -} - -func (x *GlobalReduceResponse) Reset() { - *x = GlobalReduceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GlobalReduceResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GlobalReduceResponse) ProtoMessage() {} - -func (x *GlobalReduceResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GlobalReduceResponse.ProtoReflect.Descriptor instead. -func (*GlobalReduceResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{2} -} - -func (x *GlobalReduceResponse) GetResult() *GlobalReduceResponse_Result { - if x != nil { - return x.Result - } - return nil -} - -func (x *GlobalReduceResponse) GetKeyedWindow() *KeyedWindow { - if x != nil { - return x.KeyedWindow - } - return nil -} - -func (x *GlobalReduceResponse) GetEOF() bool { - if x != nil { - return x.EOF - } - return false -} - -// * -// ReadyResponse is the health check result. -type ReadyResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` -} - -func (x *ReadyResponse) Reset() { - *x = ReadyResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ReadyResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ReadyResponse) ProtoMessage() {} - -func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. -func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{3} -} - -func (x *ReadyResponse) GetReady() bool { - if x != nil { - return x.Ready - } - return false -} - -// WindowOperation represents a window operation. -// it can be one of OPEN, CLOSE, and APPEND. -type GlobalReduceRequest_WindowOperation struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Event GlobalReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=globalreduce.v1.GlobalReduceRequest_WindowOperation_Event" json:"event,omitempty"` - KeyedWindows []*KeyedWindow `protobuf:"bytes,2,rep,name=keyedWindows,proto3" json:"keyedWindows,omitempty"` -} - -func (x *GlobalReduceRequest_WindowOperation) Reset() { - *x = GlobalReduceRequest_WindowOperation{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GlobalReduceRequest_WindowOperation) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GlobalReduceRequest_WindowOperation) ProtoMessage() {} - -func (x *GlobalReduceRequest_WindowOperation) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GlobalReduceRequest_WindowOperation.ProtoReflect.Descriptor instead. -func (*GlobalReduceRequest_WindowOperation) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *GlobalReduceRequest_WindowOperation) GetEvent() GlobalReduceRequest_WindowOperation_Event { - if x != nil { - return x.Event - } - return GlobalReduceRequest_WindowOperation_OPEN -} - -func (x *GlobalReduceRequest_WindowOperation) GetKeyedWindows() []*KeyedWindow { - if x != nil { - return x.KeyedWindows - } - return nil -} - -// Payload represents a payload element. -type GlobalReduceRequest_Payload struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` -} - -func (x *GlobalReduceRequest_Payload) Reset() { - *x = GlobalReduceRequest_Payload{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GlobalReduceRequest_Payload) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GlobalReduceRequest_Payload) ProtoMessage() {} - -func (x *GlobalReduceRequest_Payload) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GlobalReduceRequest_Payload.ProtoReflect.Descriptor instead. -func (*GlobalReduceRequest_Payload) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{1, 1} -} - -func (x *GlobalReduceRequest_Payload) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -func (x *GlobalReduceRequest_Payload) GetValue() []byte { - if x != nil { - return x.Value - } - return nil -} - -func (x *GlobalReduceRequest_Payload) GetEventTime() *timestamppb.Timestamp { - if x != nil { - return x.EventTime - } - return nil -} - -func (x *GlobalReduceRequest_Payload) GetWatermark() *timestamppb.Timestamp { - if x != nil { - return x.Watermark - } - return nil -} - -// Result represents a result element. It contains the result of the reduce function. -type GlobalReduceResponse_Result struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` - // EventTime represents the event time of the result. - EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` -} - -func (x *GlobalReduceResponse_Result) Reset() { - *x = GlobalReduceResponse_Result{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GlobalReduceResponse_Result) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GlobalReduceResponse_Result) ProtoMessage() {} - -func (x *GlobalReduceResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GlobalReduceResponse_Result.ProtoReflect.Descriptor instead. -func (*GlobalReduceResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP(), []int{2, 0} -} - -func (x *GlobalReduceResponse_Result) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -func (x *GlobalReduceResponse_Result) GetValue() []byte { - if x != nil { - return x.Value - } - return nil -} - -func (x *GlobalReduceResponse_Result) GetTags() []string { - if x != nil { - return x.Tags - } - return nil -} - -func (x *GlobalReduceResponse_Result) GetEventTime() *timestamppb.Timestamp { - if x != nil { - return x.EventTime - } - return nil -} - -var File_pkg_apis_proto_globalreduce_v1_globalreduce_proto protoreflect.FileDescriptor - -var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = []byte{ - 0x0a, 0x31, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2f, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, 0x31, - 0x2f, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x22, 0x95, 0x01, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, - 0x6f, 0x77, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, - 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xae, 0x04, 0x0a, 0x13, 0x47, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x46, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x52, 0x0a, 0x09, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, - 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xcf, - 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x3a, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, - 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, - 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, - 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x22, 0x28, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, - 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, - 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, - 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, - 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xb2, 0x02, 0x0a, 0x14, - 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x3e, 0x0a, 0x0b, 0x6b, 0x65, - 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x0b, 0x6b, - 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, - 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, - 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, 0x62, - 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x47, 0x6c, 0x6f, 0x62, - 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x24, 0x2e, 0x67, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, - 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x07, 0x49, - 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, - 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, - 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, - 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, - 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescOnce sync.Once - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData = file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc -) - -func file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescGZIP() []byte { - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescOnce.Do(func() { - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData) - }) - return file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDescData -} - -var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) -var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes = []interface{}{ - (GlobalReduceRequest_WindowOperation_Event)(0), // 0: globalreduce.v1.GlobalReduceRequest.WindowOperation.Event - (*KeyedWindow)(nil), // 1: globalreduce.v1.KeyedWindow - (*GlobalReduceRequest)(nil), // 2: globalreduce.v1.GlobalReduceRequest - (*GlobalReduceResponse)(nil), // 3: globalreduce.v1.GlobalReduceResponse - (*ReadyResponse)(nil), // 4: globalreduce.v1.ReadyResponse - (*GlobalReduceRequest_WindowOperation)(nil), // 5: globalreduce.v1.GlobalReduceRequest.WindowOperation - (*GlobalReduceRequest_Payload)(nil), // 6: globalreduce.v1.GlobalReduceRequest.Payload - (*GlobalReduceResponse_Result)(nil), // 7: globalreduce.v1.GlobalReduceResponse.Result - (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 9: google.protobuf.Empty -} -var file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = []int32{ - 8, // 0: globalreduce.v1.KeyedWindow.start:type_name -> google.protobuf.Timestamp - 8, // 1: globalreduce.v1.KeyedWindow.end:type_name -> google.protobuf.Timestamp - 6, // 2: globalreduce.v1.GlobalReduceRequest.payload:type_name -> globalreduce.v1.GlobalReduceRequest.Payload - 5, // 3: globalreduce.v1.GlobalReduceRequest.operation:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation - 7, // 4: globalreduce.v1.GlobalReduceResponse.result:type_name -> globalreduce.v1.GlobalReduceResponse.Result - 1, // 5: globalreduce.v1.GlobalReduceResponse.keyedWindow:type_name -> globalreduce.v1.KeyedWindow - 0, // 6: globalreduce.v1.GlobalReduceRequest.WindowOperation.event:type_name -> globalreduce.v1.GlobalReduceRequest.WindowOperation.Event - 1, // 7: globalreduce.v1.GlobalReduceRequest.WindowOperation.keyedWindows:type_name -> globalreduce.v1.KeyedWindow - 8, // 8: globalreduce.v1.GlobalReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp - 8, // 9: globalreduce.v1.GlobalReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp - 8, // 10: globalreduce.v1.GlobalReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp - 2, // 11: globalreduce.v1.GlobalReduce.GlobalReduceFn:input_type -> globalreduce.v1.GlobalReduceRequest - 9, // 12: globalreduce.v1.GlobalReduce.IsReady:input_type -> google.protobuf.Empty - 3, // 13: globalreduce.v1.GlobalReduce.GlobalReduceFn:output_type -> globalreduce.v1.GlobalReduceResponse - 4, // 14: globalreduce.v1.GlobalReduce.IsReady:output_type -> globalreduce.v1.ReadyResponse - 13, // [13:15] is the sub-list for method output_type - 11, // [11:13] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name -} - -func init() { file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_init() } -func file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_init() { - if File_pkg_apis_proto_globalreduce_v1_globalreduce_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KeyedWindow); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GlobalReduceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GlobalReduceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadyResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GlobalReduceRequest_WindowOperation); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GlobalReduceRequest_Payload); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GlobalReduceResponse_Result); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc, - NumEnums: 1, - NumMessages: 7, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes, - DependencyIndexes: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs, - EnumInfos: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_enumTypes, - MessageInfos: file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_msgTypes, - }.Build() - File_pkg_apis_proto_globalreduce_v1_globalreduce_proto = out.File - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_rawDesc = nil - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_goTypes = nil - file_pkg_apis_proto_globalreduce_v1_globalreduce_proto_depIdxs = nil -} diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce.proto b/pkg/apis/proto/globalreduce/v1/globalreduce.proto deleted file mode 100644 index 3d38dfb7..00000000 --- a/pkg/apis/proto/globalreduce/v1/globalreduce.proto +++ /dev/null @@ -1,85 +0,0 @@ -syntax = "proto3"; - -option go_package = "github.com/numaproj/numaflow-go/pkg/apis/proto/reducestream/v1"; - -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; - - -package globalreduce.v1; - -service GlobalReduce { - // GlobalReduceFn applies a reduce function to a request stream. - rpc GlobalReduceFn(stream GlobalReduceRequest) returns (stream GlobalReduceResponse); - - // IsReady is the heartbeat endpoint for gRPC. - rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); -} - -// KeyedWindow represents a window with keys. -// since the client track the keys, we use keyed window. -message KeyedWindow { - google.protobuf.Timestamp start = 1; - google.protobuf.Timestamp end = 2; - string slot = 3; - repeated string keys = 4; -} - -/** - * GlobalReduceRequest represents a request element. - */ -message GlobalReduceRequest { - // WindowOperation represents a window operation. - // it can be one of OPEN, CLOSE, and APPEND. - message WindowOperation { - enum Event { - OPEN = 0; - CLOSE = 1; - APPEND = 4; - } - - Event event = 1; - repeated KeyedWindow keyedWindows = 2; - } - - // Payload represents a payload element. - message Payload { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; - } - - Payload payload = 1; - WindowOperation operation = 2; -} - -/** - * GlobalReduceResponse represents a response element. - */ -message GlobalReduceResponse { - // Result represents a result element. It contains the result of the reduce function. - message Result { - repeated string keys = 1; - bytes value = 2; - repeated string tags = 3; - // EventTime represents the event time of the result. - google.protobuf.Timestamp event_time = 4; - } - - Result result = 1; - - // keyedWindow represents a window to which the result belongs. - KeyedWindow keyedWindow = 2; - - // EOF represents the end of the response for a window. - bool EOF = 3; - -} - -/** - * ReadyResponse is the health check result. - */ -message ReadyResponse { - bool ready = 1; -} \ No newline at end of file diff --git a/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go b/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go deleted file mode 100644 index 6b1dc292..00000000 --- a/pkg/apis/proto/globalreduce/v1/globalreduce_grpc.pb.go +++ /dev/null @@ -1,179 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v4.25.1 -// source: pkg/apis/proto/globalreduce/v1/globalreduce.proto - -package v1 - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// GlobalReduceClient is the client API for GlobalReduce service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type GlobalReduceClient interface { - // GlobalReduceFn applies a reduce function to a request stream. - GlobalReduceFn(ctx context.Context, opts ...grpc.CallOption) (GlobalReduce_GlobalReduceFnClient, error) - // IsReady is the heartbeat endpoint for gRPC. - IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) -} - -type globalReduceClient struct { - cc grpc.ClientConnInterface -} - -func NewGlobalReduceClient(cc grpc.ClientConnInterface) GlobalReduceClient { - return &globalReduceClient{cc} -} - -func (c *globalReduceClient) GlobalReduceFn(ctx context.Context, opts ...grpc.CallOption) (GlobalReduce_GlobalReduceFnClient, error) { - stream, err := c.cc.NewStream(ctx, &GlobalReduce_ServiceDesc.Streams[0], "/globalreduce.v1.GlobalReduce/GlobalReduceFn", opts...) - if err != nil { - return nil, err - } - x := &globalReduceGlobalReduceFnClient{stream} - return x, nil -} - -type GlobalReduce_GlobalReduceFnClient interface { - Send(*GlobalReduceRequest) error - Recv() (*GlobalReduceResponse, error) - grpc.ClientStream -} - -type globalReduceGlobalReduceFnClient struct { - grpc.ClientStream -} - -func (x *globalReduceGlobalReduceFnClient) Send(m *GlobalReduceRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *globalReduceGlobalReduceFnClient) Recv() (*GlobalReduceResponse, error) { - m := new(GlobalReduceResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *globalReduceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { - out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/globalreduce.v1.GlobalReduce/IsReady", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// GlobalReduceServer is the server API for GlobalReduce service. -// All implementations must embed UnimplementedGlobalReduceServer -// for forward compatibility -type GlobalReduceServer interface { - // GlobalReduceFn applies a reduce function to a request stream. - GlobalReduceFn(GlobalReduce_GlobalReduceFnServer) error - // IsReady is the heartbeat endpoint for gRPC. - IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) - mustEmbedUnimplementedGlobalReduceServer() -} - -// UnimplementedGlobalReduceServer must be embedded to have forward compatible implementations. -type UnimplementedGlobalReduceServer struct { -} - -func (UnimplementedGlobalReduceServer) GlobalReduceFn(GlobalReduce_GlobalReduceFnServer) error { - return status.Errorf(codes.Unimplemented, "method GlobalReduceFn not implemented") -} -func (UnimplementedGlobalReduceServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") -} -func (UnimplementedGlobalReduceServer) mustEmbedUnimplementedGlobalReduceServer() {} - -// UnsafeGlobalReduceServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to GlobalReduceServer will -// result in compilation errors. -type UnsafeGlobalReduceServer interface { - mustEmbedUnimplementedGlobalReduceServer() -} - -func RegisterGlobalReduceServer(s grpc.ServiceRegistrar, srv GlobalReduceServer) { - s.RegisterService(&GlobalReduce_ServiceDesc, srv) -} - -func _GlobalReduce_GlobalReduceFn_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(GlobalReduceServer).GlobalReduceFn(&globalReduceGlobalReduceFnServer{stream}) -} - -type GlobalReduce_GlobalReduceFnServer interface { - Send(*GlobalReduceResponse) error - Recv() (*GlobalReduceRequest, error) - grpc.ServerStream -} - -type globalReduceGlobalReduceFnServer struct { - grpc.ServerStream -} - -func (x *globalReduceGlobalReduceFnServer) Send(m *GlobalReduceResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *globalReduceGlobalReduceFnServer) Recv() (*GlobalReduceRequest, error) { - m := new(GlobalReduceRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _GlobalReduce_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(emptypb.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(GlobalReduceServer).IsReady(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/globalreduce.v1.GlobalReduce/IsReady", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(GlobalReduceServer).IsReady(ctx, req.(*emptypb.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -// GlobalReduce_ServiceDesc is the grpc.ServiceDesc for GlobalReduce service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var GlobalReduce_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "globalreduce.v1.GlobalReduce", - HandlerType: (*GlobalReduceServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "IsReady", - Handler: _GlobalReduce_IsReady_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "GlobalReduceFn", - Handler: _GlobalReduce_GlobalReduceFn_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "pkg/apis/proto/globalreduce/v1/globalreduce.proto", -} diff --git a/pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go b/pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go deleted file mode 100644 index 108a3d72..00000000 --- a/pkg/apis/proto/globalreduce/v1/globalreducemock/globalreducemock.go +++ /dev/null @@ -1,216 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1 (interfaces: GlobalReduceClient,GlobalReduce_GlobalReduceFnClient) - -// Package globalreducemock is a generated GoMock package. -package globalreducemock - -import ( - context "context" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" - grpc "google.golang.org/grpc" - metadata "google.golang.org/grpc/metadata" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -// MockGlobalReduceClient is a mock of GlobalReduceClient interface. -type MockGlobalReduceClient struct { - ctrl *gomock.Controller - recorder *MockGlobalReduceClientMockRecorder -} - -// MockGlobalReduceClientMockRecorder is the mock recorder for MockGlobalReduceClient. -type MockGlobalReduceClientMockRecorder struct { - mock *MockGlobalReduceClient -} - -// NewMockGlobalReduceClient creates a new mock instance. -func NewMockGlobalReduceClient(ctrl *gomock.Controller) *MockGlobalReduceClient { - mock := &MockGlobalReduceClient{ctrl: ctrl} - mock.recorder = &MockGlobalReduceClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockGlobalReduceClient) EXPECT() *MockGlobalReduceClientMockRecorder { - return m.recorder -} - -// GlobalReduceFn mocks base method. -func (m *MockGlobalReduceClient) GlobalReduceFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.GlobalReduce_GlobalReduceFnClient, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GlobalReduceFn", varargs...) - ret0, _ := ret[0].(v1.GlobalReduce_GlobalReduceFnClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GlobalReduceFn indicates an expected call of GlobalReduceFn. -func (mr *MockGlobalReduceClientMockRecorder) GlobalReduceFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GlobalReduceFn", reflect.TypeOf((*MockGlobalReduceClient)(nil).GlobalReduceFn), varargs...) -} - -// IsReady mocks base method. -func (m *MockGlobalReduceClient) IsReady(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*v1.ReadyResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "IsReady", varargs...) - ret0, _ := ret[0].(*v1.ReadyResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsReady indicates an expected call of IsReady. -func (mr *MockGlobalReduceClientMockRecorder) IsReady(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsReady", reflect.TypeOf((*MockGlobalReduceClient)(nil).IsReady), varargs...) -} - -// MockGlobalReduce_GlobalReduceFnClient is a mock of GlobalReduce_GlobalReduceFnClient interface. -type MockGlobalReduce_GlobalReduceFnClient struct { - ctrl *gomock.Controller - recorder *MockGlobalReduce_GlobalReduceFnClientMockRecorder -} - -// MockGlobalReduce_GlobalReduceFnClientMockRecorder is the mock recorder for MockGlobalReduce_GlobalReduceFnClient. -type MockGlobalReduce_GlobalReduceFnClientMockRecorder struct { - mock *MockGlobalReduce_GlobalReduceFnClient -} - -// NewMockGlobalReduce_GlobalReduceFnClient creates a new mock instance. -func NewMockGlobalReduce_GlobalReduceFnClient(ctrl *gomock.Controller) *MockGlobalReduce_GlobalReduceFnClient { - mock := &MockGlobalReduce_GlobalReduceFnClient{ctrl: ctrl} - mock.recorder = &MockGlobalReduce_GlobalReduceFnClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockGlobalReduce_GlobalReduceFnClient) EXPECT() *MockGlobalReduce_GlobalReduceFnClientMockRecorder { - return m.recorder -} - -// CloseSend mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) CloseSend() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CloseSend") - ret0, _ := ret[0].(error) - return ret0 -} - -// CloseSend indicates an expected call of CloseSend. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) CloseSend() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseSend", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).CloseSend)) -} - -// Context mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) Context() context.Context { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Context") - ret0, _ := ret[0].(context.Context) - return ret0 -} - -// Context indicates an expected call of Context. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Context() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Context)) -} - -// Header mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) Header() (metadata.MD, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Header") - ret0, _ := ret[0].(metadata.MD) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Header indicates an expected call of Header. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Header() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Header)) -} - -// Recv mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) Recv() (*v1.GlobalReduceResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Recv") - ret0, _ := ret[0].(*v1.GlobalReduceResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Recv indicates an expected call of Recv. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Recv() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Recv)) -} - -// RecvMsg mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) RecvMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecvMsg", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// RecvMsg indicates an expected call of RecvMsg. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).RecvMsg), arg0) -} - -// Send mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) Send(arg0 *v1.GlobalReduceRequest) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Send", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Send indicates an expected call of Send. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Send(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Send), arg0) -} - -// SendMsg mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) SendMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMsg", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMsg indicates an expected call of SendMsg. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).SendMsg), arg0) -} - -// Trailer mocks base method. -func (m *MockGlobalReduce_GlobalReduceFnClient) Trailer() metadata.MD { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Trailer") - ret0, _ := ret[0].(metadata.MD) - return ret0 -} - -// Trailer indicates an expected call of Trailer. -func (mr *MockGlobalReduce_GlobalReduceFnClientMockRecorder) Trailer() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockGlobalReduce_GlobalReduceFnClient)(nil).Trailer)) -} diff --git a/pkg/apis/proto/globalreduce/v1/mockgen.go b/pkg/apis/proto/globalreduce/v1/mockgen.go deleted file mode 100644 index 3dc4e37a..00000000 --- a/pkg/apis/proto/globalreduce/v1/mockgen.go +++ /dev/null @@ -1,3 +0,0 @@ -package v1 - -//go:generate mockgen -destination globalreducemock/globalreducemock.go -package globalreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1 GlobalReduceClient,GlobalReduce_GlobalReduceFnClient diff --git a/pkg/globalreducer/doc.go b/pkg/globalreducer/doc.go deleted file mode 100644 index e8c00ec9..00000000 --- a/pkg/globalreducer/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -// Package reducer implements the server code for reduce operation. - -// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/ - -package globalreducer diff --git a/pkg/globalreducer/interface.go b/pkg/globalreducer/interface.go deleted file mode 100644 index 3950286f..00000000 --- a/pkg/globalreducer/interface.go +++ /dev/null @@ -1,18 +0,0 @@ -package globalreducer - -import ( - "context" - "time" -) - -// Datum contains methods to get the payload information. -type Datum interface { - Value() []byte - EventTime() time.Time - Watermark() time.Time -} - -// GlobalReducer is the interface which can be used to implement a session reduce operation. -type GlobalReducer interface { - GlobalReduce(ctx context.Context, keys []string, input <-chan Datum, output <-chan Message) -} diff --git a/pkg/globalreducer/message.go b/pkg/globalreducer/message.go deleted file mode 100644 index 91167f34..00000000 --- a/pkg/globalreducer/message.go +++ /dev/null @@ -1,70 +0,0 @@ -package globalreducer - -import "fmt" - -var ( - DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ -) - -// Message is used to wrap the data return by reduce functions -type Message struct { - value []byte - keys []string - tags []string -} - -// NewMessage creates a Message with value -func NewMessage(value []byte) Message { - return Message{value: value} -} - -// MessageToDrop creates a Message to be dropped -func MessageToDrop() Message { - return Message{value: []byte{}, tags: []string{DROP}} -} - -// WithKeys is used to assign the keys to the message -func (m Message) WithKeys(keys []string) Message { - m.keys = keys - return m -} - -// WithTags is used to assign the tags to the message -// tags will be used for conditional forwarding -func (m Message) WithTags(tags []string) Message { - m.tags = tags - return m -} - -// Keys returns message keys -func (m Message) Keys() []string { - return m.keys -} - -// Value returns message value -func (m Message) Value() []byte { - return m.value -} - -// Tags returns message tags -func (m Message) Tags() []string { - return m.tags -} - -type Messages []Message - -// MessagesBuilder returns an empty instance of Messages -func MessagesBuilder() Messages { - return Messages{} -} - -// Append appends a Message -func (m Messages) Append(msg Message) Messages { - m = append(m, msg) - return m -} - -// Items returns the message list -func (m Messages) Items() []Message { - return m -} diff --git a/pkg/globalreducer/options.go b/pkg/globalreducer/options.go deleted file mode 100644 index 76a59c69..00000000 --- a/pkg/globalreducer/options.go +++ /dev/null @@ -1,43 +0,0 @@ -package globalreducer - -import ( - "github.com/numaproj/numaflow-go/pkg/info" -) - -type options struct { - sockAddr string - maxMessageSize int - serverInfoFilePath string -} - -// Option is the interface to apply options. -type Option func(*options) - -func DefaultOptions() *options { - return &options{ - sockAddr: address, - maxMessageSize: defaultMaxMessageSize, - serverInfoFilePath: info.ServerInfoFilePath, - } -} - -// WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. -func WithMaxMessageSize(size int) Option { - return func(opts *options) { - opts.maxMessageSize = size - } -} - -// WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. -func WithSockAddr(addr string) Option { - return func(opts *options) { - opts.sockAddr = addr - } -} - -// WithServerInfoFilePath sets the server info file path to the given path. -func WithServerInfoFilePath(f string) Option { - return func(opts *options) { - opts.serverInfoFilePath = f - } -} diff --git a/pkg/globalreducer/options_test.go b/pkg/globalreducer/options_test.go deleted file mode 100644 index 93029ba7..00000000 --- a/pkg/globalreducer/options_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package globalreducer - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestWithMaxMessageSize(t *testing.T) { - var ( - size = 1024 * 1024 * 10 - opts = &options{ - maxMessageSize: defaultMaxMessageSize, - } - ) - WithMaxMessageSize(1024 * 1024 * 10)(opts) - assert.Equal(t, size, opts.maxMessageSize) -} diff --git a/pkg/globalreducer/server.go b/pkg/globalreducer/server.go deleted file mode 100644 index 56833765..00000000 --- a/pkg/globalreducer/server.go +++ /dev/null @@ -1,56 +0,0 @@ -package globalreducer - -import ( - "context" - "fmt" - "os/signal" - "syscall" - - "github.com/numaproj/numaflow-go/pkg" - globalreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" - "github.com/numaproj/numaflow-go/pkg/shared" -) - -// server is a global reduce gRPC server. -type server struct { - svc *Service - opts *options -} - -// NewServer creates a new global reduce server. -func NewServer(r GlobalReducer, inputOptions ...Option) numaflow.Server { - opts := DefaultOptions() - for _, inputOption := range inputOptions { - inputOption(opts) - } - s := new(server) - s.svc = new(Service) - s.svc.globalReducer = r - s.opts = opts - return s -} - -// Start starts the global reduce gRPC server. -func (r *server) Start(ctx context.Context) error { - ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) - defer stop() - - // write server info to the file - // start listening on unix domain socket - lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath) - if err != nil { - return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) - } - // close the listener - defer func() { _ = lis.Close() }() - - // create a grpc server - grpcServer := shared.CreateGRPCServer(r.opts.maxMessageSize) - defer grpcServer.GracefulStop() - - // register the globalReduce service - globalreducepb.RegisterGlobalReduceServer(grpcServer, r.svc) - - // start the grpc server - return shared.StartGRPCServer(ctxWithSignal, grpcServer, lis) -} diff --git a/pkg/globalreducer/service.go b/pkg/globalreducer/service.go deleted file mode 100644 index d7482b3a..00000000 --- a/pkg/globalreducer/service.go +++ /dev/null @@ -1,100 +0,0 @@ -package globalreducer - -import ( - "context" - "io" - - "golang.org/x/sync/errgroup" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/emptypb" - - globalreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" -) - -const ( - uds = "unix" - defaultMaxMessageSize = 1024 * 1024 * 64 - address = "/var/run/numaflow/globalreduce.sock" - delimiter = ":" -) - -// Service implements the proto gen server interface and contains the globalreduce operation handler. -type Service struct { - globalreducepb.UnimplementedGlobalReduceServer - globalReducer GlobalReducer -} - -// IsReady returns true to indicate the gRPC connection is ready. -func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*globalreducepb.ReadyResponse, error) { - return &globalreducepb.ReadyResponse{Ready: true}, nil -} - -// GlobalReduceFn applies a global reduce function to a request stream and streams the results. -func (fs *Service) GlobalReduceFn(stream globalreducepb.GlobalReduce_GlobalReduceFnServer) error { - - ctx := stream.Context() - taskManager := newReduceTaskManager(fs.globalReducer) - // err group for the go routine which reads from the output channel and sends to the stream - var g errgroup.Group - - g.Go(func() error { - for output := range taskManager.OutputChannel() { - err := stream.Send(output) - if err != nil { - return err - } - } - return nil - }) - - for { - d, recvErr := stream.Recv() - - // if the stream is closed, close all the tasks and break - if recvErr == io.EOF { - taskManager.CloseAll() - break - } - - if recvErr != nil { - statusErr := status.Errorf(codes.Internal, recvErr.Error()) - return statusErr - } - - // invoke the appropriate task manager method based on the operation - switch d.Operation.Event { - case globalreducepb.GlobalReduceRequest_WindowOperation_OPEN: - // create a new task and start the global reduce operation - // also append the datum to the task - err := taskManager.CreateTask(ctx, d) - if err != nil { - statusErr := status.Errorf(codes.Internal, err.Error()) - return statusErr - } - case globalreducepb.GlobalReduceRequest_WindowOperation_CLOSE: - // close the task - taskManager.CloseTask(d) - case globalreducepb.GlobalReduceRequest_WindowOperation_APPEND: - // append the datum to the task - err := taskManager.AppendToTask(ctx, d) - if err != nil { - statusErr := status.Errorf(codes.Internal, err.Error()) - return statusErr - } - } - - } - - // wait for all the tasks to return - taskManager.WaitAll() - - // wait for the go routine which reads from the output channel and sends to the stream to return - err := g.Wait() - if err != nil { - statusErr := status.Errorf(codes.Internal, err.Error()) - return statusErr - } - - return nil -} diff --git a/pkg/globalreducer/task_manager.go b/pkg/globalreducer/task_manager.go deleted file mode 100644 index c71494d9..00000000 --- a/pkg/globalreducer/task_manager.go +++ /dev/null @@ -1,226 +0,0 @@ -package globalreducer - -import ( - "context" - "fmt" - "strings" - "sync" - "time" - - "go.uber.org/atomic" - "google.golang.org/protobuf/types/known/timestamppb" - - v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/globalreduce/v1" -) - -// globalReduceTask represents a global reduce task for a global reduce operation. -type globalReduceTask struct { - keyedWindow *v1.KeyedWindow - globalReducer GlobalReducer - inputCh chan Datum - outputCh chan Message - latestWatermark *atomic.Time - doneCh chan struct{} -} - -// buildGlobalReduceResponse builds the global reduce response from the messages. -func (rt *globalReduceTask) buildGlobalReduceResponse(message Message) *v1.GlobalReduceResponse { - - // the event time is the latest watermark - // since for global window, keyedWindow start and end time will be -1 - response := &v1.GlobalReduceResponse{ - Result: &v1.GlobalReduceResponse_Result{ - Keys: message.Keys(), - Value: message.Value(), - Tags: message.Tags(), - EventTime: timestamppb.New(rt.latestWatermark.Load()), - }, - KeyedWindow: rt.keyedWindow, - } - - return response -} - -func (rt *globalReduceTask) buildEOFResponse() *v1.GlobalReduceResponse { - response := &v1.GlobalReduceResponse{ - Result: &v1.GlobalReduceResponse_Result{ - EventTime: timestamppb.New(rt.latestWatermark.Load()), - }, - KeyedWindow: rt.keyedWindow, - EOF: true, - } - - return response -} - -// uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. -func (rt *globalReduceTask) uniqueKey() string { - return fmt.Sprintf("%d:%d:%s", - rt.keyedWindow.GetStart().AsTime().UnixMilli(), - rt.keyedWindow.GetEnd().AsTime().UnixMilli(), - strings.Join(rt.keyedWindow.GetKeys(), delimiter)) -} - -// globalReduceTaskManager manages the reduce tasks for a global reduce operation. -type globalReduceTaskManager struct { - globalReducer GlobalReducer - tasks map[string]*globalReduceTask - responseCh chan *v1.GlobalReduceResponse - rw sync.RWMutex -} - -func newReduceTaskManager(globalReducer GlobalReducer) *globalReduceTaskManager { - return &globalReduceTaskManager{ - tasks: make(map[string]*globalReduceTask), - responseCh: make(chan *v1.GlobalReduceResponse), - globalReducer: globalReducer, - } -} - -// CreateTask creates a new reduce task and starts the global reduce operation. -func (rtm *globalReduceTaskManager) CreateTask(ctx context.Context, request *v1.GlobalReduceRequest) error { - rtm.rw.Lock() - - if len(request.Operation.KeyedWindows) != 1 { - return fmt.Errorf("create operation error: invalid number of windows") - } - - task := &globalReduceTask{ - keyedWindow: request.Operation.KeyedWindows[0], - globalReducer: rtm.globalReducer, - inputCh: make(chan Datum), - outputCh: make(chan Message), - doneCh: make(chan struct{}), - latestWatermark: atomic.NewTime(time.Time{}), - } - - key := task.uniqueKey() - rtm.tasks[key] = task - - rtm.rw.Unlock() - - go func() { - defer close(task.doneCh) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for msg := range task.outputCh { - rtm.responseCh <- task.buildGlobalReduceResponse(msg) - } - // send the EOF response - rtm.responseCh <- task.buildEOFResponse() - }() - - // start the global reduce operation - task.globalReducer.GlobalReduce(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh) - // close the output channel after the global reduce operation is completed - close(task.outputCh) - // wait for the output channel reader to complete - wg.Wait() - // send a done signal - close(task.doneCh) - // delete the task from the tasks list - rtm.rw.Lock() - delete(rtm.tasks, key) - rtm.rw.Unlock() - }() - - // send the datum to the task if the payload is not nil - // also update the latest watermark - if request.Payload != nil { - task.inputCh <- buildDatum(request) - task.latestWatermark.Store(request.Payload.Watermark.AsTime()) - } - - return nil -} - -// AppendToTask writes the message to the reduce task. -// If the reduce task is not found, it will create a new reduce task and start the reduce operation. -func (rtm *globalReduceTaskManager) AppendToTask(ctx context.Context, request *v1.GlobalReduceRequest) error { - if len(request.Operation.KeyedWindows) != 1 { - return fmt.Errorf("append operation error: invalid number of windows") - } - - rtm.rw.RLock() - task, ok := rtm.tasks[generateKey(request.Operation.KeyedWindows[0])] - rtm.rw.RUnlock() - - if !ok { - return rtm.CreateTask(ctx, request) - } - - // send the datum to the task if the payload is not nil - if request.Payload != nil { - task.inputCh <- buildDatum(request) - task.latestWatermark.Store(request.Payload.Watermark.AsTime()) - } - - return nil -} - -// CloseTask closes the reduce task input channel. The reduce task will be closed when all the messages are processed. -func (rtm *globalReduceTaskManager) CloseTask(request *v1.GlobalReduceRequest) { - rtm.rw.RLock() - tasksToBeClosed := make([]*globalReduceTask, 0, len(request.Operation.KeyedWindows)) - for _, window := range request.Operation.KeyedWindows { - key := generateKey(window) - task, ok := rtm.tasks[key] - if ok { - tasksToBeClosed = append(tasksToBeClosed, task) - } - } - rtm.rw.RUnlock() - - // close the input channel for all the tasks - for _, task := range tasksToBeClosed { - close(task.inputCh) - } -} - -// OutputChannel returns the output channel for the reduce task manager to read the results. -func (rtm *globalReduceTaskManager) OutputChannel() <-chan *v1.GlobalReduceResponse { - return rtm.responseCh -} - -// WaitAll waits for all the reduce tasks to complete. -func (rtm *globalReduceTaskManager) WaitAll() { - rtm.rw.RLock() - tasks := make([]*globalReduceTask, 0, len(rtm.tasks)) - for _, task := range rtm.tasks { - tasks = append(tasks, task) - } - rtm.rw.RUnlock() - - for _, task := range tasks { - <-task.doneCh - } - // after all the tasks are completed, close the output channel - close(rtm.responseCh) -} - -// CloseAll closes all the reduce tasks. -func (rtm *globalReduceTaskManager) CloseAll() { - rtm.rw.Lock() - tasks := make([]*globalReduceTask, 0, len(rtm.tasks)) - for _, task := range rtm.tasks { - tasks = append(tasks, task) - } - rtm.rw.Unlock() - - for _, task := range tasks { - close(task.inputCh) - } -} - -func generateKey(keyedWindow *v1.KeyedWindow) string { - return fmt.Sprintf("%d:%d:%s", - keyedWindow.GetStart().AsTime().UnixMilli(), - keyedWindow.GetEnd().AsTime().UnixMilli(), - strings.Join(keyedWindow.GetKeys(), delimiter)) -} - -func buildDatum(request *v1.GlobalReduceRequest) Datum { - return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) -} diff --git a/pkg/globalreducer/types.go b/pkg/globalreducer/types.go deleted file mode 100644 index c49df3f1..00000000 --- a/pkg/globalreducer/types.go +++ /dev/null @@ -1,30 +0,0 @@ -package globalreducer - -import "time" - -// handlerDatum implements the Datum interface and is used in the reduce functions. -type handlerDatum struct { - value []byte - eventTime time.Time - watermark time.Time -} - -func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time) Datum { - return &handlerDatum{ - value: value, - eventTime: eventTime, - watermark: watermark, - } -} - -func (h *handlerDatum) Value() []byte { - return h.value -} - -func (h *handlerDatum) EventTime() time.Time { - return h.eventTime -} - -func (h *handlerDatum) Watermark() time.Time { - return h.watermark -} diff --git a/pkg/reduceStreamer/doc.go b/pkg/reducestreamer/doc.go similarity index 100% rename from pkg/reduceStreamer/doc.go rename to pkg/reducestreamer/doc.go diff --git a/pkg/reduceStreamer/examples/counter/Dockerfile b/pkg/reducestreamer/examples/counter/Dockerfile similarity index 100% rename from pkg/reduceStreamer/examples/counter/Dockerfile rename to pkg/reducestreamer/examples/counter/Dockerfile diff --git a/pkg/reduceStreamer/examples/counter/Makefile b/pkg/reducestreamer/examples/counter/Makefile similarity index 100% rename from pkg/reduceStreamer/examples/counter/Makefile rename to pkg/reducestreamer/examples/counter/Makefile diff --git a/pkg/reduceStreamer/examples/counter/README.md b/pkg/reducestreamer/examples/counter/README.md similarity index 100% rename from pkg/reduceStreamer/examples/counter/README.md rename to pkg/reducestreamer/examples/counter/README.md diff --git a/pkg/reduceStreamer/examples/counter/main.go b/pkg/reducestreamer/examples/counter/main.go similarity index 100% rename from pkg/reduceStreamer/examples/counter/main.go rename to pkg/reducestreamer/examples/counter/main.go diff --git a/pkg/reduceStreamer/examples/sum/Dockerfile b/pkg/reducestreamer/examples/sum/Dockerfile similarity index 100% rename from pkg/reduceStreamer/examples/sum/Dockerfile rename to pkg/reducestreamer/examples/sum/Dockerfile diff --git a/pkg/reduceStreamer/examples/sum/Makefile b/pkg/reducestreamer/examples/sum/Makefile similarity index 100% rename from pkg/reduceStreamer/examples/sum/Makefile rename to pkg/reducestreamer/examples/sum/Makefile diff --git a/pkg/reduceStreamer/examples/sum/README.md b/pkg/reducestreamer/examples/sum/README.md similarity index 100% rename from pkg/reduceStreamer/examples/sum/README.md rename to pkg/reducestreamer/examples/sum/README.md diff --git a/pkg/reduceStreamer/examples/sum/main.go b/pkg/reducestreamer/examples/sum/main.go similarity index 100% rename from pkg/reduceStreamer/examples/sum/main.go rename to pkg/reducestreamer/examples/sum/main.go diff --git a/pkg/reduceStreamer/interface.go b/pkg/reducestreamer/interface.go similarity index 100% rename from pkg/reduceStreamer/interface.go rename to pkg/reducestreamer/interface.go diff --git a/pkg/reduceStreamer/message.go b/pkg/reducestreamer/message.go similarity index 100% rename from pkg/reduceStreamer/message.go rename to pkg/reducestreamer/message.go diff --git a/pkg/reduceStreamer/options.go b/pkg/reducestreamer/options.go similarity index 100% rename from pkg/reduceStreamer/options.go rename to pkg/reducestreamer/options.go diff --git a/pkg/reduceStreamer/options_test.go b/pkg/reducestreamer/options_test.go similarity index 100% rename from pkg/reduceStreamer/options_test.go rename to pkg/reducestreamer/options_test.go diff --git a/pkg/reduceStreamer/server.go b/pkg/reducestreamer/server.go similarity index 100% rename from pkg/reduceStreamer/server.go rename to pkg/reducestreamer/server.go diff --git a/pkg/reduceStreamer/server_test.go b/pkg/reducestreamer/server_test.go similarity index 100% rename from pkg/reduceStreamer/server_test.go rename to pkg/reducestreamer/server_test.go diff --git a/pkg/reduceStreamer/service.go b/pkg/reducestreamer/service.go similarity index 100% rename from pkg/reduceStreamer/service.go rename to pkg/reducestreamer/service.go diff --git a/pkg/reduceStreamer/service_test.go b/pkg/reducestreamer/service_test.go similarity index 100% rename from pkg/reduceStreamer/service_test.go rename to pkg/reducestreamer/service_test.go diff --git a/pkg/reduceStreamer/task_manager.go b/pkg/reducestreamer/task_manager.go similarity index 100% rename from pkg/reduceStreamer/task_manager.go rename to pkg/reducestreamer/task_manager.go diff --git a/pkg/reduceStreamer/types.go b/pkg/reducestreamer/types.go similarity index 100% rename from pkg/reduceStreamer/types.go rename to pkg/reducestreamer/types.go From 2c9517d09d792567889a8e51f2fe2f34afe8f1c9 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Fri, 8 Dec 2023 12:40:18 +0530 Subject: [PATCH 20/27] clean up Signed-off-by: Yashash H L --- pkg/apis/proto/reduce/v1/reduce.pb.go | 83 ++++++++--------- pkg/apis/proto/reduce/v1/reduce.proto | 3 +- .../sessionreduce/v1/sessionreduce.pb.go | 89 ++++++++----------- .../sessionreduce/v1/sessionreduce.proto | 3 +- pkg/mapper/examples/even_odd/main.go | 2 +- pkg/mapper/examples/flatmap/main.go | 2 +- .../examples/flatmap_stream/main.go | 2 +- pkg/reducer/message.go | 2 +- pkg/reducer/service.go | 2 +- pkg/reducer/task_manager.go | 33 +++---- pkg/reducestreamer/doc.go | 4 +- pkg/reducestreamer/examples/counter/README.md | 2 +- pkg/reducestreamer/examples/counter/main.go | 4 +- pkg/reducestreamer/examples/sum/README.md | 2 +- pkg/reducestreamer/examples/sum/main.go | 2 +- pkg/reducestreamer/interface.go | 10 +-- pkg/reducestreamer/server.go | 8 +- pkg/reducestreamer/server_test.go | 2 +- pkg/reducestreamer/service.go | 10 +-- pkg/reducestreamer/service_test.go | 10 +-- pkg/reducestreamer/task_manager.go | 13 +-- pkg/reducestreamer/types.go | 4 +- pkg/sessionreducer/doc.go | 4 +- pkg/sessionreducer/examples/counter/main.go | 5 +- pkg/sessionreducer/message.go | 2 +- pkg/sessionreducer/task_manager.go | 24 ++--- pkg/sessionreducer/types.go | 2 +- 27 files changed, 139 insertions(+), 190 deletions(-) diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index 6455b93a..2f9b365c 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -310,7 +310,7 @@ func (x *ReadyResponse) GetReady() bool { } // WindowOperation represents a window operation. -// for fixed and sliding windows, OPEN, APPEND and CLOSE events are sent. +// For Aligned windows, OPEN, APPEND and CLOSE events are sent. type ReduceRequest_WindowOperation struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -444,10 +444,9 @@ type ReduceResponse_Result struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` } func (x *ReduceResponse_Result) Reset() { @@ -503,13 +502,6 @@ func (x *ReduceResponse_Result) GetTags() []string { return nil } -func (x *ReduceResponse_Result) GetEventTime() *timestamppb.Timestamp { - if x != nil { - return x.EventTime - } - return nil -} - var File_pkg_apis_proto_reduce_v1_reduce_proto protoreflect.FileDescriptor var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ @@ -559,7 +551,7 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, - 0x22, 0x8b, 0x02, 0x0a, 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x22, 0xcf, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, @@ -567,31 +559,27 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, - 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x25, - 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, - 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x18, 0x2e, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, - 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, + 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, + 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, + 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -631,16 +619,15 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs = []int32{ 2, // 7: reduce.v1.ReduceRequest.WindowOperation.windows:type_name -> reduce.v1.Window 8, // 8: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 9: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp - 8, // 10: reduce.v1.ReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp - 1, // 11: reduce.v1.Reduce.ReduceFn:input_type -> reduce.v1.ReduceRequest - 9, // 12: reduce.v1.Reduce.IsReady:input_type -> google.protobuf.Empty - 3, // 13: reduce.v1.Reduce.ReduceFn:output_type -> reduce.v1.ReduceResponse - 4, // 14: reduce.v1.Reduce.IsReady:output_type -> reduce.v1.ReadyResponse - 13, // [13:15] is the sub-list for method output_type - 11, // [11:13] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 1, // 10: reduce.v1.Reduce.ReduceFn:input_type -> reduce.v1.ReduceRequest + 9, // 11: reduce.v1.Reduce.IsReady:input_type -> google.protobuf.Empty + 3, // 12: reduce.v1.Reduce.ReduceFn:output_type -> reduce.v1.ReduceResponse + 4, // 13: reduce.v1.Reduce.IsReady:output_type -> reduce.v1.ReadyResponse + 12, // [12:14] is the sub-list for method output_type + 10, // [10:12] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_pkg_apis_proto_reduce_v1_reduce_proto_init() } diff --git a/pkg/apis/proto/reduce/v1/reduce.proto b/pkg/apis/proto/reduce/v1/reduce.proto index 5506bc65..2b2e071d 100644 --- a/pkg/apis/proto/reduce/v1/reduce.proto +++ b/pkg/apis/proto/reduce/v1/reduce.proto @@ -21,7 +21,7 @@ service Reduce { */ message ReduceRequest { // WindowOperation represents a window operation. - // for fixed and sliding windows, OPEN, APPEND and CLOSE events are sent. + // For Aligned windows, OPEN, APPEND and CLOSE events are sent. message WindowOperation { enum Event { OPEN = 0; @@ -62,7 +62,6 @@ message ReduceResponse { repeated string keys = 1; bytes value = 2; repeated string tags = 3; - google.protobuf.Timestamp event_time = 4; } Result result = 1; diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index 85b185a2..8a672b83 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -324,7 +324,7 @@ func (x *ReadyResponse) GetReady() bool { } // WindowOperation represents a window operation. -// it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. +// For Aligned window values can be one of OPEN, CLOSE, EXPAND, MERGE and APPEND. type SessionReduceRequest_WindowOperation struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -458,10 +458,9 @@ type SessionReduceResponse_Result struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` } func (x *SessionReduceResponse_Result) Reset() { @@ -517,13 +516,6 @@ func (x *SessionReduceResponse_Result) GetTags() []string { return nil } -func (x *SessionReduceResponse_Result) GetEventTime() *timestamppb.Timestamp { - if x != nil { - return x.EventTime - } - return nil -} - var File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto protoreflect.FileDescriptor var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ @@ -581,7 +573,7 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xb6, 0x02, + 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xfa, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, @@ -593,34 +585,30 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x0b, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, - 0x4f, 0x46, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, - 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x39, 0x0a, 0x0a, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xbb, 0x01, - 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, - 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, - 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, - 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, - 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, - 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4f, 0x46, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, + 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, + 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, + 0x79, 0x32, 0xbb, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, + 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, + 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, + 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, + 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, + 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, + 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -660,16 +648,15 @@ var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ 1, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.keyedWindows:type_name -> sessionreduce.v1.KeyedWindow 8, // 8: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp - 8, // 10: sessionreduce.v1.SessionReduceResponse.Result.event_time:type_name -> google.protobuf.Timestamp - 2, // 11: sessionreduce.v1.SessionReduce.SessionReduceFn:input_type -> sessionreduce.v1.SessionReduceRequest - 9, // 12: sessionreduce.v1.SessionReduce.IsReady:input_type -> google.protobuf.Empty - 3, // 13: sessionreduce.v1.SessionReduce.SessionReduceFn:output_type -> sessionreduce.v1.SessionReduceResponse - 4, // 14: sessionreduce.v1.SessionReduce.IsReady:output_type -> sessionreduce.v1.ReadyResponse - 13, // [13:15] is the sub-list for method output_type - 11, // [11:13] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 2, // 10: sessionreduce.v1.SessionReduce.SessionReduceFn:input_type -> sessionreduce.v1.SessionReduceRequest + 9, // 11: sessionreduce.v1.SessionReduce.IsReady:input_type -> google.protobuf.Empty + 3, // 12: sessionreduce.v1.SessionReduce.SessionReduceFn:output_type -> sessionreduce.v1.SessionReduceResponse + 4, // 13: sessionreduce.v1.SessionReduce.IsReady:output_type -> sessionreduce.v1.ReadyResponse + 12, // [12:14] is the sub-list for method output_type + 10, // [10:12] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() } diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto index b09407d8..07d354e6 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -30,7 +30,7 @@ message KeyedWindow { */ message SessionReduceRequest { // WindowOperation represents a window operation. - // it can be one of OPEN, CLOSE, EXPAND, MERGE, APPEND. + // For Aligned window values can be one of OPEN, CLOSE, EXPAND, MERGE and APPEND. message WindowOperation { enum Event { OPEN = 0; @@ -65,7 +65,6 @@ message SessionReduceResponse { repeated string keys = 1; bytes value = 2; repeated string tags = 3; - google.protobuf.Timestamp event_time = 4; } Result result = 1; diff --git a/pkg/mapper/examples/even_odd/main.go b/pkg/mapper/examples/even_odd/main.go index ffec9ef2..92fd0040 100644 --- a/pkg/mapper/examples/even_odd/main.go +++ b/pkg/mapper/examples/even_odd/main.go @@ -13,7 +13,7 @@ type EvenOdd struct { func (e *EvenOdd) Map(ctx context.Context, keys []string, d mapper.Datum) mapper.Messages { msg := d.Value() - _ = d.EventTime() // Operation time is available + _ = d.EventTime() // Event time is available _ = d.Watermark() // Watermark is available // If msg is not an integer, drop it, otherwise return it with "even" or "odd" key. if num, err := strconv.Atoi(string(msg)); err != nil { diff --git a/pkg/mapper/examples/flatmap/main.go b/pkg/mapper/examples/flatmap/main.go index b6ad0400..2a5fa18e 100644 --- a/pkg/mapper/examples/flatmap/main.go +++ b/pkg/mapper/examples/flatmap/main.go @@ -10,7 +10,7 @@ import ( func mapFn(_ context.Context, keys []string, d mapper.Datum) mapper.Messages { msg := d.Value() - _ = d.EventTime() // Operation time is available + _ = d.EventTime() // Event time is available _ = d.Watermark() // Watermark is available // Split the msg into an array with comma. strs := strings.Split(string(msg), ",") diff --git a/pkg/mapstreamer/examples/flatmap_stream/main.go b/pkg/mapstreamer/examples/flatmap_stream/main.go index f333be48..a72ab3f3 100644 --- a/pkg/mapstreamer/examples/flatmap_stream/main.go +++ b/pkg/mapstreamer/examples/flatmap_stream/main.go @@ -15,7 +15,7 @@ type FlatMap struct { func (f *FlatMap) MapStream(ctx context.Context, keys []string, d mapstreamer.Datum, messageCh chan<- mapstreamer.Message) { defer close(messageCh) msg := d.Value() - _ = d.EventTime() // Operation time is available + _ = d.EventTime() // Event time is available _ = d.Watermark() // Watermark is available // Split the msg into an array with comma. strs := strings.Split(string(msg), ",") diff --git a/pkg/reducer/message.go b/pkg/reducer/message.go index 44ec0c2e..80209972 100644 --- a/pkg/reducer/message.go +++ b/pkg/reducer/message.go @@ -6,7 +6,7 @@ var ( DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ ) -// Message is used to wrap the data return by reduceStream function +// Message is used to wrap the data return by reduce function type Message struct { value []byte keys []string diff --git a/pkg/reducer/service.go b/pkg/reducer/service.go index f4dc53cc..cbe048d9 100644 --- a/pkg/reducer/service.go +++ b/pkg/reducer/service.go @@ -68,7 +68,7 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { return recvErr } - // for fixed and sliding, its just open or append operation + // for Aligned windows, its just open or append operation // close signal will be sent to all the reducers when grpc // input stream gets EOF. switch d.Operation.Event { diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index 0a685c3e..836527f0 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -5,15 +5,12 @@ import ( "fmt" "strings" "sync" - "time" - - "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" ) -// reduceStreamTask represents a task for a performing reduceStream operation. -type reduceStreamTask struct { +// reduceTask represents a task for a performing reduceStream operation. +type reduceTask struct { keys []string window *v1.Window reducer Reducer @@ -23,14 +20,13 @@ type reduceStreamTask struct { } // buildReduceResponse builds the reduce response from the messages. -func (rt *reduceStreamTask) buildReduceResponse(message Message) *v1.ReduceResponse { +func (rt *reduceTask) buildReduceResponse(message Message) *v1.ReduceResponse { response := &v1.ReduceResponse{ Result: &v1.ReduceResponse_Result{ - Keys: message.Keys(), - Value: message.Value(), - Tags: message.Tags(), - EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), }, Window: rt.window, } @@ -38,11 +34,8 @@ func (rt *reduceStreamTask) buildReduceResponse(message Message) *v1.ReduceRespo return response } -func (rt *reduceStreamTask) buildEOFResponse() *v1.ReduceResponse { +func (rt *reduceTask) buildEOFResponse() *v1.ReduceResponse { response := &v1.ReduceResponse{ - Result: &v1.ReduceResponse_Result{ - EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), - }, Window: rt.window, EOF: true, } @@ -51,7 +44,7 @@ func (rt *reduceStreamTask) buildEOFResponse() *v1.ReduceResponse { } // uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. -func (rt *reduceStreamTask) uniqueKey() string { +func (rt *reduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", rt.window.GetStart().AsTime().UnixMilli(), rt.window.GetEnd().AsTime().UnixMilli(), @@ -61,7 +54,7 @@ func (rt *reduceStreamTask) uniqueKey() string { // reduceTaskManager manages the reduce tasks for a reduce operation. type reduceTaskManager struct { reducer Reducer - tasks map[string]*reduceStreamTask + tasks map[string]*reduceTask responseCh chan *v1.ReduceResponse rw sync.RWMutex } @@ -69,7 +62,7 @@ type reduceTaskManager struct { func newReduceTaskManager(reducer Reducer) *reduceTaskManager { return &reduceTaskManager{ reducer: reducer, - tasks: make(map[string]*reduceStreamTask), + tasks: make(map[string]*reduceTask), responseCh: make(chan *v1.ReduceResponse), } } @@ -84,7 +77,7 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), request.Operation.Windows[0].GetEnd().AsTime())) - task := &reduceStreamTask{ + task := &reduceTask{ keys: request.GetPayload().GetKeys(), window: request.Operation.Windows[0], inputCh: make(chan Datum), @@ -147,7 +140,7 @@ func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { // WaitAll waits for all the reduce tasks to complete. func (rtm *reduceTaskManager) WaitAll() { rtm.rw.RLock() - tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) + tasks := make([]*reduceTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } @@ -163,7 +156,7 @@ func (rtm *reduceTaskManager) WaitAll() { // CloseAll closes all the reduce tasks. func (rtm *reduceTaskManager) CloseAll() { rtm.rw.Lock() - tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) + tasks := make([]*reduceTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } diff --git a/pkg/reducestreamer/doc.go b/pkg/reducestreamer/doc.go index c7160045..046c779e 100644 --- a/pkg/reducestreamer/doc.go +++ b/pkg/reducestreamer/doc.go @@ -1,5 +1,5 @@ -// Package reduceStreamer implements the server code for reduce operation. +// Package reduceStreamer implements the server code for reduceStream operation. -// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/ +// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducestreamer/examples/ package reducestreamer diff --git a/pkg/reducestreamer/examples/counter/README.md b/pkg/reducestreamer/examples/counter/README.md index 5a72f269..8ba9bd9f 100644 --- a/pkg/reducestreamer/examples/counter/README.md +++ b/pkg/reducestreamer/examples/counter/README.md @@ -1,3 +1,3 @@ # Counter -An example User Defined Function that counts the number of events. +An example User Defined Function that count the incoming events and output the count every 10 events. diff --git a/pkg/reducestreamer/examples/counter/main.go b/pkg/reducestreamer/examples/counter/main.go index 22de7e55..17e54259 100644 --- a/pkg/reducestreamer/examples/counter/main.go +++ b/pkg/reducestreamer/examples/counter/main.go @@ -7,6 +7,8 @@ import ( "github.com/numaproj/numaflow-go/pkg/reducestreamer" ) +// reduceCounter is a ReduceStreamer that count the incoming events and output the count every 10 events. +// The output message is the count of the events. func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducestreamer.Datum, outputCh chan<- reducestreamer.Message, md reducestreamer.Metadata) { // count the incoming events var resultKeys = keys @@ -25,5 +27,5 @@ func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducestream } func main() { - reducestreamer.NewServer(reducestreamer.ReducerFunc(reduceCounter)).Start(context.Background()) + reducestreamer.NewServer(reducestreamer.ReduceStreamerFunc(reduceCounter)).Start(context.Background()) } diff --git a/pkg/reducestreamer/examples/sum/README.md b/pkg/reducestreamer/examples/sum/README.md index 62af7a17..1029db38 100644 --- a/pkg/reducestreamer/examples/sum/README.md +++ b/pkg/reducestreamer/examples/sum/README.md @@ -1,3 +1,3 @@ # Sum -This is a User Defined Function example which receives a stream of numbers and returns the sum of the numbers. +This is a User Defined Function example which sum up the values for the given keys and output the sum when the sum is greater than 100 diff --git a/pkg/reducestreamer/examples/sum/main.go b/pkg/reducestreamer/examples/sum/main.go index 5c5e3038..f182ba16 100644 --- a/pkg/reducestreamer/examples/sum/main.go +++ b/pkg/reducestreamer/examples/sum/main.go @@ -9,7 +9,7 @@ import ( "github.com/numaproj/numaflow-go/pkg/reducestreamer" ) -// Sum is a reducestreamer that sum up the values for the given keys +// Sum is a reducestreamer that sum up the values for the given keys and output the sum when the sum is greater than 100. type Sum struct { } diff --git a/pkg/reducestreamer/interface.go b/pkg/reducestreamer/interface.go index 8d0a01c8..bc3d2db6 100644 --- a/pkg/reducestreamer/interface.go +++ b/pkg/reducestreamer/interface.go @@ -12,7 +12,7 @@ type Datum interface { Watermark() time.Time } -// Metadata contains methods to get the metadata for the reduce operation. +// Metadata contains methods to get the metadata for the reduceStream operation. type Metadata interface { IntervalWindow() IntervalWindow } @@ -28,10 +28,10 @@ type ReduceStreamer interface { ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) } -// ReducerFunc is a utility type used to convert a ReduceStream function to a ReduceStreamer. -type ReducerFunc func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) +// ReduceStreamerFunc is a utility type used to convert a ReduceStream function to a ReduceStreamer. +type ReduceStreamerFunc func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) -// ReduceStream implements the function of reduce function. -func (rf ReducerFunc) ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { +// ReduceStream implements the function of ReduceStream function. +func (rf ReduceStreamerFunc) ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { rf(ctx, keys, inputCh, outputCh, md) } diff --git a/pkg/reducestreamer/server.go b/pkg/reducestreamer/server.go index af85ce9c..21f0c951 100644 --- a/pkg/reducestreamer/server.go +++ b/pkg/reducestreamer/server.go @@ -11,7 +11,7 @@ import ( "github.com/numaproj/numaflow-go/pkg/shared" ) -// server is a reduce gRPC server. +// server is a reduceStream gRPC server. type server struct { svc *Service opts *options @@ -25,12 +25,12 @@ func NewServer(r ReduceStreamer, inputOptions ...Option) numaflow.Server { } s := new(server) s.svc = new(Service) - s.svc.reducerHandle = r + s.svc.reduceStreamerHandle = r s.opts = opts return s } -// Start starts the reduce gRPC server. +// Start starts the reduceStream gRPC server. func (r *server) Start(ctx context.Context) error { ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -48,7 +48,7 @@ func (r *server) Start(ctx context.Context) error { grpcServer := shared.CreateGRPCServer(r.opts.maxMessageSize) defer grpcServer.GracefulStop() - // register the reduce service + // register the reduceStream service reducepb.RegisterReduceServer(grpcServer, r.svc) // start the grpc server diff --git a/pkg/reducestreamer/server_test.go b/pkg/reducestreamer/server_test.go index 06d0ddb4..ff54d18d 100644 --- a/pkg/reducestreamer/server_test.go +++ b/pkg/reducestreamer/server_test.go @@ -21,7 +21,7 @@ func TestReduceServer_Start(t *testing.T) { _ = os.RemoveAll(serverInfoFile.Name()) }() - var reduceHandler = ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + var reduceHandler = ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) diff --git a/pkg/reducestreamer/service.go b/pkg/reducestreamer/service.go index ccd5bb79..8c2e8728 100644 --- a/pkg/reducestreamer/service.go +++ b/pkg/reducestreamer/service.go @@ -21,10 +21,10 @@ const ( delimiter = ":" ) -// Service implements the proto gen server interface and contains the reduce operation handler. +// Service implements the proto gen server interface and contains the reduceStream operation handler. type Service struct { reducepb.UnimplementedReduceServer - reducerHandle ReduceStreamer + reduceStreamerHandle ReduceStreamer } // IsReady returns true to indicate the gRPC connection is ready. @@ -32,7 +32,7 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*reducepb.ReadyResp return &reducepb.ReadyResponse{Ready: true}, nil } -// ReduceFn applies a reduce function to a request stream and returns a list of results. +// ReduceFn applies a reduce function to a request stream and streams the results. func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { var ( err error @@ -40,7 +40,7 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { g errgroup.Group ) - taskManager := newReduceTaskManager(fs.reducerHandle) + taskManager := newReduceTaskManager(fs.reduceStreamerHandle) // err group for the go routine which reads from the output channel and sends to the stream g.Go(func() error { @@ -68,7 +68,7 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { return recvErr } - // for fixed and sliding, its just open or append operation + // for Aligned, its just open or append operation // close signal will be sent to all the reducers when grpc // input stream gets EOF. switch d.Operation.Event { diff --git a/pkg/reducestreamer/service_test.go b/pkg/reducestreamer/service_test.go index c8bafc6b..f754f48d 100644 --- a/pkg/reducestreamer/service_test.go +++ b/pkg/reducestreamer/service_test.go @@ -62,7 +62,7 @@ func TestService_ReduceFn(t *testing.T) { }{ { name: "reduce_fn_forward_msg_same_keys", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) @@ -145,7 +145,7 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_multiple_keys", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) @@ -308,7 +308,7 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_forward_to_all", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) @@ -389,7 +389,7 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_drop_msg", - handler: ReducerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) @@ -474,7 +474,7 @@ func TestService_ReduceFn(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := &Service{ - reducerHandle: tt.handler, + reduceStreamerHandle: tt.handler, } // here's a trick for testing: // because we are not using gRPC, we directly set a new incoming ctx diff --git a/pkg/reducestreamer/task_manager.go b/pkg/reducestreamer/task_manager.go index cf5a44fc..7d20fb09 100644 --- a/pkg/reducestreamer/task_manager.go +++ b/pkg/reducestreamer/task_manager.go @@ -5,9 +5,6 @@ import ( "fmt" "strings" "sync" - "time" - - "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" ) @@ -27,10 +24,9 @@ func (rt *reduceStreamTask) buildReduceResponse(message Message) *v1.ReduceRespo response := &v1.ReduceResponse{ Result: &v1.ReduceResponse_Result{ - Keys: message.Keys(), - Value: message.Value(), - Tags: message.Tags(), - EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), }, Window: rt.window, } @@ -40,9 +36,6 @@ func (rt *reduceStreamTask) buildReduceResponse(message Message) *v1.ReduceRespo func (rt *reduceStreamTask) buildEOFResponse() *v1.ReduceResponse { response := &v1.ReduceResponse{ - Result: &v1.ReduceResponse_Result{ - EventTime: timestamppb.New(rt.window.End.AsTime().Add(-1 * time.Millisecond)), - }, Window: rt.window, EOF: true, } diff --git a/pkg/reducestreamer/types.go b/pkg/reducestreamer/types.go index fac5b63a..1f19c27a 100644 --- a/pkg/reducestreamer/types.go +++ b/pkg/reducestreamer/types.go @@ -2,7 +2,7 @@ package reducestreamer import "time" -// handlerDatum implements the Datum interface and is used in the reduce functions. +// handlerDatum implements the Datum interface and is used in the reduceStream functions. type handlerDatum struct { value []byte eventTime time.Time @@ -51,7 +51,7 @@ func (i *intervalWindow) EndTime() time.Time { return i.endTime } -// metadata implements Metadata interface which will be passed to reduce function. +// metadata implements Metadata interface which will be passed to reduceStream function. type metadata struct { intervalWindow IntervalWindow } diff --git a/pkg/sessionreducer/doc.go b/pkg/sessionreducer/doc.go index dc190ed2..ddfbc457 100644 --- a/pkg/sessionreducer/doc.go +++ b/pkg/sessionreducer/doc.go @@ -1,5 +1,5 @@ -// Package reducer implements the server code for reduce operation. +// Package sessionreducer implements the server code for sessionReduce operation. -// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/ +// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/sessionreducer/examples/ package sessionreducer diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go index 83d547f3..7dc11aca 100644 --- a/pkg/sessionreducer/examples/counter/main.go +++ b/pkg/sessionreducer/examples/counter/main.go @@ -11,26 +11,23 @@ import ( "github.com/numaproj/numaflow-go/pkg/sessionreducer" ) +// Counter is a simple session reducer which counts the number of events in a session. type Counter struct { count *atomic.Int32 } func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan sessionreducer.Datum, outputCh chan<- sessionreducer.Message) { - log.Println("SessionReduce invoked") for range input { c.count.Inc() } - log.Println("SessionReduce done") outputCh <- sessionreducer.NewMessage([]byte(fmt.Sprintf("%d", c.count.Load()))).WithKeys(keys) } func (c *Counter) Accumulator(ctx context.Context) []byte { - log.Println("Accumulator invoked") return []byte(strconv.Itoa(int(c.count.Load()))) } func (c *Counter) MergeAccumulator(ctx context.Context, accumulator []byte) { - log.Println("MergeAccumulator invoked") val, err := strconv.Atoi(string(accumulator)) if err != nil { log.Println("unable to convert the accumulator value to int: ", err.Error()) diff --git a/pkg/sessionreducer/message.go b/pkg/sessionreducer/message.go index bfc69af0..8fcfe026 100644 --- a/pkg/sessionreducer/message.go +++ b/pkg/sessionreducer/message.go @@ -6,7 +6,7 @@ var ( DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ ) -// Message is used to wrap the data return by reduce functions +// Message is used to wrap the data return by SessionReduce functions type Message struct { value []byte keys []string diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index bfd3b279..e1d34b77 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -5,10 +5,8 @@ import ( "fmt" "strings" "sync" - "time" "go.uber.org/atomic" - "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" ) @@ -32,8 +30,6 @@ func (rt *sessionReduceTask) buildSessionReduceResponse(message Message) *v1.Ses Keys: message.Keys(), Value: message.Value(), Tags: message.Tags(), - // event time is the end time of the window - 1 millisecond - EventTime: timestamppb.New(rt.keyedWindow.GetEnd().AsTime().Add(-1 * time.Millisecond)), }, KeyedWindow: rt.keyedWindow, } @@ -44,10 +40,6 @@ func (rt *sessionReduceTask) buildSessionReduceResponse(message Message) *v1.Ses // buildEOFResponse builds the EOF response for the session reduce task. func (rt *sessionReduceTask) buildEOFResponse() *v1.SessionReduceResponse { response := &v1.SessionReduceResponse{ - Result: &v1.SessionReduceResponse_Result{ - // event time is the end time of the window - 1 millisecond - EventTime: timestamppb.New(rt.keyedWindow.GetEnd().AsTime().Add(-1 * time.Millisecond)), - }, KeyedWindow: rt.keyedWindow, EOF: true, } @@ -55,7 +47,7 @@ func (rt *sessionReduceTask) buildEOFResponse() *v1.SessionReduceResponse { return response } -// uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. +// uniqueKey returns the unique key for the session reduce task to be used in the task manager to identify the task. func (rt *sessionReduceTask) uniqueKey() string { return fmt.Sprintf("%d:%d:%s", rt.keyedWindow.GetStart().AsTime().UnixMilli(), @@ -63,7 +55,7 @@ func (rt *sessionReduceTask) uniqueKey() string { strings.Join(rt.keyedWindow.GetKeys(), delimiter)) } -// sessionReduceTaskManager manages the reduce tasks for a session reduce operation. +// sessionReduceTaskManager manages the tasks for a session reduce operation. type sessionReduceTaskManager struct { sessionReducerFactory CreateSessionReducer tasks map[string]*sessionReduceTask @@ -143,7 +135,7 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 } // AppendToTask writes the message to the reduce task. -// If the reduce task is not found, it will create a new reduce task and start the reduce operation. +// If the task is not found, it will create a new task and starts the session reduce operation. func (rtm *sessionReduceTaskManager) AppendToTask(ctx context.Context, request *v1.SessionReduceRequest) error { // for append operation, there should be exactly one keyedWindow @@ -155,7 +147,7 @@ func (rtm *sessionReduceTaskManager) AppendToTask(ctx context.Context, request * task, ok := rtm.tasks[generateKey(request.Operation.KeyedWindows[0])] rtm.rw.RUnlock() - // if the task is not found, create a new task and start the reduce operation + // if the task is not found, create a new task and start the session reduce operation if !ok { return rtm.CreateTask(ctx, request) } @@ -187,7 +179,7 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) } // MergeTasks merges the session reduce tasks. It will create a new task with the merged window and -// merges the accumulators from the other tasks. +// merges the accumulators from the other tasks to the merged task. func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { rtm.rw.Lock() mergedWindow := request.Operation.KeyedWindows[0] @@ -252,9 +244,9 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 return nil } -// ExpandTask expands the reduce task. It will update the keyedWindow of the reduce task +// ExpandTask expands session reduce task. It will update the keyedWindow of the task // expects request.Operation.KeyedWindows to have exactly two windows. The first is the old window and the second -// is the new window. +// is the new expanded window. func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { // for expand operation, there should be exactly two windows if len(request.Operation.KeyedWindows) != 2 { @@ -285,7 +277,7 @@ func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest return nil } -// OutputChannel returns the output channel for the reduce task manager to read the results. +// OutputChannel returns the output channel of the task manager to read the results. func (rtm *sessionReduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { return rtm.responseCh } diff --git a/pkg/sessionreducer/types.go b/pkg/sessionreducer/types.go index 68bbec9f..ee5c4966 100644 --- a/pkg/sessionreducer/types.go +++ b/pkg/sessionreducer/types.go @@ -2,7 +2,7 @@ package sessionreducer import "time" -// handlerDatum implements the Datum interface and is used in the reduce functions. +// handlerDatum implements the Datum interface and is used in the SessionReduce functions. type handlerDatum struct { value []byte eventTime time.Time From a8f7865b5ab3cde86168a2e7732bd7c73eaf611e Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Fri, 8 Dec 2023 16:29:41 +0530 Subject: [PATCH 21/27] fix tests Signed-off-by: Yashash H L --- pkg/reducer/service_test.go | 28 ++++++++++-------------- pkg/reducestreamer/service_test.go | 28 ++++++++++-------------- pkg/sessionreducer/service_test.go | 35 ++++++++++++------------------ 3 files changed, 36 insertions(+), 55 deletions(-) diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index c057bea3..91af6dc9 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -129,9 +129,8 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client_test"}, - Value: []byte(strconv.Itoa(60)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -266,9 +265,8 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client1_test"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -279,9 +277,8 @@ func TestService_ReduceFn(t *testing.T) { }, { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client2_test"}, - Value: []byte(strconv.Itoa(40)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -292,9 +289,8 @@ func TestService_ReduceFn(t *testing.T) { }, { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client3_test"}, - Value: []byte(strconv.Itoa(60)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client3_test"}, + Value: []byte(strconv.Itoa(60)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -375,8 +371,7 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Value: []byte(strconv.Itoa(60)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Value: []byte(strconv.Itoa(60)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -456,9 +451,8 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Tags: []string{DROP}, - Value: []byte{}, - EventTime: timestamppb.New(time.UnixMilli(119999)), + Tags: []string{DROP}, + Value: []byte{}, }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), diff --git a/pkg/reducestreamer/service_test.go b/pkg/reducestreamer/service_test.go index f754f48d..85e9c05a 100644 --- a/pkg/reducestreamer/service_test.go +++ b/pkg/reducestreamer/service_test.go @@ -129,9 +129,8 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client_test"}, - Value: []byte(strconv.Itoa(60)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -266,9 +265,8 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client1_test"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -279,9 +277,8 @@ func TestService_ReduceFn(t *testing.T) { }, { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client2_test"}, - Value: []byte(strconv.Itoa(40)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -292,9 +289,8 @@ func TestService_ReduceFn(t *testing.T) { }, { Result: &reducepb.ReduceResponse_Result{ - Keys: []string{"client3_test"}, - Value: []byte(strconv.Itoa(60)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client3_test"}, + Value: []byte(strconv.Itoa(60)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -375,8 +371,7 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Value: []byte(strconv.Itoa(60)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Value: []byte(strconv.Itoa(60)), }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -456,9 +451,8 @@ func TestService_ReduceFn(t *testing.T) { expected: []*reducepb.ReduceResponse{ { Result: &reducepb.ReduceResponse_Result{ - Tags: []string{DROP}, - Value: []byte{}, - EventTime: timestamppb.New(time.UnixMilli(119999)), + Tags: []string{DROP}, + Value: []byte{}, }, Window: &reducepb.Window{ Start: timestamppb.New(time.UnixMilli(60000)), diff --git a/pkg/sessionreducer/service_test.go b/pkg/sessionreducer/service_test.go index 14328cf2..922c8c47 100644 --- a/pkg/sessionreducer/service_test.go +++ b/pkg/sessionreducer/service_test.go @@ -170,9 +170,8 @@ func TestService_SessionReduceFn(t *testing.T) { expected: []*sessionreducepb.SessionReduceResponse{ { Result: &sessionreducepb.SessionReduceResponse_Result{ - Keys: []string{"client_test"}, - Value: []byte(strconv.Itoa(60)), - EventTime: timestamppb.New(time.UnixMilli(119999)), + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), }, KeyedWindow: &sessionreducepb.KeyedWindow{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -300,9 +299,8 @@ func TestService_SessionReduceFn(t *testing.T) { expected: []*sessionreducepb.SessionReduceResponse{ { Result: &sessionreducepb.SessionReduceResponse_Result{ - Keys: []string{"client1_test"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.UnixMilli(74999)), + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), }, KeyedWindow: &sessionreducepb.KeyedWindow{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -314,9 +312,8 @@ func TestService_SessionReduceFn(t *testing.T) { }, { Result: &sessionreducepb.SessionReduceResponse_Result{ - Keys: []string{"client2_test"}, - Value: []byte(strconv.Itoa(40)), - EventTime: timestamppb.New(time.UnixMilli(78999)), + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), }, KeyedWindow: &sessionreducepb.KeyedWindow{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -470,9 +467,8 @@ func TestService_SessionReduceFn(t *testing.T) { expected: []*sessionreducepb.SessionReduceResponse{ { Result: &sessionreducepb.SessionReduceResponse_Result{ - Keys: []string{"client1_test"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.UnixMilli(84999)), + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), }, KeyedWindow: &sessionreducepb.KeyedWindow{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -484,9 +480,8 @@ func TestService_SessionReduceFn(t *testing.T) { }, { Result: &sessionreducepb.SessionReduceResponse_Result{ - Keys: []string{"client2_test"}, - Value: []byte(strconv.Itoa(40)), - EventTime: timestamppb.New(time.UnixMilli(87999)), + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), }, KeyedWindow: &sessionreducepb.KeyedWindow{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -728,9 +723,8 @@ func TestService_SessionReduceFn(t *testing.T) { expected: []*sessionreducepb.SessionReduceResponse{ { Result: &sessionreducepb.SessionReduceResponse_Result{ - Keys: []string{"client1_test"}, - Value: []byte(strconv.Itoa(40)), - EventTime: timestamppb.New(time.UnixMilli(94999)), + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(40)), }, KeyedWindow: &sessionreducepb.KeyedWindow{ Start: timestamppb.New(time.UnixMilli(60000)), @@ -742,9 +736,8 @@ func TestService_SessionReduceFn(t *testing.T) { }, { Result: &sessionreducepb.SessionReduceResponse_Result{ - Keys: []string{"client2_test"}, - Value: []byte(strconv.Itoa(80)), - EventTime: timestamppb.New(time.UnixMilli(97999)), + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(80)), }, KeyedWindow: &sessionreducepb.KeyedWindow{ Start: timestamppb.New(time.UnixMilli(60000)), From d7512d864de10229d8c4edccc2e37c78613de455 Mon Sep 17 00:00:00 2001 From: Vigith Maurice Date: Fri, 8 Dec 2023 08:36:55 -0800 Subject: [PATCH 22/27] chore: code review Signed-off-by: Vigith Maurice --- pkg/reducer/task_manager.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index 836527f0..69576ac6 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -51,7 +51,7 @@ func (rt *reduceTask) uniqueKey() string { strings.Join(rt.keys, delimiter)) } -// reduceTaskManager manages the reduce tasks for a reduce operation. +// reduceTaskManager manages the reduce tasks for a reduce operation. type reduceTaskManager struct { reducer Reducer tasks map[string]*reduceTask @@ -67,13 +67,12 @@ func newReduceTaskManager(reducer Reducer) *reduceTaskManager { } } -// CreateTask creates a new reduce task and starts the reduce operation. +// CreateTask creates a new reduce task and starts the reduce operation. func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest) error { if len(request.Operation.Windows) != 1 { return fmt.Errorf("create operation error: invalid number of windows") } - rtm.rw.Lock() md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), request.Operation.Windows[0].GetEnd().AsTime())) @@ -86,8 +85,9 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce } key := task.uniqueKey() - rtm.tasks[key] = task + rtm.rw.Lock() + rtm.tasks[key] = task rtm.rw.Unlock() go func() { @@ -118,8 +118,9 @@ func (rtm *reduceTaskManager) AppendToTask(ctx context.Context, request *v1.Redu return fmt.Errorf("append operation error: invalid number of windows") } - rtm.rw.RLock() gKey := generateKey(request.Operation.Windows[0], request.Payload.Keys) + + rtm.rw.RLock() task, ok := rtm.tasks[gKey] rtm.rw.RUnlock() @@ -149,6 +150,7 @@ func (rtm *reduceTaskManager) WaitAll() { for _, task := range tasks { <-task.doneCh } + // after all the tasks are completed, close the output channel close(rtm.responseCh) } From 1231c4c278e07d68daa4f59bdbfd5858170e0a3f Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Mon, 11 Dec 2023 12:44:30 +0530 Subject: [PATCH 23/27] remove lock Signed-off-by: Yashash H L --- pkg/reducestreamer/task_manager.go | 10 ---------- pkg/sessionreducer/service.go | 3 +-- pkg/sessionreducer/service_test.go | 13 +++++++++++++ pkg/sessionreducer/task_manager.go | 23 +---------------------- 4 files changed, 15 insertions(+), 34 deletions(-) diff --git a/pkg/reducestreamer/task_manager.go b/pkg/reducestreamer/task_manager.go index 7d20fb09..6b71fec8 100644 --- a/pkg/reducestreamer/task_manager.go +++ b/pkg/reducestreamer/task_manager.go @@ -56,7 +56,6 @@ type reduceStreamTaskManager struct { reduceStreamer ReduceStreamer tasks map[string]*reduceStreamTask responseCh chan *v1.ReduceResponse - rw sync.RWMutex } func newReduceTaskManager(reduceStreamer ReduceStreamer) *reduceStreamTaskManager { @@ -73,7 +72,6 @@ func (rtm *reduceStreamTaskManager) CreateTask(ctx context.Context, request *v1. return fmt.Errorf("create operation error: invalid number of windows") } - rtm.rw.Lock() md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), request.Operation.Windows[0].GetEnd().AsTime())) @@ -88,8 +86,6 @@ func (rtm *reduceStreamTaskManager) CreateTask(ctx context.Context, request *v1. key := task.uniqueKey() rtm.tasks[key] = task - rtm.rw.Unlock() - go func() { var wg sync.WaitGroup wg.Add(1) @@ -125,10 +121,8 @@ func (rtm *reduceStreamTaskManager) AppendToTask(ctx context.Context, request *v return fmt.Errorf("append operation error: invalid number of windows") } - rtm.rw.RLock() gKey := generateKey(request.Operation.Windows[0], request.Payload.Keys) task, ok := rtm.tasks[gKey] - rtm.rw.RUnlock() // if the task is not found, create a new task if !ok { @@ -146,12 +140,10 @@ func (rtm *reduceStreamTaskManager) OutputChannel() <-chan *v1.ReduceResponse { // WaitAll waits for all the reduceStream tasks to complete. func (rtm *reduceStreamTaskManager) WaitAll() { - rtm.rw.RLock() tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.rw.RUnlock() for _, task := range tasks { <-task.doneCh @@ -162,12 +154,10 @@ func (rtm *reduceStreamTaskManager) WaitAll() { // CloseAll closes all the reduceStream tasks. func (rtm *reduceStreamTaskManager) CloseAll() { - rtm.rw.Lock() tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.rw.Unlock() for _, task := range tasks { close(task.inputCh) diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go index d7680a32..ad846fb9 100644 --- a/pkg/sessionreducer/service.go +++ b/pkg/sessionreducer/service.go @@ -51,9 +51,8 @@ func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionR for { d, recvErr := stream.Recv() - // if the stream is closed, close all the tasks and break + // if the stream is closed, break and wait for the tasks to return if recvErr == io.EOF { - taskManager.CloseAll() break } diff --git a/pkg/sessionreducer/service_test.go b/pkg/sessionreducer/service_test.go index 922c8c47..8b757f4c 100644 --- a/pkg/sessionreducer/service_test.go +++ b/pkg/sessionreducer/service_test.go @@ -166,6 +166,19 @@ func TestService_SessionReduceFn(t *testing.T) { }, }, }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, }, expected: []*sessionreducepb.SessionReduceResponse{ { diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index e1d34b77..d6fb593f 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -19,7 +19,6 @@ type sessionReduceTask struct { outputCh chan Message doneCh chan struct{} merged *atomic.Bool - closed *atomic.Bool } // buildSessionReduceResponse builds the session reduce response from the messages. @@ -87,7 +86,6 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 outputCh: make(chan Message), doneCh: make(chan struct{}), merged: atomic.NewBool(false), - closed: atomic.NewBool(false), } // add the task to the tasks list @@ -159,7 +157,7 @@ func (rtm *sessionReduceTaskManager) AppendToTask(ctx context.Context, request * return nil } -// CloseTask closes the input channel of the reduce tasks. +// CloseTask closes the input channel of the session reduce tasks. func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { rtm.rw.RLock() tasksToBeClosed := make([]*sessionReduceTask, 0, len(request.Operation.KeyedWindows)) @@ -173,7 +171,6 @@ func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) rtm.rw.RUnlock() for _, task := range tasksToBeClosed { - task.closed.Store(true) close(task.inputCh) } } @@ -298,24 +295,6 @@ func (rtm *sessionReduceTaskManager) WaitAll() { close(rtm.responseCh) } -// CloseAll closes all the reduce tasks. -func (rtm *sessionReduceTaskManager) CloseAll() { - rtm.rw.Lock() - tasks := make([]*sessionReduceTask, 0, len(rtm.tasks)) - for _, task := range rtm.tasks { - tasks = append(tasks, task) - } - rtm.rw.Unlock() - - for _, task := range tasks { - if task.closed.Load() || task.merged.Load() { - continue - } - task.closed.Store(true) - close(task.inputCh) - } -} - func generateKey(keyedWindows *v1.KeyedWindow) string { return fmt.Sprintf("%d:%d:%s", keyedWindows.GetStart().AsTime().UnixMilli(), From b5b6953315ad1b62f7326f12999de5d7671461f2 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 12 Dec 2023 13:22:42 +0530 Subject: [PATCH 24/27] add go mod Signed-off-by: Yashash H L --- pkg/reducer/examples/counter/go.mod | 16 ++++++++++++ pkg/reducer/examples/counter/go.sum | 28 ++++++++++++++++++++ pkg/reducer/examples/sum/go.mod | 16 ++++++++++++ pkg/reducer/examples/sum/go.sum | 28 ++++++++++++++++++++ pkg/reducer/task_manager.go | 15 +---------- pkg/reducestreamer/examples/counter/go.mod | 16 ++++++++++++ pkg/reducestreamer/examples/counter/go.sum | 28 ++++++++++++++++++++ pkg/reducestreamer/examples/sum/go.mod | 16 ++++++++++++ pkg/reducestreamer/examples/sum/go.sum | 28 ++++++++++++++++++++ pkg/sessionreducer/examples/counter/go.mod | 19 ++++++++++++++ pkg/sessionreducer/examples/counter/go.sum | 30 ++++++++++++++++++++++ 11 files changed, 226 insertions(+), 14 deletions(-) create mode 100644 pkg/reducer/examples/counter/go.mod create mode 100644 pkg/reducer/examples/counter/go.sum create mode 100644 pkg/reducer/examples/sum/go.mod create mode 100644 pkg/reducer/examples/sum/go.sum create mode 100644 pkg/reducestreamer/examples/counter/go.mod create mode 100644 pkg/reducestreamer/examples/counter/go.sum create mode 100644 pkg/reducestreamer/examples/sum/go.mod create mode 100644 pkg/reducestreamer/examples/sum/go.sum create mode 100644 pkg/sessionreducer/examples/counter/go.mod create mode 100644 pkg/sessionreducer/examples/counter/go.sum diff --git a/pkg/reducer/examples/counter/go.mod b/pkg/reducer/examples/counter/go.mod new file mode 100644 index 00000000..fa65ee4b --- /dev/null +++ b/pkg/reducer/examples/counter/go.mod @@ -0,0 +1,16 @@ +module counter + +go 1.20 + +require github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/reducer/examples/counter/go.sum b/pkg/reducer/examples/counter/go.sum new file mode 100644 index 00000000..ad0bbff5 --- /dev/null +++ b/pkg/reducer/examples/counter/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/reducer/examples/sum/go.mod b/pkg/reducer/examples/sum/go.mod new file mode 100644 index 00000000..9f2cf6b2 --- /dev/null +++ b/pkg/reducer/examples/sum/go.mod @@ -0,0 +1,16 @@ +module sum + +go 1.20 + +require github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/reducer/examples/sum/go.sum b/pkg/reducer/examples/sum/go.sum new file mode 100644 index 00000000..ad0bbff5 --- /dev/null +++ b/pkg/reducer/examples/sum/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index 69576ac6..8ebb03c3 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "sync" v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" ) @@ -56,7 +55,6 @@ type reduceTaskManager struct { reducer Reducer tasks map[string]*reduceTask responseCh chan *v1.ReduceResponse - rw sync.RWMutex } func newReduceTaskManager(reducer Reducer) *reduceTaskManager { @@ -85,10 +83,7 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce } key := task.uniqueKey() - - rtm.rw.Lock() rtm.tasks[key] = task - rtm.rw.Unlock() go func() { // invoke the reduce function @@ -118,11 +113,7 @@ func (rtm *reduceTaskManager) AppendToTask(ctx context.Context, request *v1.Redu return fmt.Errorf("append operation error: invalid number of windows") } - gKey := generateKey(request.Operation.Windows[0], request.Payload.Keys) - - rtm.rw.RLock() - task, ok := rtm.tasks[gKey] - rtm.rw.RUnlock() + task, ok := rtm.tasks[generateKey(request.Operation.Windows[0], request.Payload.Keys)] // if the task is not found, create a new task if !ok { @@ -140,12 +131,10 @@ func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { // WaitAll waits for all the reduce tasks to complete. func (rtm *reduceTaskManager) WaitAll() { - rtm.rw.RLock() tasks := make([]*reduceTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.rw.RUnlock() for _, task := range tasks { <-task.doneCh @@ -157,12 +146,10 @@ func (rtm *reduceTaskManager) WaitAll() { // CloseAll closes all the reduce tasks. func (rtm *reduceTaskManager) CloseAll() { - rtm.rw.Lock() tasks := make([]*reduceTask, 0, len(rtm.tasks)) for _, task := range rtm.tasks { tasks = append(tasks, task) } - rtm.rw.Unlock() for _, task := range tasks { close(task.inputCh) diff --git a/pkg/reducestreamer/examples/counter/go.mod b/pkg/reducestreamer/examples/counter/go.mod new file mode 100644 index 00000000..fa65ee4b --- /dev/null +++ b/pkg/reducestreamer/examples/counter/go.mod @@ -0,0 +1,16 @@ +module counter + +go 1.20 + +require github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/reducestreamer/examples/counter/go.sum b/pkg/reducestreamer/examples/counter/go.sum new file mode 100644 index 00000000..ad0bbff5 --- /dev/null +++ b/pkg/reducestreamer/examples/counter/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/reducestreamer/examples/sum/go.mod b/pkg/reducestreamer/examples/sum/go.mod new file mode 100644 index 00000000..9f2cf6b2 --- /dev/null +++ b/pkg/reducestreamer/examples/sum/go.mod @@ -0,0 +1,16 @@ +module sum + +go 1.20 + +require github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/reducestreamer/examples/sum/go.sum b/pkg/reducestreamer/examples/sum/go.sum new file mode 100644 index 00000000..ad0bbff5 --- /dev/null +++ b/pkg/reducestreamer/examples/sum/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/sessionreducer/examples/counter/go.mod b/pkg/sessionreducer/examples/counter/go.mod new file mode 100644 index 00000000..88b0b2de --- /dev/null +++ b/pkg/sessionreducer/examples/counter/go.mod @@ -0,0 +1,19 @@ +module counter + +go 1.20 + +require ( + github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 + go.uber.org/atomic v1.11.0 +) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sessionreducer/examples/counter/go.sum b/pkg/sessionreducer/examples/counter/go.sum new file mode 100644 index 00000000..85aa8eda --- /dev/null +++ b/pkg/sessionreducer/examples/counter/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From e76b04f9069ee07c4f76e7031eab2b035d98b906 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 19 Dec 2023 12:16:41 +0530 Subject: [PATCH 25/27] resolving conflicts Signed-off-by: Yashash H L --- pkg/reducer/server.go | 2 +- pkg/reducer/service.go | 4 ++-- pkg/reducer/service_test.go | 2 +- pkg/reducer/task_manager.go | 18 ++++++++++-------- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/reducer/server.go b/pkg/reducer/server.go index 49c03ebe..494282e7 100644 --- a/pkg/reducer/server.go +++ b/pkg/reducer/server.go @@ -25,7 +25,7 @@ func NewServer(r ReducerCreator, inputOptions ...Option) numaflow.Server { } s := new(server) s.svc = new(Service) - s.svc.reducerHandle = r + s.svc.reducerCreatorHandle = r s.opts = opts return s } diff --git a/pkg/reducer/service.go b/pkg/reducer/service.go index cbe048d9..1a5148a8 100644 --- a/pkg/reducer/service.go +++ b/pkg/reducer/service.go @@ -24,7 +24,7 @@ const ( // Service implements the proto gen server interface and contains the reduce operation handler. type Service struct { reducepb.UnimplementedReduceServer - reducerHandle Reducer + reducerCreatorHandle ReducerCreator } // IsReady returns true to indicate the gRPC connection is ready. @@ -40,7 +40,7 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { g errgroup.Group ) - taskManager := newReduceTaskManager(fs.reducerHandle) + taskManager := newReduceTaskManager(fs.reducerCreatorHandle) // err group for the go routine which reads from the output channel and sends to the stream g.Go(func() error { diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index 93aaf661..485cd3ba 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -468,7 +468,7 @@ func TestService_ReduceFn(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := &Service{ - reducerHandle: tt.handler, + reducerCreatorHandle: SimpleCreatorWithReduceFn(tt.handler), } // here's a trick for testing: // because we are not using gRPC, we directly set a new incoming ctx diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index 8ebb03c3..53e37503 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -52,16 +52,16 @@ func (rt *reduceTask) uniqueKey() string { // reduceTaskManager manages the reduce tasks for a reduce operation. type reduceTaskManager struct { - reducer Reducer - tasks map[string]*reduceTask - responseCh chan *v1.ReduceResponse + reducerCreatorHandle ReducerCreator + tasks map[string]*reduceTask + responseCh chan *v1.ReduceResponse } -func newReduceTaskManager(reducer Reducer) *reduceTaskManager { +func newReduceTaskManager(reducerCreatorHandle ReducerCreator) *reduceTaskManager { return &reduceTaskManager{ - reducer: reducer, - tasks: make(map[string]*reduceTask), - responseCh: make(chan *v1.ReduceResponse), + reducerCreatorHandle: reducerCreatorHandle, + tasks: make(map[string]*reduceTask), + responseCh: make(chan *v1.ReduceResponse), } } @@ -87,7 +87,9 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce go func() { // invoke the reduce function - messages := rtm.reducer.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) + // create a new reducer, since we got a new key + reducerHandle := rtm.reducerCreatorHandle.Create() + messages := reducerHandle.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) for _, message := range messages { // write the output to the output channel, service will forward it to downstream From d096c415a42f2baf0bab9e04da75f769c50cb8b7 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 19 Dec 2023 13:36:35 +0530 Subject: [PATCH 26/27] fix interface Signed-off-by: Yashash H L --- pkg/reducer/interface.go | 10 +++++----- pkg/reducestreamer/interface.go | 29 +++++++++++++++++++++++++---- pkg/reducestreamer/server.go | 4 ++-- pkg/reducestreamer/server_test.go | 6 +++--- pkg/reducestreamer/service.go | 4 ++-- pkg/reducestreamer/service_test.go | 20 ++++++++++---------- pkg/reducestreamer/task_manager.go | 17 +++++++++-------- pkg/sessionreducer/interface.go | 12 ++++++++++-- pkg/sessionreducer/server.go | 4 ++-- pkg/sessionreducer/server_test.go | 5 +---- pkg/sessionreducer/service.go | 4 ++-- pkg/sessionreducer/service_test.go | 19 +++++++++++++------ pkg/sessionreducer/task_manager.go | 18 +++++++++--------- 13 files changed, 93 insertions(+), 59 deletions(-) diff --git a/pkg/reducer/interface.go b/pkg/reducer/interface.go index 2acbbf95..e49d858a 100644 --- a/pkg/reducer/interface.go +++ b/pkg/reducer/interface.go @@ -23,6 +23,11 @@ type IntervalWindow interface { EndTime() time.Time } +// Reducer is the interface of reduce function implementation. +type Reducer interface { + Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, md Metadata) Messages +} + // ReducerCreator is the interface which is used to create a Reducer. type ReducerCreator interface { // Create creates a Reducer, will be invoked once for every keyed window. @@ -44,11 +49,6 @@ func SimpleCreatorWithReduceFn(f func(context.Context, []string, <-chan Datum, M return &simpleReducerCreator{f: f} } -// Reducer is the interface of reduce function implementation. -type Reducer interface { - Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, md Metadata) Messages -} - // reducerFn is a utility type used to convert a Reduce function to a Reducer. type reducerFn func(ctx context.Context, keys []string, reduceCh <-chan Datum, md Metadata) Messages diff --git a/pkg/reducestreamer/interface.go b/pkg/reducestreamer/interface.go index bc3d2db6..40dd4faa 100644 --- a/pkg/reducestreamer/interface.go +++ b/pkg/reducestreamer/interface.go @@ -28,10 +28,31 @@ type ReduceStreamer interface { ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) } -// ReduceStreamerFunc is a utility type used to convert a ReduceStream function to a ReduceStreamer. -type ReduceStreamerFunc func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) +// ReduceStreamerCreator is the interface which is used to create a ReduceStreamer. +type ReduceStreamerCreator interface { + // Create creates a ReduceStreamer, will be invoked once for every keyed window. + Create() ReduceStreamer +} + +// simpleReducerCreator is an implementation of ReduceStreamerCreator, which creates a ReduceStreamer for the given function. +type simpleReduceStreamerCreator struct { + f func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) +} + +// Create creates a Reducer for the given function. +func (s *simpleReduceStreamerCreator) Create() ReduceStreamer { + return reduceStreamFn(s.f) +} + +// SimpleCreatorWithReduceStreamFn creates a simple ReduceStreamerCreator for the given reduceStream function. +func SimpleCreatorWithReduceStreamFn(f func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata)) ReduceStreamerCreator { + return &simpleReduceStreamerCreator{f: f} +} + +// reduceStreamFn is a utility type used to convert a ReduceStream function to a ReduceStreamer. +type reduceStreamFn func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) -// ReduceStream implements the function of ReduceStream function. -func (rf ReduceStreamerFunc) ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { +// ReduceStream implements the function of ReduceStreamer interface. +func (rf reduceStreamFn) ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { rf(ctx, keys, inputCh, outputCh, md) } diff --git a/pkg/reducestreamer/server.go b/pkg/reducestreamer/server.go index 21f0c951..1e180637 100644 --- a/pkg/reducestreamer/server.go +++ b/pkg/reducestreamer/server.go @@ -18,14 +18,14 @@ type server struct { } // NewServer creates a new reduceStream server. -func NewServer(r ReduceStreamer, inputOptions ...Option) numaflow.Server { +func NewServer(r ReduceStreamerCreator, inputOptions ...Option) numaflow.Server { opts := DefaultOptions() for _, inputOption := range inputOptions { inputOption(opts) } s := new(server) s.svc = new(Service) - s.svc.reduceStreamerHandle = r + s.svc.creatorHandle = r s.opts = opts return s } diff --git a/pkg/reducestreamer/server_test.go b/pkg/reducestreamer/server_test.go index ff54d18d..aa595492 100644 --- a/pkg/reducestreamer/server_test.go +++ b/pkg/reducestreamer/server_test.go @@ -21,17 +21,17 @@ func TestReduceServer_Start(t *testing.T) { _ = os.RemoveAll(serverInfoFile.Name()) }() - var reduceHandler = ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + var reduceStreamHandle = func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } och <- NewMessage([]byte(strconv.Itoa(sum))) - }) + } // note: using actual uds connection ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) defer cancel() - err := NewServer(reduceHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) + err := NewServer(SimpleCreatorWithReduceStreamFn(reduceStreamHandle), WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) assert.NoError(t, err) } diff --git a/pkg/reducestreamer/service.go b/pkg/reducestreamer/service.go index 8c2e8728..641ce86d 100644 --- a/pkg/reducestreamer/service.go +++ b/pkg/reducestreamer/service.go @@ -24,7 +24,7 @@ const ( // Service implements the proto gen server interface and contains the reduceStream operation handler. type Service struct { reducepb.UnimplementedReduceServer - reduceStreamerHandle ReduceStreamer + creatorHandle ReduceStreamerCreator } // IsReady returns true to indicate the gRPC connection is ready. @@ -40,7 +40,7 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { g errgroup.Group ) - taskManager := newReduceTaskManager(fs.reduceStreamerHandle) + taskManager := newReduceTaskManager(fs.creatorHandle) // err group for the go routine which reads from the output channel and sends to the stream g.Go(func() error { diff --git a/pkg/reducestreamer/service_test.go b/pkg/reducestreamer/service_test.go index 85e9c05a..498151cf 100644 --- a/pkg/reducestreamer/service_test.go +++ b/pkg/reducestreamer/service_test.go @@ -55,21 +55,21 @@ func TestService_ReduceFn(t *testing.T) { tests := []struct { name string - handler ReduceStreamer + handler func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) input []*reducepb.ReduceRequest expected []*reducepb.ReduceResponse expectedErr bool }{ { name: "reduce_fn_forward_msg_same_keys", - handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) - }), + }, input: []*reducepb.ReduceRequest{ { Payload: &reducepb.ReduceRequest_Payload{ @@ -144,14 +144,14 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_multiple_keys", - handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) - }), + }, input: []*reducepb.ReduceRequest{ { Payload: &reducepb.ReduceRequest_Payload{ @@ -304,14 +304,14 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_forward_to_all", - handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } och <- NewMessage([]byte(strconv.Itoa(sum))) - }), + }, input: []*reducepb.ReduceRequest{ { Payload: &reducepb.ReduceRequest_Payload{ @@ -384,14 +384,14 @@ func TestService_ReduceFn(t *testing.T) { }, { name: "reduce_fn_forward_msg_drop_msg", - handler: ReduceStreamerFunc(func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { sum := 0 for val := range rch { msgVal, _ := strconv.Atoi(string(val.Value())) sum += msgVal } och <- MessageToDrop() - }), + }, input: []*reducepb.ReduceRequest{ { Payload: &reducepb.ReduceRequest_Payload{ @@ -468,7 +468,7 @@ func TestService_ReduceFn(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := &Service{ - reduceStreamerHandle: tt.handler, + creatorHandle: SimpleCreatorWithReduceStreamFn(tt.handler), } // here's a trick for testing: // because we are not using gRPC, we directly set a new incoming ctx diff --git a/pkg/reducestreamer/task_manager.go b/pkg/reducestreamer/task_manager.go index 6b71fec8..a6d5feee 100644 --- a/pkg/reducestreamer/task_manager.go +++ b/pkg/reducestreamer/task_manager.go @@ -53,16 +53,16 @@ func (rt *reduceStreamTask) uniqueKey() string { // reduceStreamTaskManager manages the reduceStream tasks. type reduceStreamTaskManager struct { - reduceStreamer ReduceStreamer - tasks map[string]*reduceStreamTask - responseCh chan *v1.ReduceResponse + creatorHandle ReduceStreamerCreator + tasks map[string]*reduceStreamTask + responseCh chan *v1.ReduceResponse } -func newReduceTaskManager(reduceStreamer ReduceStreamer) *reduceStreamTaskManager { +func newReduceTaskManager(reduceStreamerCreator ReduceStreamerCreator) *reduceStreamTaskManager { return &reduceStreamTaskManager{ - reduceStreamer: reduceStreamer, - tasks: make(map[string]*reduceStreamTask), - responseCh: make(chan *v1.ReduceResponse), + creatorHandle: reduceStreamerCreator, + tasks: make(map[string]*reduceStreamTask), + responseCh: make(chan *v1.ReduceResponse), } } @@ -99,8 +99,9 @@ func (rtm *reduceStreamTaskManager) CreateTask(ctx context.Context, request *v1. rtm.responseCh <- task.buildEOFResponse() }() + reduceStreamerHandle := rtm.creatorHandle.Create() // invoke the reduceStream function - rtm.reduceStreamer.ReduceStream(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh, md) + reduceStreamerHandle.ReduceStream(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh, md) // close the output channel after the reduceStream function is done close(task.outputCh) // wait for the output to be forwarded diff --git a/pkg/sessionreducer/interface.go b/pkg/sessionreducer/interface.go index c356a5ae..905ae380 100644 --- a/pkg/sessionreducer/interface.go +++ b/pkg/sessionreducer/interface.go @@ -14,10 +14,18 @@ type Datum interface { // SessionReducer is the interface which can be used to implement a session reduce operation. type SessionReducer interface { + // SessionReduce applies a session reduce function to a request stream and streams the results. SessionReduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message) + // Accumulator returns the accumulator for the session reducer, will be invoked when this session is merged + // with another session. Accumulator(ctx context.Context) []byte + // MergeAccumulator merges the accumulator for the session reducer, will be invoked when another session is merged + // with this session. MergeAccumulator(ctx context.Context, accumulator []byte) } -// CreateSessionReducer is a function which returns a new instance of SessionReducer. -type CreateSessionReducer func() SessionReducer +// SessionReducerCreator is the interface which can be used to create a session reducer. +type SessionReducerCreator interface { + // Create creates a session reducer, will be invoked once for every keyed window. + Create() SessionReducer +} diff --git a/pkg/sessionreducer/server.go b/pkg/sessionreducer/server.go index c7aa1634..5b630bf3 100644 --- a/pkg/sessionreducer/server.go +++ b/pkg/sessionreducer/server.go @@ -18,14 +18,14 @@ type server struct { } // NewServer creates a new session reduce server. -func NewServer(r CreateSessionReducer, inputOptions ...Option) numaflow.Server { +func NewServer(r SessionReducerCreator, inputOptions ...Option) numaflow.Server { opts := DefaultOptions() for _, inputOption := range inputOptions { inputOption(opts) } s := new(server) s.svc = new(Service) - s.svc.createSessionReducer = r + s.svc.creatorHandle = r s.opts = opts return s } diff --git a/pkg/sessionreducer/server_test.go b/pkg/sessionreducer/server_test.go index c7ce0c9e..c020b6e2 100644 --- a/pkg/sessionreducer/server_test.go +++ b/pkg/sessionreducer/server_test.go @@ -20,12 +20,9 @@ func TestReduceServer_Start(t *testing.T) { _ = os.RemoveAll(serverInfoFile.Name()) }() - var reduceHandler = CreateSessionReducer(func() SessionReducer { - return nil - }) // note: using actual uds connection ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) defer cancel() - err := NewServer(reduceHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) + err := NewServer(&SessionSumCreator{}, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) assert.NoError(t, err) } diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go index ad846fb9..b3ddd08b 100644 --- a/pkg/sessionreducer/service.go +++ b/pkg/sessionreducer/service.go @@ -22,7 +22,7 @@ const ( // Service implements the proto gen server interface and contains the sesionreduce operation handler. type Service struct { sessionreducepb.UnimplementedSessionReduceServer - createSessionReducer CreateSessionReducer + creatorHandle SessionReducerCreator } // IsReady returns true to indicate the gRPC connection is ready. @@ -34,7 +34,7 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*sessionreducepb.Re func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionReduceFnServer) error { ctx := stream.Context() - taskManager := newReduceTaskManager(fs.createSessionReducer) + taskManager := newReduceTaskManager(fs.creatorHandle) // err group for the go routine which reads from the output channel and sends to the stream var g errgroup.Group diff --git a/pkg/sessionreducer/service_test.go b/pkg/sessionreducer/service_test.go index 8b757f4c..bcce002d 100644 --- a/pkg/sessionreducer/service_test.go +++ b/pkg/sessionreducer/service_test.go @@ -77,6 +77,13 @@ func (s *SessionSum) MergeAccumulator(ctx context.Context, accumulator []byte) { s.sum.Add(int32(val)) } +type SessionSumCreator struct { +} + +func (s *SessionSumCreator) Create() SessionReducer { + return NewSessionSum() +} + func NewSessionSum() SessionReducer { return &SessionSum{ sum: atomic.NewInt32(0), @@ -87,14 +94,14 @@ func TestService_SessionReduceFn(t *testing.T) { tests := []struct { name string - handler CreateSessionReducer + handler SessionReducerCreator input []*sessionreducepb.SessionReduceRequest expected []*sessionreducepb.SessionReduceResponse expectedErr bool }{ { name: "open_append_close", - handler: NewSessionSum, + handler: &SessionSumCreator{}, input: []*sessionreducepb.SessionReduceRequest{ { Payload: &sessionreducepb.SessionReduceRequest_Payload{ @@ -199,7 +206,7 @@ func TestService_SessionReduceFn(t *testing.T) { }, { name: "open_expand_close", - handler: NewSessionSum, + handler: &SessionSumCreator{}, input: []*sessionreducepb.SessionReduceRequest{ { Payload: &sessionreducepb.SessionReduceRequest_Payload{ @@ -341,7 +348,7 @@ func TestService_SessionReduceFn(t *testing.T) { }, { name: "open_merge_close", - handler: NewSessionSum, + handler: &SessionSumCreator{}, input: []*sessionreducepb.SessionReduceRequest{ { Payload: &sessionreducepb.SessionReduceRequest_Payload{ @@ -509,7 +516,7 @@ func TestService_SessionReduceFn(t *testing.T) { }, { name: "open_expand_append_merge_close", - handler: NewSessionSum, + handler: &SessionSumCreator{}, input: []*sessionreducepb.SessionReduceRequest{ { Payload: &sessionreducepb.SessionReduceRequest_Payload{ @@ -767,7 +774,7 @@ func TestService_SessionReduceFn(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := &Service{ - createSessionReducer: tt.handler, + creatorHandle: tt.handler, } // here's a trick for testing: // because we are not using gRPC, we directly set a new incoming ctx diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index d6fb593f..2abd58c6 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -56,17 +56,17 @@ func (rt *sessionReduceTask) uniqueKey() string { // sessionReduceTaskManager manages the tasks for a session reduce operation. type sessionReduceTaskManager struct { - sessionReducerFactory CreateSessionReducer - tasks map[string]*sessionReduceTask - responseCh chan *v1.SessionReduceResponse - rw sync.RWMutex + creatorHandle SessionReducerCreator + tasks map[string]*sessionReduceTask + responseCh chan *v1.SessionReduceResponse + rw sync.RWMutex } -func newReduceTaskManager(sessionReducerFactory CreateSessionReducer) *sessionReduceTaskManager { +func newReduceTaskManager(sessionReducerFactory SessionReducerCreator) *sessionReduceTaskManager { return &sessionReduceTaskManager{ - sessionReducerFactory: sessionReducerFactory, - tasks: make(map[string]*sessionReduceTask), - responseCh: make(chan *v1.SessionReduceResponse), + creatorHandle: sessionReducerFactory, + tasks: make(map[string]*sessionReduceTask), + responseCh: make(chan *v1.SessionReduceResponse), } } @@ -81,7 +81,7 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 task := &sessionReduceTask{ keyedWindow: request.Operation.KeyedWindows[0], - sessionReducer: rtm.sessionReducerFactory(), + sessionReducer: rtm.creatorHandle.Create(), inputCh: make(chan Datum), outputCh: make(chan Message), doneCh: make(chan struct{}), From 7ba4dd46205db62b8ae8bce603f8fa0ee57c9ce2 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Tue, 19 Dec 2023 13:49:40 +0530 Subject: [PATCH 27/27] update examples Signed-off-by: Yashash H L --- pkg/reducer/examples/counter/go.mod | 2 +- pkg/reducer/examples/counter/go.sum | 2 ++ pkg/reducer/examples/sum/go.mod | 2 +- pkg/reducer/examples/sum/go.sum | 2 ++ pkg/reducestreamer/examples/counter/go.mod | 2 +- pkg/reducestreamer/examples/counter/go.sum | 2 ++ pkg/reducestreamer/examples/counter/main.go | 2 +- pkg/reducestreamer/examples/sum/go.mod | 2 +- pkg/reducestreamer/examples/sum/go.sum | 2 ++ pkg/reducestreamer/examples/sum/main.go | 9 ++++++++- pkg/sessionreducer/examples/counter/Makefile | 2 +- pkg/sessionreducer/examples/counter/go.mod | 2 +- pkg/sessionreducer/examples/counter/go.sum | 2 ++ pkg/sessionreducer/examples/counter/main.go | 9 ++++++++- 14 files changed, 33 insertions(+), 9 deletions(-) diff --git a/pkg/reducer/examples/counter/go.mod b/pkg/reducer/examples/counter/go.mod index 16ca9c17..21d79eb1 100644 --- a/pkg/reducer/examples/counter/go.mod +++ b/pkg/reducer/examples/counter/go.mod @@ -2,7 +2,7 @@ module counter go 1.20 -require github.com/numaproj/numaflow-go v0.6.0 +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducer/examples/counter/go.sum b/pkg/reducer/examples/counter/go.sum index 028be282..e4aac392 100644 --- a/pkg/reducer/examples/counter/go.sum +++ b/pkg/reducer/examples/counter/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.6.0 h1:gqTX1u1pFJJhX/3l3zYM8aLqRSHEainYrgBIollL0js= github.com/numaproj/numaflow-go v0.6.0/go.mod h1:5zwvvREIbqaCPCKsNE1MVjVToD0kvkCh2Z90Izlhw5U= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/reducer/examples/sum/go.mod b/pkg/reducer/examples/sum/go.mod index c5df5c1b..27de5f05 100644 --- a/pkg/reducer/examples/sum/go.mod +++ b/pkg/reducer/examples/sum/go.mod @@ -2,7 +2,7 @@ module sum go 1.20 -require github.com/numaproj/numaflow-go v0.6.0 +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducer/examples/sum/go.sum b/pkg/reducer/examples/sum/go.sum index 028be282..e4aac392 100644 --- a/pkg/reducer/examples/sum/go.sum +++ b/pkg/reducer/examples/sum/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.6.0 h1:gqTX1u1pFJJhX/3l3zYM8aLqRSHEainYrgBIollL0js= github.com/numaproj/numaflow-go v0.6.0/go.mod h1:5zwvvREIbqaCPCKsNE1MVjVToD0kvkCh2Z90Izlhw5U= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/reducestreamer/examples/counter/go.mod b/pkg/reducestreamer/examples/counter/go.mod index fa65ee4b..21d79eb1 100644 --- a/pkg/reducestreamer/examples/counter/go.mod +++ b/pkg/reducestreamer/examples/counter/go.mod @@ -2,7 +2,7 @@ module counter go 1.20 -require github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducestreamer/examples/counter/go.sum b/pkg/reducestreamer/examples/counter/go.sum index ad0bbff5..32cb64f7 100644 --- a/pkg/reducestreamer/examples/counter/go.sum +++ b/pkg/reducestreamer/examples/counter/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/reducestreamer/examples/counter/main.go b/pkg/reducestreamer/examples/counter/main.go index 17e54259..9b1caf9b 100644 --- a/pkg/reducestreamer/examples/counter/main.go +++ b/pkg/reducestreamer/examples/counter/main.go @@ -27,5 +27,5 @@ func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducestream } func main() { - reducestreamer.NewServer(reducestreamer.ReduceStreamerFunc(reduceCounter)).Start(context.Background()) + reducestreamer.NewServer(reducestreamer.SimpleCreatorWithReduceStreamFn(reduceCounter)).Start(context.Background()) } diff --git a/pkg/reducestreamer/examples/sum/go.mod b/pkg/reducestreamer/examples/sum/go.mod index 9f2cf6b2..27de5f05 100644 --- a/pkg/reducestreamer/examples/sum/go.mod +++ b/pkg/reducestreamer/examples/sum/go.mod @@ -2,7 +2,7 @@ module sum go 1.20 -require github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducestreamer/examples/sum/go.sum b/pkg/reducestreamer/examples/sum/go.sum index ad0bbff5..32cb64f7 100644 --- a/pkg/reducestreamer/examples/sum/go.sum +++ b/pkg/reducestreamer/examples/sum/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/reducestreamer/examples/sum/main.go b/pkg/reducestreamer/examples/sum/main.go index f182ba16..8aa66720 100644 --- a/pkg/reducestreamer/examples/sum/main.go +++ b/pkg/reducestreamer/examples/sum/main.go @@ -45,8 +45,15 @@ func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan re outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) } +// SumCreator is the creator for the sum reducestreamer. +type SumCreator struct{} + +func (s *SumCreator) Create() reducestreamer.ReduceStreamer { + return &Sum{} +} + func main() { - err := reducestreamer.NewServer(&Sum{}).Start(context.Background()) + err := reducestreamer.NewServer(&SumCreator{}).Start(context.Background()) if err != nil { log.Panic("unable to start the server due to: ", err) } diff --git a/pkg/sessionreducer/examples/counter/Makefile b/pkg/sessionreducer/examples/counter/Makefile index cb151461..eb4555da 100644 --- a/pkg/sessionreducer/examples/counter/Makefile +++ b/pkg/sessionreducer/examples/counter/Makefile @@ -4,7 +4,7 @@ build: .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/session-counter:v0.5.3" --target counter . + docker build -t "quay.io/numaio/numaflow-go/session-counter:v0.6.1" --target counter . clean: -rm -rf ./dist diff --git a/pkg/sessionreducer/examples/counter/go.mod b/pkg/sessionreducer/examples/counter/go.mod index 88b0b2de..85215d54 100644 --- a/pkg/sessionreducer/examples/counter/go.mod +++ b/pkg/sessionreducer/examples/counter/go.mod @@ -3,7 +3,7 @@ module counter go 1.20 require ( - github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 + github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f go.uber.org/atomic v1.11.0 ) diff --git a/pkg/sessionreducer/examples/counter/go.sum b/pkg/sessionreducer/examples/counter/go.sum index 85aa8eda..00ceacec 100644 --- a/pkg/sessionreducer/examples/counter/go.sum +++ b/pkg/sessionreducer/examples/counter/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go index 7dc11aca..8eafdffb 100644 --- a/pkg/sessionreducer/examples/counter/main.go +++ b/pkg/sessionreducer/examples/counter/main.go @@ -42,6 +42,13 @@ func NewSessionCounter() sessionreducer.SessionReducer { } } +// SessionCounterCreator is the creator for the session reducer. +type SessionCounterCreator struct{} + +func (s *SessionCounterCreator) Create() sessionreducer.SessionReducer { + return NewSessionCounter() +} + func main() { - sessionreducer.NewServer(NewSessionCounter).Start(context.Background()) + sessionreducer.NewServer(&SessionCounterCreator{}).Start(context.Background()) }