From 2903e2744139336ef2d3e3fb92662961c7e36f54 Mon Sep 17 00:00:00 2001 From: Yashash H L Date: Wed, 20 Dec 2023 04:21:29 +0530 Subject: [PATCH] feat: Session Window and Reduce Streaming (#87) Signed-off-by: Yashash H L Signed-off-by: Vigith Maurice Co-authored-by: Vigith Maurice --- go.mod | 1 + go.sum | 2 + pkg/apis/proto/reduce/v1/reduce.pb.go | 493 +++++++++-- pkg/apis/proto/reduce/v1/reduce.proto | 45 +- pkg/apis/proto/sessionreduce/v1/mockgen.go | 3 + .../sessionreduce/v1/sessionreduce.pb.go | 772 ++++++++++++++++ .../sessionreduce/v1/sessionreduce.proto | 84 ++ .../sessionreduce/v1/sessionreduce_grpc.pb.go | 179 ++++ .../v1/sessionreducemock/sessionreducemock.go | 216 +++++ pkg/reducer/examples/counter/go.mod | 4 +- pkg/reducer/examples/counter/go.sum | 2 + pkg/reducer/examples/counter/main.go | 4 +- pkg/reducer/examples/sum/go.mod | 2 +- pkg/reducer/examples/sum/go.sum | 2 + pkg/reducer/examples/sum/main.go | 4 +- pkg/reducer/interface.go | 10 +- pkg/reducer/message.go | 2 +- pkg/reducer/server.go | 4 +- pkg/reducer/service.go | 153 +--- pkg/reducer/service_test.go | 383 ++++++-- pkg/reducer/task_manager.go | 170 ++++ pkg/reducestreamer/doc.go | 5 + .../examples/counter/Dockerfile | 20 + pkg/reducestreamer/examples/counter/Makefile | 10 + pkg/reducestreamer/examples/counter/README.md | 3 + pkg/reducestreamer/examples/counter/go.mod | 16 + pkg/reducestreamer/examples/counter/go.sum | 30 + pkg/reducestreamer/examples/counter/main.go | 31 + pkg/reducestreamer/examples/sum/Dockerfile | 20 + pkg/reducestreamer/examples/sum/Makefile | 10 + pkg/reducestreamer/examples/sum/README.md | 3 + pkg/reducestreamer/examples/sum/go.mod | 16 + pkg/reducestreamer/examples/sum/go.sum | 30 + pkg/reducestreamer/examples/sum/main.go | 60 ++ pkg/reducestreamer/interface.go | 58 ++ pkg/reducestreamer/message.go | 52 ++ pkg/reducestreamer/options.go | 43 + pkg/reducestreamer/options_test.go | 18 + pkg/reducestreamer/server.go | 56 ++ pkg/reducestreamer/server_test.go | 37 + pkg/reducestreamer/service.go | 101 +++ pkg/reducestreamer/service_test.go | 525 +++++++++++ pkg/reducestreamer/task_manager.go | 177 ++++ pkg/reducestreamer/types.go | 65 ++ pkg/sessionreducer/doc.go | 5 + .../examples/counter/Dockerfile | 20 + pkg/sessionreducer/examples/counter/Makefile | 10 + pkg/sessionreducer/examples/counter/README.md | 3 + pkg/sessionreducer/examples/counter/go.mod | 19 + pkg/sessionreducer/examples/counter/go.sum | 32 + pkg/sessionreducer/examples/counter/main.go | 54 ++ pkg/sessionreducer/interface.go | 31 + pkg/sessionreducer/message.go | 52 ++ pkg/sessionreducer/options.go | 43 + pkg/sessionreducer/options_test.go | 18 + pkg/sessionreducer/server.go | 56 ++ pkg/sessionreducer/server_test.go | 28 + pkg/sessionreducer/service.go | 113 +++ pkg/sessionreducer/service_test.go | 830 ++++++++++++++++++ pkg/sessionreducer/task_manager.go | 307 +++++++ pkg/sessionreducer/types.go | 30 + 61 files changed, 5276 insertions(+), 296 deletions(-) create mode 100644 pkg/apis/proto/sessionreduce/v1/mockgen.go create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreduce.proto create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go create mode 100644 pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go create mode 100644 pkg/reducer/task_manager.go create mode 100644 pkg/reducestreamer/doc.go create mode 100644 pkg/reducestreamer/examples/counter/Dockerfile create mode 100644 pkg/reducestreamer/examples/counter/Makefile create mode 100644 pkg/reducestreamer/examples/counter/README.md create mode 100644 pkg/reducestreamer/examples/counter/go.mod create mode 100644 pkg/reducestreamer/examples/counter/go.sum create mode 100644 pkg/reducestreamer/examples/counter/main.go create mode 100644 pkg/reducestreamer/examples/sum/Dockerfile create mode 100644 pkg/reducestreamer/examples/sum/Makefile create mode 100644 pkg/reducestreamer/examples/sum/README.md create mode 100644 pkg/reducestreamer/examples/sum/go.mod create mode 100644 pkg/reducestreamer/examples/sum/go.sum create mode 100644 pkg/reducestreamer/examples/sum/main.go create mode 100644 pkg/reducestreamer/interface.go create mode 100644 pkg/reducestreamer/message.go create mode 100644 pkg/reducestreamer/options.go create mode 100644 pkg/reducestreamer/options_test.go create mode 100644 pkg/reducestreamer/server.go create mode 100644 pkg/reducestreamer/server_test.go create mode 100644 pkg/reducestreamer/service.go create mode 100644 pkg/reducestreamer/service_test.go create mode 100644 pkg/reducestreamer/task_manager.go create mode 100644 pkg/reducestreamer/types.go create mode 100644 pkg/sessionreducer/doc.go create mode 100644 pkg/sessionreducer/examples/counter/Dockerfile create mode 100644 pkg/sessionreducer/examples/counter/Makefile create mode 100644 pkg/sessionreducer/examples/counter/README.md create mode 100644 pkg/sessionreducer/examples/counter/go.mod create mode 100644 pkg/sessionreducer/examples/counter/go.sum create mode 100644 pkg/sessionreducer/examples/counter/main.go create mode 100644 pkg/sessionreducer/interface.go create mode 100644 pkg/sessionreducer/message.go create mode 100644 pkg/sessionreducer/options.go create mode 100644 pkg/sessionreducer/options_test.go create mode 100644 pkg/sessionreducer/server.go create mode 100644 pkg/sessionreducer/server_test.go create mode 100644 pkg/sessionreducer/service.go create mode 100644 pkg/sessionreducer/service_test.go create mode 100644 pkg/sessionreducer/task_manager.go create mode 100644 pkg/sessionreducer/types.go diff --git a/go.mod b/go.mod index a6ed38f4..423fb94f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/golang/mock v1.6.0 github.com/stretchr/testify v1.8.1 + go.uber.org/atomic v1.11.0 golang.org/x/net v0.9.0 golang.org/x/sync v0.1.0 google.golang.org/grpc v1.57.0 diff --git a/go.sum b/go.sum index 90eb1d42..c20884dc 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index a64affa1..2f9b365c 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -22,6 +22,55 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ReduceRequest_WindowOperation_Event int32 + +const ( + ReduceRequest_WindowOperation_OPEN ReduceRequest_WindowOperation_Event = 0 + ReduceRequest_WindowOperation_CLOSE ReduceRequest_WindowOperation_Event = 1 + ReduceRequest_WindowOperation_APPEND ReduceRequest_WindowOperation_Event = 4 +) + +// Enum value maps for ReduceRequest_WindowOperation_Event. +var ( + ReduceRequest_WindowOperation_Event_name = map[int32]string{ + 0: "OPEN", + 1: "CLOSE", + 4: "APPEND", + } + ReduceRequest_WindowOperation_Event_value = map[string]int32{ + "OPEN": 0, + "CLOSE": 1, + "APPEND": 4, + } +) + +func (x ReduceRequest_WindowOperation_Event) Enum() *ReduceRequest_WindowOperation_Event { + p := new(ReduceRequest_WindowOperation_Event) + *p = x + return p +} + +func (x ReduceRequest_WindowOperation_Event) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ReduceRequest_WindowOperation_Event) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes[0].Descriptor() +} + +func (ReduceRequest_WindowOperation_Event) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes[0] +} + +func (x ReduceRequest_WindowOperation_Event) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ReduceRequest_WindowOperation_Event.Descriptor instead. +func (ReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0, 0, 0} +} + // * // ReduceRequest represents a request element. type ReduceRequest struct { @@ -29,10 +78,8 @@ type ReduceRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` + Payload *ReduceRequest_Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + Operation *ReduceRequest_WindowOperation `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` } func (x *ReduceRequest) Reset() { @@ -67,34 +114,85 @@ func (*ReduceRequest) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0} } -func (x *ReduceRequest) GetKeys() []string { +func (x *ReduceRequest) GetPayload() *ReduceRequest_Payload { if x != nil { - return x.Keys + return x.Payload } return nil } -func (x *ReduceRequest) GetValue() []byte { +func (x *ReduceRequest) GetOperation() *ReduceRequest_WindowOperation { if x != nil { - return x.Value + return x.Operation } return nil } -func (x *ReduceRequest) GetEventTime() *timestamppb.Timestamp { +// Window represents a window. +// Since the client doesn't track keys, window doesn't have a keys field. +type Window struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` + End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` + Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` +} + +func (x *Window) Reset() { + *x = Window{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Window) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Window) ProtoMessage() {} + +func (x *Window) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Window.ProtoReflect.Descriptor instead. +func (*Window) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{1} +} + +func (x *Window) GetStart() *timestamppb.Timestamp { if x != nil { - return x.EventTime + return x.Start } return nil } -func (x *ReduceRequest) GetWatermark() *timestamppb.Timestamp { +func (x *Window) GetEnd() *timestamppb.Timestamp { if x != nil { - return x.Watermark + return x.End } return nil } +func (x *Window) GetSlot() string { + if x != nil { + return x.Slot + } + return "" +} + // * // ReduceResponse represents a response element. type ReduceResponse struct { @@ -102,13 +200,17 @@ type ReduceResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Results []*ReduceResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Result *ReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + // window represents a window to which the result belongs. + Window *Window `protobuf:"bytes,2,opt,name=window,proto3" json:"window,omitempty"` + // EOF represents the end of the response for a window. + EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` } func (x *ReduceResponse) Reset() { *x = ReduceResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -121,7 +223,7 @@ func (x *ReduceResponse) String() string { func (*ReduceResponse) ProtoMessage() {} func (x *ReduceResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -134,16 +236,30 @@ func (x *ReduceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReduceResponse.ProtoReflect.Descriptor instead. func (*ReduceResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{1} + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{2} +} + +func (x *ReduceResponse) GetResult() *ReduceResponse_Result { + if x != nil { + return x.Result + } + return nil } -func (x *ReduceResponse) GetResults() []*ReduceResponse_Result { +func (x *ReduceResponse) GetWindow() *Window { if x != nil { - return x.Results + return x.Window } return nil } +func (x *ReduceResponse) GetEOF() bool { + if x != nil { + return x.EOF + } + return false +} + // * // ReadyResponse is the health check result. type ReadyResponse struct { @@ -157,7 +273,7 @@ type ReadyResponse struct { func (x *ReadyResponse) Reset() { *x = ReadyResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -170,7 +286,7 @@ func (x *ReadyResponse) String() string { func (*ReadyResponse) ProtoMessage() {} func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -183,7 +299,7 @@ func (x *ReadyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{2} + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{3} } func (x *ReadyResponse) GetReady() bool { @@ -193,6 +309,136 @@ func (x *ReadyResponse) GetReady() bool { return false } +// WindowOperation represents a window operation. +// For Aligned windows, OPEN, APPEND and CLOSE events are sent. +type ReduceRequest_WindowOperation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Event ReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=reduce.v1.ReduceRequest_WindowOperation_Event" json:"event,omitempty"` + Windows []*Window `protobuf:"bytes,2,rep,name=windows,proto3" json:"windows,omitempty"` +} + +func (x *ReduceRequest_WindowOperation) Reset() { + *x = ReduceRequest_WindowOperation{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReduceRequest_WindowOperation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReduceRequest_WindowOperation) ProtoMessage() {} + +func (x *ReduceRequest_WindowOperation) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReduceRequest_WindowOperation.ProtoReflect.Descriptor instead. +func (*ReduceRequest_WindowOperation) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *ReduceRequest_WindowOperation) GetEvent() ReduceRequest_WindowOperation_Event { + if x != nil { + return x.Event + } + return ReduceRequest_WindowOperation_OPEN +} + +func (x *ReduceRequest_WindowOperation) GetWindows() []*Window { + if x != nil { + return x.Windows + } + return nil +} + +// Payload represents a payload element. +type ReduceRequest_Payload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` +} + +func (x *ReduceRequest_Payload) Reset() { + *x = ReduceRequest_Payload{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReduceRequest_Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReduceRequest_Payload) ProtoMessage() {} + +func (x *ReduceRequest_Payload) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReduceRequest_Payload.ProtoReflect.Descriptor instead. +func (*ReduceRequest_Payload) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{0, 1} +} + +func (x *ReduceRequest_Payload) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *ReduceRequest_Payload) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *ReduceRequest_Payload) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *ReduceRequest_Payload) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +// Result represents a result element. It contains the result of the reduce function. type ReduceResponse_Result struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -206,7 +452,7 @@ type ReduceResponse_Result struct { func (x *ReduceResponse_Result) Reset() { *x = ReduceResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -219,7 +465,7 @@ func (x *ReduceResponse_Result) String() string { func (*ReduceResponse_Result) ProtoMessage() {} func (x *ReduceResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3] + mi := &file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -232,7 +478,7 @@ func (x *ReduceResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use ReduceResponse_Result.ProtoReflect.Descriptor instead. func (*ReduceResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{1, 0} + return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP(), []int{2, 0} } func (x *ReduceResponse_Result) GetKeys() []string { @@ -266,42 +512,74 @@ var file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc = []byte{ 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0xae, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, - 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, - 0x6b, 0x22, 0x94, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, - 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, - 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, - 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, - 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, - 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, - 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, - 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, - 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x22, 0xef, 0x03, 0x0a, 0x0d, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x46, + 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xae, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x05, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x72, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x12, 0x2b, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x22, 0x28, 0x0a, + 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x00, + 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, + 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, + 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, + 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, + 0x72, 0x6b, 0x22, 0x7c, 0x0a, 0x06, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x30, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, + 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, + 0x22, 0xcf, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x29, 0x0a, + 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, 0x4f, 0x46, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x8a, 0x01, 0x0a, 0x06, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, + 0x12, 0x18, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x72, + 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, + 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, + 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -316,28 +594,40 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP() []byte { return file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescData } -var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []interface{}{ - (*ReduceRequest)(nil), // 0: reduce.v1.ReduceRequest - (*ReduceResponse)(nil), // 1: reduce.v1.ReduceResponse - (*ReadyResponse)(nil), // 2: reduce.v1.ReadyResponse - (*ReduceResponse_Result)(nil), // 3: reduce.v1.ReduceResponse.Result - (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 5: google.protobuf.Empty + (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Event + (*ReduceRequest)(nil), // 1: reduce.v1.ReduceRequest + (*Window)(nil), // 2: reduce.v1.Window + (*ReduceResponse)(nil), // 3: reduce.v1.ReduceResponse + (*ReadyResponse)(nil), // 4: reduce.v1.ReadyResponse + (*ReduceRequest_WindowOperation)(nil), // 5: reduce.v1.ReduceRequest.WindowOperation + (*ReduceRequest_Payload)(nil), // 6: reduce.v1.ReduceRequest.Payload + (*ReduceResponse_Result)(nil), // 7: reduce.v1.ReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs = []int32{ - 4, // 0: reduce.v1.ReduceRequest.event_time:type_name -> google.protobuf.Timestamp - 4, // 1: reduce.v1.ReduceRequest.watermark:type_name -> google.protobuf.Timestamp - 3, // 2: reduce.v1.ReduceResponse.results:type_name -> reduce.v1.ReduceResponse.Result - 0, // 3: reduce.v1.Reduce.ReduceFn:input_type -> reduce.v1.ReduceRequest - 5, // 4: reduce.v1.Reduce.IsReady:input_type -> google.protobuf.Empty - 1, // 5: reduce.v1.Reduce.ReduceFn:output_type -> reduce.v1.ReduceResponse - 2, // 6: reduce.v1.Reduce.IsReady:output_type -> reduce.v1.ReadyResponse - 5, // [5:7] is the sub-list for method output_type - 3, // [3:5] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 6, // 0: reduce.v1.ReduceRequest.payload:type_name -> reduce.v1.ReduceRequest.Payload + 5, // 1: reduce.v1.ReduceRequest.operation:type_name -> reduce.v1.ReduceRequest.WindowOperation + 8, // 2: reduce.v1.Window.start:type_name -> google.protobuf.Timestamp + 8, // 3: reduce.v1.Window.end:type_name -> google.protobuf.Timestamp + 7, // 4: reduce.v1.ReduceResponse.result:type_name -> reduce.v1.ReduceResponse.Result + 2, // 5: reduce.v1.ReduceResponse.window:type_name -> reduce.v1.Window + 0, // 6: reduce.v1.ReduceRequest.WindowOperation.event:type_name -> reduce.v1.ReduceRequest.WindowOperation.Event + 2, // 7: reduce.v1.ReduceRequest.WindowOperation.windows:type_name -> reduce.v1.Window + 8, // 8: reduce.v1.ReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 9: reduce.v1.ReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 1, // 10: reduce.v1.Reduce.ReduceFn:input_type -> reduce.v1.ReduceRequest + 9, // 11: reduce.v1.Reduce.IsReady:input_type -> google.protobuf.Empty + 3, // 12: reduce.v1.Reduce.ReduceFn:output_type -> reduce.v1.ReduceResponse + 4, // 13: reduce.v1.Reduce.IsReady:output_type -> reduce.v1.ReadyResponse + 12, // [12:14] is the sub-list for method output_type + 10, // [10:12] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_pkg_apis_proto_reduce_v1_reduce_proto_init() } @@ -359,7 +649,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { } } file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReduceResponse); i { + switch v := v.(*Window); i { case 0: return &v.state case 1: @@ -371,7 +661,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { } } file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadyResponse); i { + switch v := v.(*ReduceResponse); i { case 0: return &v.state case 1: @@ -383,6 +673,42 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { } } file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReadyResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReduceRequest_WindowOperation); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReduceRequest_Payload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ReduceResponse_Result); i { case 0: return &v.state @@ -400,13 +726,14 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_reduce_v1_reduce_proto_rawDesc, - NumEnums: 0, - NumMessages: 4, + NumEnums: 1, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, GoTypes: file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes, DependencyIndexes: file_pkg_apis_proto_reduce_v1_reduce_proto_depIdxs, + EnumInfos: file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes, MessageInfos: file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes, }.Build() File_pkg_apis_proto_reduce_v1_reduce_proto = out.File diff --git a/pkg/apis/proto/reduce/v1/reduce.proto b/pkg/apis/proto/reduce/v1/reduce.proto index 5f596a81..2b2e071d 100644 --- a/pkg/apis/proto/reduce/v1/reduce.proto +++ b/pkg/apis/proto/reduce/v1/reduce.proto @@ -20,22 +20,57 @@ service Reduce { * ReduceRequest represents a request element. */ message ReduceRequest { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; + // WindowOperation represents a window operation. + // For Aligned windows, OPEN, APPEND and CLOSE events are sent. + message WindowOperation { + enum Event { + OPEN = 0; + CLOSE = 1; + APPEND = 4; + } + + Event event = 1; + repeated Window windows = 2; + } + + // Payload represents a payload element. + message Payload { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + } + + Payload payload = 1; + WindowOperation operation = 2; +} + +// Window represents a window. +// Since the client doesn't track keys, window doesn't have a keys field. +message Window { + google.protobuf.Timestamp start = 1; + google.protobuf.Timestamp end = 2; + string slot = 3; } /** * ReduceResponse represents a response element. */ message ReduceResponse { + // Result represents a result element. It contains the result of the reduce function. message Result { repeated string keys = 1; bytes value = 2; repeated string tags = 3; } - repeated Result results = 1; + + Result result = 1; + + // window represents a window to which the result belongs. + Window window = 2; + + // EOF represents the end of the response for a window. + bool EOF = 3; } /** diff --git a/pkg/apis/proto/sessionreduce/v1/mockgen.go b/pkg/apis/proto/sessionreduce/v1/mockgen.go new file mode 100644 index 00000000..53aa925b --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/mockgen.go @@ -0,0 +1,3 @@ +package v1 + +//go:generate mockgen -destination sessionreducemock/sessionreducemock.go -package sessionreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1 SessionReduceClient,SessionReduce_SessionReduceFnClient diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go new file mode 100644 index 00000000..8a672b83 --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -0,0 +1,772 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v4.25.1 +// source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto + +package v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SessionReduceRequest_WindowOperation_Event int32 + +const ( + SessionReduceRequest_WindowOperation_OPEN SessionReduceRequest_WindowOperation_Event = 0 + SessionReduceRequest_WindowOperation_CLOSE SessionReduceRequest_WindowOperation_Event = 1 + SessionReduceRequest_WindowOperation_EXPAND SessionReduceRequest_WindowOperation_Event = 2 + SessionReduceRequest_WindowOperation_MERGE SessionReduceRequest_WindowOperation_Event = 3 + SessionReduceRequest_WindowOperation_APPEND SessionReduceRequest_WindowOperation_Event = 4 +) + +// Enum value maps for SessionReduceRequest_WindowOperation_Event. +var ( + SessionReduceRequest_WindowOperation_Event_name = map[int32]string{ + 0: "OPEN", + 1: "CLOSE", + 2: "EXPAND", + 3: "MERGE", + 4: "APPEND", + } + SessionReduceRequest_WindowOperation_Event_value = map[string]int32{ + "OPEN": 0, + "CLOSE": 1, + "EXPAND": 2, + "MERGE": 3, + "APPEND": 4, + } +) + +func (x SessionReduceRequest_WindowOperation_Event) Enum() *SessionReduceRequest_WindowOperation_Event { + p := new(SessionReduceRequest_WindowOperation_Event) + *p = x + return p +} + +func (x SessionReduceRequest_WindowOperation_Event) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SessionReduceRequest_WindowOperation_Event) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0].Descriptor() +} + +func (SessionReduceRequest_WindowOperation_Event) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes[0] +} + +func (x SessionReduceRequest_WindowOperation_Event) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SessionReduceRequest_WindowOperation_Event.Descriptor instead. +func (SessionReduceRequest_WindowOperation_Event) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0, 0} +} + +// KeyedWindow represents a window with keys. +// since the client track the keys, we use keyed window. +type KeyedWindow struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Start *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` + End *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` + Slot string `protobuf:"bytes,3,opt,name=slot,proto3" json:"slot,omitempty"` + Keys []string `protobuf:"bytes,4,rep,name=keys,proto3" json:"keys,omitempty"` +} + +func (x *KeyedWindow) Reset() { + *x = KeyedWindow{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *KeyedWindow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*KeyedWindow) ProtoMessage() {} + +func (x *KeyedWindow) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use KeyedWindow.ProtoReflect.Descriptor instead. +func (*KeyedWindow) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{0} +} + +func (x *KeyedWindow) GetStart() *timestamppb.Timestamp { + if x != nil { + return x.Start + } + return nil +} + +func (x *KeyedWindow) GetEnd() *timestamppb.Timestamp { + if x != nil { + return x.End + } + return nil +} + +func (x *KeyedWindow) GetSlot() string { + if x != nil { + return x.Slot + } + return "" +} + +func (x *KeyedWindow) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +// * +// SessionReduceRequest represents a request element. +type SessionReduceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Payload *SessionReduceRequest_Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + Operation *SessionReduceRequest_WindowOperation `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` +} + +func (x *SessionReduceRequest) Reset() { + *x = SessionReduceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceRequest) ProtoMessage() {} + +func (x *SessionReduceRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceRequest.ProtoReflect.Descriptor instead. +func (*SessionReduceRequest) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1} +} + +func (x *SessionReduceRequest) GetPayload() *SessionReduceRequest_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SessionReduceRequest) GetOperation() *SessionReduceRequest_WindowOperation { + if x != nil { + return x.Operation + } + return nil +} + +// * +// SessionReduceResponse represents a response element. +type SessionReduceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Result *SessionReduceResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + // keyedWindow represents a window to which the result belongs. + KeyedWindow *KeyedWindow `protobuf:"bytes,2,opt,name=keyedWindow,proto3" json:"keyedWindow,omitempty"` + // EOF represents the end of the response for a window. + EOF bool `protobuf:"varint,3,opt,name=EOF,proto3" json:"EOF,omitempty"` +} + +func (x *SessionReduceResponse) Reset() { + *x = SessionReduceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceResponse) ProtoMessage() {} + +func (x *SessionReduceResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceResponse.ProtoReflect.Descriptor instead. +func (*SessionReduceResponse) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{2} +} + +func (x *SessionReduceResponse) GetResult() *SessionReduceResponse_Result { + if x != nil { + return x.Result + } + return nil +} + +func (x *SessionReduceResponse) GetKeyedWindow() *KeyedWindow { + if x != nil { + return x.KeyedWindow + } + return nil +} + +func (x *SessionReduceResponse) GetEOF() bool { + if x != nil { + return x.EOF + } + return false +} + +// * +// ReadyResponse is the health check result. +type ReadyResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` +} + +func (x *ReadyResponse) Reset() { + *x = ReadyResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReadyResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadyResponse) ProtoMessage() {} + +func (x *ReadyResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. +func (*ReadyResponse) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{3} +} + +func (x *ReadyResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +// WindowOperation represents a window operation. +// For Aligned window values can be one of OPEN, CLOSE, EXPAND, MERGE and APPEND. +type SessionReduceRequest_WindowOperation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Event SessionReduceRequest_WindowOperation_Event `protobuf:"varint,1,opt,name=event,proto3,enum=sessionreduce.v1.SessionReduceRequest_WindowOperation_Event" json:"event,omitempty"` + KeyedWindows []*KeyedWindow `protobuf:"bytes,2,rep,name=keyedWindows,proto3" json:"keyedWindows,omitempty"` +} + +func (x *SessionReduceRequest_WindowOperation) Reset() { + *x = SessionReduceRequest_WindowOperation{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceRequest_WindowOperation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceRequest_WindowOperation) ProtoMessage() {} + +func (x *SessionReduceRequest_WindowOperation) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceRequest_WindowOperation.ProtoReflect.Descriptor instead. +func (*SessionReduceRequest_WindowOperation) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *SessionReduceRequest_WindowOperation) GetEvent() SessionReduceRequest_WindowOperation_Event { + if x != nil { + return x.Event + } + return SessionReduceRequest_WindowOperation_OPEN +} + +func (x *SessionReduceRequest_WindowOperation) GetKeyedWindows() []*KeyedWindow { + if x != nil { + return x.KeyedWindows + } + return nil +} + +// Payload represents a payload element. +type SessionReduceRequest_Payload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` +} + +func (x *SessionReduceRequest_Payload) Reset() { + *x = SessionReduceRequest_Payload{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceRequest_Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceRequest_Payload) ProtoMessage() {} + +func (x *SessionReduceRequest_Payload) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceRequest_Payload.ProtoReflect.Descriptor instead. +func (*SessionReduceRequest_Payload) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *SessionReduceRequest_Payload) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *SessionReduceRequest_Payload) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *SessionReduceRequest_Payload) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *SessionReduceRequest_Payload) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +// Result represents a result element. It contains the result of the reduce function. +type SessionReduceResponse_Result struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` +} + +func (x *SessionReduceResponse_Result) Reset() { + *x = SessionReduceResponse_Result{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionReduceResponse_Result) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionReduceResponse_Result) ProtoMessage() {} + +func (x *SessionReduceResponse_Result) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionReduceResponse_Result.ProtoReflect.Descriptor instead. +func (*SessionReduceResponse_Result) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP(), []int{2, 0} +} + +func (x *SessionReduceResponse_Result) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *SessionReduceResponse_Result) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *SessionReduceResponse_Result) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +var File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto protoreflect.FileDescriptor + +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = []byte{ + 0x0a, 0x33, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2f, 0x76, + 0x31, 0x2f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x95, 0x01, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0xcd, 0x04, + 0x0a, 0x14, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x12, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xe9, 0x01, 0x0a, 0x0f, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x05, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x41, + 0x0a, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x52, 0x0c, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4f, 0x50, + 0x45, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x01, 0x12, + 0x0a, 0x0a, 0x06, 0x45, 0x58, 0x50, 0x41, 0x4e, 0x44, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4d, + 0x45, 0x52, 0x47, 0x45, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, + 0x10, 0x04, 0x1a, 0xa8, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, + 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x22, 0xfa, 0x01, + 0x0a, 0x15, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x3f, 0x0a, 0x0b, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x52, 0x0b, 0x6b, 0x65, 0x79, 0x65, 0x64, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x12, 0x10, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x45, + 0x4f, 0x46, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, + 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, + 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, + 0x79, 0x32, 0xbb, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x0f, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x64, 0x75, 0x63, 0x65, 0x46, 0x6e, 0x12, 0x26, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, + 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x07, 0x49, + 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, + 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, + 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, + 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, + 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescOnce sync.Once + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData = file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc +) + +func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP() []byte { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescOnce.Do(func() { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData) + }) + return file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescData +} + +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []interface{}{ + (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + (*KeyedWindow)(nil), // 1: sessionreduce.v1.KeyedWindow + (*SessionReduceRequest)(nil), // 2: sessionreduce.v1.SessionReduceRequest + (*SessionReduceResponse)(nil), // 3: sessionreduce.v1.SessionReduceResponse + (*ReadyResponse)(nil), // 4: sessionreduce.v1.ReadyResponse + (*SessionReduceRequest_WindowOperation)(nil), // 5: sessionreduce.v1.SessionReduceRequest.WindowOperation + (*SessionReduceRequest_Payload)(nil), // 6: sessionreduce.v1.SessionReduceRequest.Payload + (*SessionReduceResponse_Result)(nil), // 7: sessionreduce.v1.SessionReduceResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty +} +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = []int32{ + 8, // 0: sessionreduce.v1.KeyedWindow.start:type_name -> google.protobuf.Timestamp + 8, // 1: sessionreduce.v1.KeyedWindow.end:type_name -> google.protobuf.Timestamp + 6, // 2: sessionreduce.v1.SessionReduceRequest.payload:type_name -> sessionreduce.v1.SessionReduceRequest.Payload + 5, // 3: sessionreduce.v1.SessionReduceRequest.operation:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation + 7, // 4: sessionreduce.v1.SessionReduceResponse.result:type_name -> sessionreduce.v1.SessionReduceResponse.Result + 1, // 5: sessionreduce.v1.SessionReduceResponse.keyedWindow:type_name -> sessionreduce.v1.KeyedWindow + 0, // 6: sessionreduce.v1.SessionReduceRequest.WindowOperation.event:type_name -> sessionreduce.v1.SessionReduceRequest.WindowOperation.Event + 1, // 7: sessionreduce.v1.SessionReduceRequest.WindowOperation.keyedWindows:type_name -> sessionreduce.v1.KeyedWindow + 8, // 8: sessionreduce.v1.SessionReduceRequest.Payload.event_time:type_name -> google.protobuf.Timestamp + 8, // 9: sessionreduce.v1.SessionReduceRequest.Payload.watermark:type_name -> google.protobuf.Timestamp + 2, // 10: sessionreduce.v1.SessionReduce.SessionReduceFn:input_type -> sessionreduce.v1.SessionReduceRequest + 9, // 11: sessionreduce.v1.SessionReduce.IsReady:input_type -> google.protobuf.Empty + 3, // 12: sessionreduce.v1.SessionReduce.SessionReduceFn:output_type -> sessionreduce.v1.SessionReduceResponse + 4, // 13: sessionreduce.v1.SessionReduce.IsReady:output_type -> sessionreduce.v1.ReadyResponse + 12, // [12:14] is the sub-list for method output_type + 10, // [10:12] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name +} + +func init() { file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() } +func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { + if File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*KeyedWindow); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReadyResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceRequest_WindowOperation); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceRequest_Payload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SessionReduceResponse_Result); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc, + NumEnums: 1, + NumMessages: 7, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes, + DependencyIndexes: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs, + EnumInfos: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes, + MessageInfos: file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes, + }.Build() + File_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto = out.File + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDesc = nil + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = nil + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_depIdxs = nil +} diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto new file mode 100644 index 00000000..07d354e6 --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -0,0 +1,84 @@ +syntax = "proto3"; + +option go_package = "github.com/numaproj/numaflow-go/pkg/apis/proto/reducestream/v1"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + + +package sessionreduce.v1; + +service SessionReduce { + // SessionReduceFn applies a reduce function to a request stream. + rpc SessionReduceFn(stream SessionReduceRequest) returns (stream SessionReduceResponse); + + // IsReady is the heartbeat endpoint for gRPC. + rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); +} + +// KeyedWindow represents a window with keys. +// since the client track the keys, we use keyed window. +message KeyedWindow { + google.protobuf.Timestamp start = 1; + google.protobuf.Timestamp end = 2; + string slot = 3; + repeated string keys = 4; +} + +/** + * SessionReduceRequest represents a request element. + */ +message SessionReduceRequest { + // WindowOperation represents a window operation. + // For Aligned window values can be one of OPEN, CLOSE, EXPAND, MERGE and APPEND. + message WindowOperation { + enum Event { + OPEN = 0; + CLOSE = 1; + EXPAND = 2; + MERGE = 3; + APPEND = 4; + } + + Event event = 1; + repeated KeyedWindow keyedWindows = 2; + } + + // Payload represents a payload element. + message Payload { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + } + + Payload payload = 1; + WindowOperation operation = 2; +} + +/** + * SessionReduceResponse represents a response element. + */ +message SessionReduceResponse { + // Result represents a result element. It contains the result of the reduce function. + message Result { + repeated string keys = 1; + bytes value = 2; + repeated string tags = 3; + } + + Result result = 1; + + // keyedWindow represents a window to which the result belongs. + KeyedWindow keyedWindow = 2; + + // EOF represents the end of the response for a window. + bool EOF = 3; +} + +/** + * ReadyResponse is the health check result. + */ +message ReadyResponse { + bool ready = 1; +} \ No newline at end of file diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go new file mode 100644 index 00000000..51b7bd6c --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go @@ -0,0 +1,179 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v4.25.1 +// source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SessionReduceClient is the client API for SessionReduce service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SessionReduceClient interface { + // SessionReduceFn applies a reduce function to a request stream. + SessionReduceFn(ctx context.Context, opts ...grpc.CallOption) (SessionReduce_SessionReduceFnClient, error) + // IsReady is the heartbeat endpoint for gRPC. + IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) +} + +type sessionReduceClient struct { + cc grpc.ClientConnInterface +} + +func NewSessionReduceClient(cc grpc.ClientConnInterface) SessionReduceClient { + return &sessionReduceClient{cc} +} + +func (c *sessionReduceClient) SessionReduceFn(ctx context.Context, opts ...grpc.CallOption) (SessionReduce_SessionReduceFnClient, error) { + stream, err := c.cc.NewStream(ctx, &SessionReduce_ServiceDesc.Streams[0], "/sessionreduce.v1.SessionReduce/SessionReduceFn", opts...) + if err != nil { + return nil, err + } + x := &sessionReduceSessionReduceFnClient{stream} + return x, nil +} + +type SessionReduce_SessionReduceFnClient interface { + Send(*SessionReduceRequest) error + Recv() (*SessionReduceResponse, error) + grpc.ClientStream +} + +type sessionReduceSessionReduceFnClient struct { + grpc.ClientStream +} + +func (x *sessionReduceSessionReduceFnClient) Send(m *SessionReduceRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *sessionReduceSessionReduceFnClient) Recv() (*SessionReduceResponse, error) { + m := new(SessionReduceResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *sessionReduceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + out := new(ReadyResponse) + err := c.cc.Invoke(ctx, "/sessionreduce.v1.SessionReduce/IsReady", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SessionReduceServer is the server API for SessionReduce service. +// All implementations must embed UnimplementedSessionReduceServer +// for forward compatibility +type SessionReduceServer interface { + // SessionReduceFn applies a reduce function to a request stream. + SessionReduceFn(SessionReduce_SessionReduceFnServer) error + // IsReady is the heartbeat endpoint for gRPC. + IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) + mustEmbedUnimplementedSessionReduceServer() +} + +// UnimplementedSessionReduceServer must be embedded to have forward compatible implementations. +type UnimplementedSessionReduceServer struct { +} + +func (UnimplementedSessionReduceServer) SessionReduceFn(SessionReduce_SessionReduceFnServer) error { + return status.Errorf(codes.Unimplemented, "method SessionReduceFn not implemented") +} +func (UnimplementedSessionReduceServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") +} +func (UnimplementedSessionReduceServer) mustEmbedUnimplementedSessionReduceServer() {} + +// UnsafeSessionReduceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SessionReduceServer will +// result in compilation errors. +type UnsafeSessionReduceServer interface { + mustEmbedUnimplementedSessionReduceServer() +} + +func RegisterSessionReduceServer(s grpc.ServiceRegistrar, srv SessionReduceServer) { + s.RegisterService(&SessionReduce_ServiceDesc, srv) +} + +func _SessionReduce_SessionReduceFn_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SessionReduceServer).SessionReduceFn(&sessionReduceSessionReduceFnServer{stream}) +} + +type SessionReduce_SessionReduceFnServer interface { + Send(*SessionReduceResponse) error + Recv() (*SessionReduceRequest, error) + grpc.ServerStream +} + +type sessionReduceSessionReduceFnServer struct { + grpc.ServerStream +} + +func (x *sessionReduceSessionReduceFnServer) Send(m *SessionReduceResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *sessionReduceSessionReduceFnServer) Recv() (*SessionReduceRequest, error) { + m := new(SessionReduceRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _SessionReduce_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(emptypb.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SessionReduceServer).IsReady(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/sessionreduce.v1.SessionReduce/IsReady", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SessionReduceServer).IsReady(ctx, req.(*emptypb.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// SessionReduce_ServiceDesc is the grpc.ServiceDesc for SessionReduce service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SessionReduce_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "sessionreduce.v1.SessionReduce", + HandlerType: (*SessionReduceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "IsReady", + Handler: _SessionReduce_IsReady_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "SessionReduceFn", + Handler: _SessionReduce_SessionReduceFn_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "pkg/apis/proto/sessionreduce/v1/sessionreduce.proto", +} diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go b/pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go new file mode 100644 index 00000000..d1572efb --- /dev/null +++ b/pkg/apis/proto/sessionreduce/v1/sessionreducemock/sessionreducemock.go @@ -0,0 +1,216 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1 (interfaces: SessionReduceClient,SessionReduce_SessionReduceFnClient) + +// Package sessionreducemock is a generated GoMock package. +package sessionreducemock + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" + grpc "google.golang.org/grpc" + metadata "google.golang.org/grpc/metadata" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// MockSessionReduceClient is a mock of SessionReduceClient interface. +type MockSessionReduceClient struct { + ctrl *gomock.Controller + recorder *MockSessionReduceClientMockRecorder +} + +// MockSessionReduceClientMockRecorder is the mock recorder for MockSessionReduceClient. +type MockSessionReduceClientMockRecorder struct { + mock *MockSessionReduceClient +} + +// NewMockSessionReduceClient creates a new mock instance. +func NewMockSessionReduceClient(ctrl *gomock.Controller) *MockSessionReduceClient { + mock := &MockSessionReduceClient{ctrl: ctrl} + mock.recorder = &MockSessionReduceClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSessionReduceClient) EXPECT() *MockSessionReduceClientMockRecorder { + return m.recorder +} + +// IsReady mocks base method. +func (m *MockSessionReduceClient) IsReady(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*v1.ReadyResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "IsReady", varargs...) + ret0, _ := ret[0].(*v1.ReadyResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsReady indicates an expected call of IsReady. +func (mr *MockSessionReduceClientMockRecorder) IsReady(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsReady", reflect.TypeOf((*MockSessionReduceClient)(nil).IsReady), varargs...) +} + +// SessionReduceFn mocks base method. +func (m *MockSessionReduceClient) SessionReduceFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.SessionReduce_SessionReduceFnClient, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "SessionReduceFn", varargs...) + ret0, _ := ret[0].(v1.SessionReduce_SessionReduceFnClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SessionReduceFn indicates an expected call of SessionReduceFn. +func (mr *MockSessionReduceClientMockRecorder) SessionReduceFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SessionReduceFn", reflect.TypeOf((*MockSessionReduceClient)(nil).SessionReduceFn), varargs...) +} + +// MockSessionReduce_SessionReduceFnClient is a mock of SessionReduce_SessionReduceFnClient interface. +type MockSessionReduce_SessionReduceFnClient struct { + ctrl *gomock.Controller + recorder *MockSessionReduce_SessionReduceFnClientMockRecorder +} + +// MockSessionReduce_SessionReduceFnClientMockRecorder is the mock recorder for MockSessionReduce_SessionReduceFnClient. +type MockSessionReduce_SessionReduceFnClientMockRecorder struct { + mock *MockSessionReduce_SessionReduceFnClient +} + +// NewMockSessionReduce_SessionReduceFnClient creates a new mock instance. +func NewMockSessionReduce_SessionReduceFnClient(ctrl *gomock.Controller) *MockSessionReduce_SessionReduceFnClient { + mock := &MockSessionReduce_SessionReduceFnClient{ctrl: ctrl} + mock.recorder = &MockSessionReduce_SessionReduceFnClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSessionReduce_SessionReduceFnClient) EXPECT() *MockSessionReduce_SessionReduceFnClientMockRecorder { + return m.recorder +} + +// CloseSend mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) CloseSend() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloseSend") + ret0, _ := ret[0].(error) + return ret0 +} + +// CloseSend indicates an expected call of CloseSend. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) CloseSend() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseSend", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).CloseSend)) +} + +// Context mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Context() context.Context { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Context") + ret0, _ := ret[0].(context.Context) + return ret0 +} + +// Context indicates an expected call of Context. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Context() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Context)) +} + +// Header mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Header() (metadata.MD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Header") + ret0, _ := ret[0].(metadata.MD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Header indicates an expected call of Header. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Header() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Header)) +} + +// Recv mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Recv() (*v1.SessionReduceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Recv") + ret0, _ := ret[0].(*v1.SessionReduceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Recv indicates an expected call of Recv. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Recv() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Recv)) +} + +// RecvMsg mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) RecvMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecvMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecvMsg indicates an expected call of RecvMsg. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).RecvMsg), arg0) +} + +// Send mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Send(arg0 *v1.SessionReduceRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Send(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Send), arg0) +} + +// SendMsg mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) SendMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMsg indicates an expected call of SendMsg. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).SendMsg), arg0) +} + +// Trailer mocks base method. +func (m *MockSessionReduce_SessionReduceFnClient) Trailer() metadata.MD { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Trailer") + ret0, _ := ret[0].(metadata.MD) + return ret0 +} + +// Trailer indicates an expected call of Trailer. +func (mr *MockSessionReduce_SessionReduceFnClientMockRecorder) Trailer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockSessionReduce_SessionReduceFnClient)(nil).Trailer)) +} diff --git a/pkg/reducer/examples/counter/go.mod b/pkg/reducer/examples/counter/go.mod index 1f937708..21d79eb1 100644 --- a/pkg/reducer/examples/counter/go.mod +++ b/pkg/reducer/examples/counter/go.mod @@ -1,8 +1,8 @@ -module even_odd +module counter go 1.20 -require github.com/numaproj/numaflow-go v0.6.0 +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducer/examples/counter/go.sum b/pkg/reducer/examples/counter/go.sum index 028be282..e4aac392 100644 --- a/pkg/reducer/examples/counter/go.sum +++ b/pkg/reducer/examples/counter/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.6.0 h1:gqTX1u1pFJJhX/3l3zYM8aLqRSHEainYrgBIollL0js= github.com/numaproj/numaflow-go v0.6.0/go.mod h1:5zwvvREIbqaCPCKsNE1MVjVToD0kvkCh2Z90Izlhw5U= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/reducer/examples/counter/main.go b/pkg/reducer/examples/counter/main.go index 1ce455e3..4d3cf790 100644 --- a/pkg/reducer/examples/counter/main.go +++ b/pkg/reducer/examples/counter/main.go @@ -7,12 +7,12 @@ import ( "github.com/numaproj/numaflow-go/pkg/reducer" ) -func reduceCounter(_ context.Context, keys []string, reduceCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { +func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { // count the incoming events var resultKeys = keys var resultVal []byte var counter = 0 - for range reduceCh { + for range inputCh { counter++ } resultVal = []byte(strconv.Itoa(counter)) diff --git a/pkg/reducer/examples/sum/go.mod b/pkg/reducer/examples/sum/go.mod index c5df5c1b..27de5f05 100644 --- a/pkg/reducer/examples/sum/go.mod +++ b/pkg/reducer/examples/sum/go.mod @@ -2,7 +2,7 @@ module sum go 1.20 -require github.com/numaproj/numaflow-go v0.6.0 +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f require ( github.com/golang/protobuf v1.5.3 // indirect diff --git a/pkg/reducer/examples/sum/go.sum b/pkg/reducer/examples/sum/go.sum index 028be282..e4aac392 100644 --- a/pkg/reducer/examples/sum/go.sum +++ b/pkg/reducer/examples/sum/go.sum @@ -6,6 +6,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/numaproj/numaflow-go v0.6.0 h1:gqTX1u1pFJJhX/3l3zYM8aLqRSHEainYrgBIollL0js= github.com/numaproj/numaflow-go v0.6.0/go.mod h1:5zwvvREIbqaCPCKsNE1MVjVToD0kvkCh2Z90Izlhw5U= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= diff --git a/pkg/reducer/examples/sum/main.go b/pkg/reducer/examples/sum/main.go index efc91e00..abdb482f 100644 --- a/pkg/reducer/examples/sum/main.go +++ b/pkg/reducer/examples/sum/main.go @@ -22,14 +22,14 @@ type Sum struct { sum int } -func (s *Sum) Reduce(ctx context.Context, keys []string, reduceCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { +func (s *Sum) Reduce(ctx context.Context, keys []string, inputCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { // sum up values for the same keys intervalWindow := md.IntervalWindow() _ = intervalWindow var resultKeys = keys var resultVal []byte // sum up the values - for d := range reduceCh { + for d := range inputCh { val := d.Value() // event time and watermark can be fetched from the datum diff --git a/pkg/reducer/interface.go b/pkg/reducer/interface.go index 1a7a4419..e49d858a 100644 --- a/pkg/reducer/interface.go +++ b/pkg/reducer/interface.go @@ -23,6 +23,11 @@ type IntervalWindow interface { EndTime() time.Time } +// Reducer is the interface of reduce function implementation. +type Reducer interface { + Reduce(ctx context.Context, keys []string, inputCh <-chan Datum, md Metadata) Messages +} + // ReducerCreator is the interface which is used to create a Reducer. type ReducerCreator interface { // Create creates a Reducer, will be invoked once for every keyed window. @@ -44,11 +49,6 @@ func SimpleCreatorWithReduceFn(f func(context.Context, []string, <-chan Datum, M return &simpleReducerCreator{f: f} } -// Reducer is the interface of reduce function implementation. -type Reducer interface { - Reduce(ctx context.Context, keys []string, reduceCh <-chan Datum, md Metadata) Messages -} - // reducerFn is a utility type used to convert a Reduce function to a Reducer. type reducerFn func(ctx context.Context, keys []string, reduceCh <-chan Datum, md Metadata) Messages diff --git a/pkg/reducer/message.go b/pkg/reducer/message.go index e7114887..80209972 100644 --- a/pkg/reducer/message.go +++ b/pkg/reducer/message.go @@ -6,7 +6,7 @@ var ( DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ ) -// Message is used to wrap the data return by reduce functions +// Message is used to wrap the data return by reduce function type Message struct { value []byte keys []string diff --git a/pkg/reducer/server.go b/pkg/reducer/server.go index 6b8ef5aa..494282e7 100644 --- a/pkg/reducer/server.go +++ b/pkg/reducer/server.go @@ -6,7 +6,7 @@ import ( "os/signal" "syscall" - "github.com/numaproj/numaflow-go/pkg" + numaflow "github.com/numaproj/numaflow-go/pkg" reducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" "github.com/numaproj/numaflow-go/pkg/shared" ) @@ -25,7 +25,7 @@ func NewServer(r ReducerCreator, inputOptions ...Option) numaflow.Server { } s := new(server) s.svc = new(Service) - s.svc.CreateReduceHandler = r + s.svc.reducerCreatorHandle = r s.opts = opts return s } diff --git a/pkg/reducer/service.go b/pkg/reducer/service.go index 4e4321dd..1a5148a8 100644 --- a/pkg/reducer/service.go +++ b/pkg/reducer/service.go @@ -2,16 +2,10 @@ package reducer import ( "context" - "fmt" "io" - "strconv" - "strings" - "sync" - "time" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" - grpcmd "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" @@ -30,7 +24,7 @@ const ( // Service implements the proto gen server interface and contains the reduce operation handler. type Service struct { reducepb.UnimplementedReduceServer - CreateReduceHandler ReducerCreator + reducerCreatorHandle ReducerCreator } // IsReady returns true to indicate the gRPC connection is ready. @@ -41,44 +35,23 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*reducepb.ReadyResp // ReduceFn applies a reduce function to a request stream and returns a list of results. func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { var ( - md Metadata - err error - startTime int64 - endTime int64 - ctx = stream.Context() - chanMap = make(map[string]chan Datum) - mu sync.RWMutex - g errgroup.Group + err error + ctx = stream.Context() + g errgroup.Group ) - grpcMD, ok := grpcmd.FromIncomingContext(ctx) - if !ok { - statusErr := status.Errorf(codes.InvalidArgument, "keys and window information are not passed in grpc metadata") - return statusErr - } - - // get window start and end time from grpc metadata - var st, et string - st, err = getValueFromMetadata(grpcMD, winStartTime) - if err != nil { - statusErr := status.Errorf(codes.InvalidArgument, err.Error()) - return statusErr - } - - et, err = getValueFromMetadata(grpcMD, winEndTime) - if err != nil { - statusErr := status.Errorf(codes.InvalidArgument, err.Error()) - return statusErr - } - - startTime, _ = strconv.ParseInt(st, 10, 64) - endTime, _ = strconv.ParseInt(et, 10, 64) + taskManager := newReduceTaskManager(fs.reducerCreatorHandle) - // create interval window interface using the start and end time - iw := NewIntervalWindow(time.UnixMilli(startTime), time.UnixMilli(endTime)) - - // create metadata using interval window interface - md = NewMetadata(iw) + // err group for the go routine which reads from the output channel and sends to the stream + g.Go(func() error { + for output := range taskManager.OutputChannel() { + sendErr := stream.Send(output) + if sendErr != nil { + return sendErr + } + } + return nil + }) // read messages from the stream and write the messages to corresponding channels // if the channel is not created, create the channel and invoke the reduceFn @@ -86,87 +59,43 @@ func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { d, recvErr := stream.Recv() // if EOF, close all the channels if recvErr == io.EOF { - closeChannels(chanMap) + taskManager.CloseAll() break } if recvErr != nil { - closeChannels(chanMap) // the error here is returned by stream.Recv() // it's already a gRPC error return recvErr } - unifiedKey := strings.Join(d.GetKeys(), delimiter) - var hd = NewHandlerDatum(d.GetValue(), d.EventTime.AsTime(), d.Watermark.AsTime()) - - ch, chok := chanMap[unifiedKey] - if !chok { - ch = make(chan Datum) - chanMap[unifiedKey] = ch - func(k []string, ch chan Datum) { - g.Go(func() error { - // we stream the messages to the user by writing messages - // to channel and wait until we get the result and stream - // the result back to the client (numaflow). - - // create a new reducer, since we got a new key - reducer := fs.CreateReduceHandler.Create() - messages := reducer.Reduce(ctx, k, ch, md) - datumList := buildDatumList(messages) - - // stream.Send() is not thread safe. - mu.Lock() - defer mu.Unlock() - sendErr := stream.Send(datumList) - if sendErr != nil { - // the error here is returned by stream.Send() - // it's already a gRPC error - return sendErr - } - return nil - }) - }(d.GetKeys(), ch) + // for Aligned windows, its just open or append operation + // close signal will be sent to all the reducers when grpc + // input stream gets EOF. + switch d.Operation.Event { + case reducepb.ReduceRequest_WindowOperation_OPEN: + // create a new reduce task and start the reduce operation + err = taskManager.CreateTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case reducepb.ReduceRequest_WindowOperation_APPEND: + // append the datum to the reduce task + err = taskManager.AppendToTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } } - ch <- hd } - // wait until all the mapfn return - return g.Wait() -} - -func buildDatumList(messages Messages) *reducepb.ReduceResponse { - response := &reducepb.ReduceResponse{} - for _, msg := range messages { - response.Results = append(response.Results, &reducepb.ReduceResponse_Result{ - Keys: msg.Keys(), - Value: msg.Value(), - Tags: msg.Tags(), - }) - } - - return response -} - -func closeChannels(chanMap map[string]chan Datum) { - for _, ch := range chanMap { - close(ch) + taskManager.WaitAll() + // wait for the go routine which reads from the output channel and sends to the stream to return + err = g.Wait() + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr } -} -func getValueFromMetadata(md grpcmd.MD, k string) (string, error) { - var value string - - keyValue := md.Get(k) - - if len(keyValue) > 1 { - return value, fmt.Errorf("expected extactly one value for keys %s in metadata but got %d values, %s", k, len(keyValue), keyValue) - } else if len(keyValue) == 1 { - value = keyValue[0] - } else { - // the length equals zero is invalid for reduce - // since we are using a global keys, and start and end time - // cannot be empty - return value, fmt.Errorf("expected non empty value for keys %s in metadata but got an empty value", k) - } - return value, nil + return nil } diff --git a/pkg/reducer/service_test.go b/pkg/reducer/service_test.go index 98deeb56..485cd3ba 100644 --- a/pkg/reducer/service_test.go +++ b/pkg/reducer/service_test.go @@ -57,7 +57,7 @@ func TestService_ReduceFn(t *testing.T) { name string handler func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages input []*reducepb.ReduceRequest - expected *reducepb.ReduceResponse + expected []*reducepb.ReduceResponse expectedErr bool }{ { @@ -72,30 +72,72 @@ func TestService_ReduceFn(t *testing.T) { }, input: []*reducepb.ReduceRequest{ { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ Keys: []string{"client_test"}, Value: []byte(strconv.Itoa(60)), }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, }, expectedErr: false, @@ -112,56 +154,150 @@ func TestService_ReduceFn(t *testing.T) { }, input: []*reducepb.ReduceRequest{ { - Keys: []string{"client1"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client2"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client3"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client1"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client2"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client3"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ Keys: []string{"client1_test"}, Value: []byte(strconv.Itoa(20)), }, - { + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + { + Result: &reducepb.ReduceResponse_Result{ Keys: []string{"client2_test"}, Value: []byte(strconv.Itoa(40)), }, - { + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + { + Result: &reducepb.ReduceResponse_Result{ Keys: []string{"client3_test"}, Value: []byte(strconv.Itoa(60)), }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, }, expectedErr: false, @@ -178,32 +314,73 @@ func TestService_ReduceFn(t *testing.T) { }, input: []*reducepb.ReduceRequest{ { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ Value: []byte(strconv.Itoa(60)), }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, }, - expectedErr: false, }, { name: "reduce_fn_forward_msg_drop_msg", @@ -217,30 +394,72 @@ func TestService_ReduceFn(t *testing.T) { }, input: []*reducepb.ReduceRequest{ { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, { - Keys: []string{"client"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, }, }, - expected: &reducepb.ReduceResponse{ - Results: []*reducepb.ReduceResponse_Result{ - { + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ Tags: []string{DROP}, Value: []byte{}, }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, }, }, expectedErr: false, @@ -249,7 +468,7 @@ func TestService_ReduceFn(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := &Service{ - CreateReduceHandler: SimpleCreatorWithReduceFn(tt.handler), + reducerCreatorHandle: SimpleCreatorWithReduceFn(tt.handler), } // here's a trick for testing: // because we are not using gRPC, we directly set a new incoming ctx @@ -258,7 +477,7 @@ func TestService_ReduceFn(t *testing.T) { inputCh := make(chan *reducepb.ReduceRequest) outputCh := make(chan *reducepb.ReduceResponse) - result := &reducepb.ReduceResponse{} + result := make([]*reducepb.ReduceResponse, 0) udfReduceFnStream := NewReduceFnServerTest(ctx, inputCh, outputCh) @@ -276,7 +495,9 @@ func TestService_ReduceFn(t *testing.T) { go func() { defer wg.Done() for msg := range outputCh { - result.Results = append(result.Results, msg.Results...) + if !msg.EOF { + result = append(result, msg) + } } }() @@ -292,8 +513,8 @@ func TestService_ReduceFn(t *testing.T) { } //sort and compare, since order of the output doesn't matter - sort.Slice(result.Results, func(i, j int) bool { - return string(result.Results[i].Value) < string(result.Results[j].Value) + sort.Slice(result, func(i, j int) bool { + return string(result[i].Result.Value) < string(result[j].Result.Value) }) if !reflect.DeepEqual(result, tt.expected) { diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go new file mode 100644 index 00000000..53e37503 --- /dev/null +++ b/pkg/reducer/task_manager.go @@ -0,0 +1,170 @@ +package reducer + +import ( + "context" + "fmt" + "strings" + + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +// reduceTask represents a task for a performing reduceStream operation. +type reduceTask struct { + keys []string + window *v1.Window + reducer Reducer + inputCh chan Datum + outputCh chan Message + doneCh chan struct{} +} + +// buildReduceResponse builds the reduce response from the messages. +func (rt *reduceTask) buildReduceResponse(message Message) *v1.ReduceResponse { + + response := &v1.ReduceResponse{ + Result: &v1.ReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + }, + Window: rt.window, + } + + return response +} + +func (rt *reduceTask) buildEOFResponse() *v1.ReduceResponse { + response := &v1.ReduceResponse{ + Window: rt.window, + EOF: true, + } + + return response +} + +// uniqueKey returns the unique key for the reduce task to be used in the task manager to identify the task. +func (rt *reduceTask) uniqueKey() string { + return fmt.Sprintf("%d:%d:%s", + rt.window.GetStart().AsTime().UnixMilli(), + rt.window.GetEnd().AsTime().UnixMilli(), + strings.Join(rt.keys, delimiter)) +} + +// reduceTaskManager manages the reduce tasks for a reduce operation. +type reduceTaskManager struct { + reducerCreatorHandle ReducerCreator + tasks map[string]*reduceTask + responseCh chan *v1.ReduceResponse +} + +func newReduceTaskManager(reducerCreatorHandle ReducerCreator) *reduceTaskManager { + return &reduceTaskManager{ + reducerCreatorHandle: reducerCreatorHandle, + tasks: make(map[string]*reduceTask), + responseCh: make(chan *v1.ReduceResponse), + } +} + +// CreateTask creates a new reduce task and starts the reduce operation. +func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest) error { + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("create operation error: invalid number of windows") + } + + md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), + request.Operation.Windows[0].GetEnd().AsTime())) + + task := &reduceTask{ + keys: request.GetPayload().GetKeys(), + window: request.Operation.Windows[0], + inputCh: make(chan Datum), + outputCh: make(chan Message), + doneCh: make(chan struct{}), + } + + key := task.uniqueKey() + rtm.tasks[key] = task + + go func() { + // invoke the reduce function + // create a new reducer, since we got a new key + reducerHandle := rtm.reducerCreatorHandle.Create() + messages := reducerHandle.Reduce(ctx, request.GetPayload().GetKeys(), task.inputCh, md) + + for _, message := range messages { + // write the output to the output channel, service will forward it to downstream + rtm.responseCh <- task.buildReduceResponse(message) + } + // send EOF + rtm.responseCh <- task.buildEOFResponse() + // close the output channel after the reduce function is done + close(task.outputCh) + // send a done signal + close(task.doneCh) + }() + + // write the first message to the input channel + task.inputCh <- buildDatum(request) + return nil +} + +// AppendToTask writes the message to the reduce task. +// If the task is not found, it creates a new task and starts the reduce operation. +func (rtm *reduceTaskManager) AppendToTask(ctx context.Context, request *v1.ReduceRequest) error { + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("append operation error: invalid number of windows") + } + + task, ok := rtm.tasks[generateKey(request.Operation.Windows[0], request.Payload.Keys)] + + // if the task is not found, create a new task + if !ok { + return rtm.CreateTask(ctx, request) + } + + task.inputCh <- buildDatum(request) + return nil +} + +// OutputChannel returns the output channel for the reduce task manager to read the results. +func (rtm *reduceTaskManager) OutputChannel() <-chan *v1.ReduceResponse { + return rtm.responseCh +} + +// WaitAll waits for all the reduce tasks to complete. +func (rtm *reduceTaskManager) WaitAll() { + tasks := make([]*reduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { + tasks = append(tasks, task) + } + + for _, task := range tasks { + <-task.doneCh + } + + // after all the tasks are completed, close the output channel + close(rtm.responseCh) +} + +// CloseAll closes all the reduce tasks. +func (rtm *reduceTaskManager) CloseAll() { + tasks := make([]*reduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { + tasks = append(tasks, task) + } + + for _, task := range tasks { + close(task.inputCh) + } +} + +func generateKey(window *v1.Window, keys []string) string { + return fmt.Sprintf("%d:%d:%s", + window.GetStart().AsTime().UnixMilli(), + window.GetEnd().AsTime().UnixMilli(), + strings.Join(keys, delimiter)) +} + +func buildDatum(request *v1.ReduceRequest) Datum { + return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +} diff --git a/pkg/reducestreamer/doc.go b/pkg/reducestreamer/doc.go new file mode 100644 index 00000000..046c779e --- /dev/null +++ b/pkg/reducestreamer/doc.go @@ -0,0 +1,5 @@ +// Package reduceStreamer implements the server code for reduceStream operation. + +// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/reducestreamer/examples/ + +package reducestreamer diff --git a/pkg/reducestreamer/examples/counter/Dockerfile b/pkg/reducestreamer/examples/counter/Dockerfile new file mode 100644 index 00000000..7f1e3e5c --- /dev/null +++ b/pkg/reducestreamer/examples/counter/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/counter-example /bin/counter-example +RUN chmod +x /bin/counter-example + +#################################################################################################### +# counter +#################################################################################################### +FROM scratch as counter +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/counter-example /bin/counter-example +ENTRYPOINT [ "/bin/counter-example" ] diff --git a/pkg/reducestreamer/examples/counter/Makefile b/pkg/reducestreamer/examples/counter/Makefile new file mode 100644 index 00000000..ffe3036c --- /dev/null +++ b/pkg/reducestreamer/examples/counter/Makefile @@ -0,0 +1,10 @@ +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:v0.5.3" --target counter . + +clean: + -rm -rf ./dist diff --git a/pkg/reducestreamer/examples/counter/README.md b/pkg/reducestreamer/examples/counter/README.md new file mode 100644 index 00000000..8ba9bd9f --- /dev/null +++ b/pkg/reducestreamer/examples/counter/README.md @@ -0,0 +1,3 @@ +# Counter + +An example User Defined Function that count the incoming events and output the count every 10 events. diff --git a/pkg/reducestreamer/examples/counter/go.mod b/pkg/reducestreamer/examples/counter/go.mod new file mode 100644 index 00000000..21d79eb1 --- /dev/null +++ b/pkg/reducestreamer/examples/counter/go.mod @@ -0,0 +1,16 @@ +module counter + +go 1.20 + +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/reducestreamer/examples/counter/go.sum b/pkg/reducestreamer/examples/counter/go.sum new file mode 100644 index 00000000..32cb64f7 --- /dev/null +++ b/pkg/reducestreamer/examples/counter/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/reducestreamer/examples/counter/main.go b/pkg/reducestreamer/examples/counter/main.go new file mode 100644 index 00000000..9b1caf9b --- /dev/null +++ b/pkg/reducestreamer/examples/counter/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "strconv" + + "github.com/numaproj/numaflow-go/pkg/reducestreamer" +) + +// reduceCounter is a ReduceStreamer that count the incoming events and output the count every 10 events. +// The output message is the count of the events. +func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducestreamer.Datum, outputCh chan<- reducestreamer.Message, md reducestreamer.Metadata) { + // count the incoming events + var resultKeys = keys + var resultVal []byte + var counter = 0 + for range inputCh { + counter++ + if counter >= 10 { + resultVal = []byte(strconv.Itoa(counter)) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) + counter = 0 + } + } + resultVal = []byte(strconv.Itoa(counter)) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) +} + +func main() { + reducestreamer.NewServer(reducestreamer.SimpleCreatorWithReduceStreamFn(reduceCounter)).Start(context.Background()) +} diff --git a/pkg/reducestreamer/examples/sum/Dockerfile b/pkg/reducestreamer/examples/sum/Dockerfile new file mode 100644 index 00000000..4b237f86 --- /dev/null +++ b/pkg/reducestreamer/examples/sum/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/sum-example /bin/sum-example +RUN chmod +x /bin/sum-example + +#################################################################################################### +# sum +#################################################################################################### +FROM scratch as sum +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/sum-example /bin/sum-example +ENTRYPOINT [ "/bin/sum-example" ] diff --git a/pkg/reducestreamer/examples/sum/Makefile b/pkg/reducestreamer/examples/sum/Makefile new file mode 100644 index 00000000..ec0a4e6a --- /dev/null +++ b/pkg/reducestreamer/examples/sum/Makefile @@ -0,0 +1,10 @@ +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:v0.5.3" --target sum . + +clean: + -rm -rf ./dist diff --git a/pkg/reducestreamer/examples/sum/README.md b/pkg/reducestreamer/examples/sum/README.md new file mode 100644 index 00000000..1029db38 --- /dev/null +++ b/pkg/reducestreamer/examples/sum/README.md @@ -0,0 +1,3 @@ +# Sum + +This is a User Defined Function example which sum up the values for the given keys and output the sum when the sum is greater than 100 diff --git a/pkg/reducestreamer/examples/sum/go.mod b/pkg/reducestreamer/examples/sum/go.mod new file mode 100644 index 00000000..27de5f05 --- /dev/null +++ b/pkg/reducestreamer/examples/sum/go.mod @@ -0,0 +1,16 @@ +module sum + +go 1.20 + +require github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/reducestreamer/examples/sum/go.sum b/pkg/reducestreamer/examples/sum/go.sum new file mode 100644 index 00000000..32cb64f7 --- /dev/null +++ b/pkg/reducestreamer/examples/sum/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/reducestreamer/examples/sum/main.go b/pkg/reducestreamer/examples/sum/main.go new file mode 100644 index 00000000..8aa66720 --- /dev/null +++ b/pkg/reducestreamer/examples/sum/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "fmt" + "log" + "strconv" + + "github.com/numaproj/numaflow-go/pkg/reducestreamer" +) + +// Sum is a reducestreamer that sum up the values for the given keys and output the sum when the sum is greater than 100. +type Sum struct { +} + +func (s *Sum) ReduceStream(ctx context.Context, keys []string, inputCh <-chan reducestreamer.Datum, outputCh chan<- reducestreamer.Message, md reducestreamer.Metadata) { + // sum up values for the same keys + intervalWindow := md.IntervalWindow() + _ = intervalWindow + var resultKeys = keys + var resultVal []byte + var sum = 0 + // sum up the values + for d := range inputCh { + val := d.Value() + eventTime := d.EventTime() + _ = eventTime + watermark := d.Watermark() + _ = watermark + + v, err := strconv.Atoi(string(val)) + if err != nil { + fmt.Printf("unable to convert the value to int: %v\n", err) + continue + } + sum += v + + if sum >= 100 { + resultVal = []byte(strconv.Itoa(sum)) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) + sum = 0 + } + } + resultVal = []byte(strconv.Itoa(sum)) + outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) +} + +// SumCreator is the creator for the sum reducestreamer. +type SumCreator struct{} + +func (s *SumCreator) Create() reducestreamer.ReduceStreamer { + return &Sum{} +} + +func main() { + err := reducestreamer.NewServer(&SumCreator{}).Start(context.Background()) + if err != nil { + log.Panic("unable to start the server due to: ", err) + } +} diff --git a/pkg/reducestreamer/interface.go b/pkg/reducestreamer/interface.go new file mode 100644 index 00000000..40dd4faa --- /dev/null +++ b/pkg/reducestreamer/interface.go @@ -0,0 +1,58 @@ +package reducestreamer + +import ( + "context" + "time" +) + +// Datum contains methods to get the payload information. +type Datum interface { + Value() []byte + EventTime() time.Time + Watermark() time.Time +} + +// Metadata contains methods to get the metadata for the reduceStream operation. +type Metadata interface { + IntervalWindow() IntervalWindow +} + +// IntervalWindow contains methods to get the information for a given interval window. +type IntervalWindow interface { + StartTime() time.Time + EndTime() time.Time +} + +// ReduceStreamer is the interface of reduceStream function implementation. +type ReduceStreamer interface { + ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) +} + +// ReduceStreamerCreator is the interface which is used to create a ReduceStreamer. +type ReduceStreamerCreator interface { + // Create creates a ReduceStreamer, will be invoked once for every keyed window. + Create() ReduceStreamer +} + +// simpleReducerCreator is an implementation of ReduceStreamerCreator, which creates a ReduceStreamer for the given function. +type simpleReduceStreamerCreator struct { + f func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) +} + +// Create creates a Reducer for the given function. +func (s *simpleReduceStreamerCreator) Create() ReduceStreamer { + return reduceStreamFn(s.f) +} + +// SimpleCreatorWithReduceStreamFn creates a simple ReduceStreamerCreator for the given reduceStream function. +func SimpleCreatorWithReduceStreamFn(f func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata)) ReduceStreamerCreator { + return &simpleReduceStreamerCreator{f: f} +} + +// reduceStreamFn is a utility type used to convert a ReduceStream function to a ReduceStreamer. +type reduceStreamFn func(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) + +// ReduceStream implements the function of ReduceStreamer interface. +func (rf reduceStreamFn) ReduceStream(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message, md Metadata) { + rf(ctx, keys, inputCh, outputCh, md) +} diff --git a/pkg/reducestreamer/message.go b/pkg/reducestreamer/message.go new file mode 100644 index 00000000..2fdf5c64 --- /dev/null +++ b/pkg/reducestreamer/message.go @@ -0,0 +1,52 @@ +package reducestreamer + +import "fmt" + +var ( + DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ +) + +// Message is used to wrap the data return by reduceStream function +type Message struct { + value []byte + keys []string + tags []string +} + +// NewMessage creates a Message with value +func NewMessage(value []byte) Message { + return Message{value: value} +} + +// MessageToDrop creates a Message to be dropped +func MessageToDrop() Message { + return Message{value: []byte{}, tags: []string{DROP}} +} + +// WithKeys is used to assign the keys to the message +func (m Message) WithKeys(keys []string) Message { + m.keys = keys + return m +} + +// WithTags is used to assign the tags to the message +// tags will be used for conditional forwarding +func (m Message) WithTags(tags []string) Message { + m.tags = tags + return m +} + +// Keys returns message keys +func (m Message) Keys() []string { + return m.keys +} + +// Value returns message value +func (m Message) Value() []byte { + return m.value +} + +// Tags returns message tags +func (m Message) Tags() []string { + return m.tags +} diff --git a/pkg/reducestreamer/options.go b/pkg/reducestreamer/options.go new file mode 100644 index 00000000..7862fb03 --- /dev/null +++ b/pkg/reducestreamer/options.go @@ -0,0 +1,43 @@ +package reducestreamer + +import ( + "github.com/numaproj/numaflow-go/pkg/info" +) + +type options struct { + sockAddr string + maxMessageSize int + serverInfoFilePath string +} + +// Option is the interface to apply options. +type Option func(*options) + +func DefaultOptions() *options { + return &options{ + sockAddr: address, + maxMessageSize: defaultMaxMessageSize, + serverInfoFilePath: info.ServerInfoFilePath, + } +} + +// WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. +func WithMaxMessageSize(size int) Option { + return func(opts *options) { + opts.maxMessageSize = size + } +} + +// WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. +func WithSockAddr(addr string) Option { + return func(opts *options) { + opts.sockAddr = addr + } +} + +// WithServerInfoFilePath sets the server info file path to the given path. +func WithServerInfoFilePath(f string) Option { + return func(opts *options) { + opts.serverInfoFilePath = f + } +} diff --git a/pkg/reducestreamer/options_test.go b/pkg/reducestreamer/options_test.go new file mode 100644 index 00000000..947d80e0 --- /dev/null +++ b/pkg/reducestreamer/options_test.go @@ -0,0 +1,18 @@ +package reducestreamer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithMaxMessageSize(t *testing.T) { + var ( + size = 1024 * 1024 * 10 + opts = &options{ + maxMessageSize: defaultMaxMessageSize, + } + ) + WithMaxMessageSize(1024 * 1024 * 10)(opts) + assert.Equal(t, size, opts.maxMessageSize) +} diff --git a/pkg/reducestreamer/server.go b/pkg/reducestreamer/server.go new file mode 100644 index 00000000..1e180637 --- /dev/null +++ b/pkg/reducestreamer/server.go @@ -0,0 +1,56 @@ +package reducestreamer + +import ( + "context" + "fmt" + "os/signal" + "syscall" + + "github.com/numaproj/numaflow-go/pkg" + reducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" + "github.com/numaproj/numaflow-go/pkg/shared" +) + +// server is a reduceStream gRPC server. +type server struct { + svc *Service + opts *options +} + +// NewServer creates a new reduceStream server. +func NewServer(r ReduceStreamerCreator, inputOptions ...Option) numaflow.Server { + opts := DefaultOptions() + for _, inputOption := range inputOptions { + inputOption(opts) + } + s := new(server) + s.svc = new(Service) + s.svc.creatorHandle = r + s.opts = opts + return s +} + +// Start starts the reduceStream gRPC server. +func (r *server) Start(ctx context.Context) error { + ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // write server info to the file + // start listening on unix domain socket + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath) + if err != nil { + return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) + } + // close the listener + defer func() { _ = lis.Close() }() + + // create a grpc server + grpcServer := shared.CreateGRPCServer(r.opts.maxMessageSize) + defer grpcServer.GracefulStop() + + // register the reduceStream service + reducepb.RegisterReduceServer(grpcServer, r.svc) + + // start the grpc server + return shared.StartGRPCServer(ctxWithSignal, grpcServer, lis) +} diff --git a/pkg/reducestreamer/server_test.go b/pkg/reducestreamer/server_test.go new file mode 100644 index 00000000..aa595492 --- /dev/null +++ b/pkg/reducestreamer/server_test.go @@ -0,0 +1,37 @@ +package reducestreamer + +import ( + "context" + "os" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestReduceServer_Start(t *testing.T) { + socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") + defer func() { + _ = os.RemoveAll(socketFile.Name()) + }() + + serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") + defer func() { + _ = os.RemoveAll(serverInfoFile.Name()) + }() + + var reduceStreamHandle = func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))) + } + // note: using actual uds connection + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + err := NewServer(SimpleCreatorWithReduceStreamFn(reduceStreamHandle), WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) + assert.NoError(t, err) +} diff --git a/pkg/reducestreamer/service.go b/pkg/reducestreamer/service.go new file mode 100644 index 00000000..641ce86d --- /dev/null +++ b/pkg/reducestreamer/service.go @@ -0,0 +1,101 @@ +package reducestreamer + +import ( + "context" + "io" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + reducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +const ( + uds = "unix" + defaultMaxMessageSize = 1024 * 1024 * 64 + address = "/var/run/numaflow/reducestream.sock" + winStartTime = "x-numaflow-win-start-time" + winEndTime = "x-numaflow-win-end-time" + delimiter = ":" +) + +// Service implements the proto gen server interface and contains the reduceStream operation handler. +type Service struct { + reducepb.UnimplementedReduceServer + creatorHandle ReduceStreamerCreator +} + +// IsReady returns true to indicate the gRPC connection is ready. +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*reducepb.ReadyResponse, error) { + return &reducepb.ReadyResponse{Ready: true}, nil +} + +// ReduceFn applies a reduce function to a request stream and streams the results. +func (fs *Service) ReduceFn(stream reducepb.Reduce_ReduceFnServer) error { + var ( + err error + ctx = stream.Context() + g errgroup.Group + ) + + taskManager := newReduceTaskManager(fs.creatorHandle) + + // err group for the go routine which reads from the output channel and sends to the stream + g.Go(func() error { + for output := range taskManager.OutputChannel() { + sendErr := stream.Send(output) + if sendErr != nil { + return sendErr + } + } + return nil + }) + + // read messages from the stream and write the messages to corresponding channels + // if the channel is not created, create the channel and invoke the reduceFn + for { + d, recvErr := stream.Recv() + // if EOF, close all the channels + if recvErr == io.EOF { + taskManager.CloseAll() + break + } + if recvErr != nil { + // the error here is returned by stream.Recv() + // it's already a gRPC error + return recvErr + } + + // for Aligned, its just open or append operation + // close signal will be sent to all the reducers when grpc + // input stream gets EOF. + switch d.Operation.Event { + case reducepb.ReduceRequest_WindowOperation_OPEN: + // create a new reduce task and start the reduce operation + err = taskManager.CreateTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case reducepb.ReduceRequest_WindowOperation_APPEND: + // append the datum to the reduce task + err = taskManager.AppendToTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + } + } + + taskManager.WaitAll() + // wait for the go routine which reads from the output channel and sends to the stream to return + err = g.Wait() + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + + return nil +} diff --git a/pkg/reducestreamer/service_test.go b/pkg/reducestreamer/service_test.go new file mode 100644 index 00000000..498151cf --- /dev/null +++ b/pkg/reducestreamer/service_test.go @@ -0,0 +1,525 @@ +package reducestreamer + +import ( + "context" + "io" + "reflect" + "sort" + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + grpcmd "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/types/known/timestamppb" + + reducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +type ReduceFnServerTest struct { + ctx context.Context + inputCh chan *reducepb.ReduceRequest + outputCh chan *reducepb.ReduceResponse + grpc.ServerStream +} + +func NewReduceFnServerTest(ctx context.Context, + inputCh chan *reducepb.ReduceRequest, + outputCh chan *reducepb.ReduceResponse) *ReduceFnServerTest { + return &ReduceFnServerTest{ + ctx: ctx, + inputCh: inputCh, + outputCh: outputCh, + } +} + +func (u *ReduceFnServerTest) Send(list *reducepb.ReduceResponse) error { + u.outputCh <- list + return nil +} + +func (u *ReduceFnServerTest) Recv() (*reducepb.ReduceRequest, error) { + val, ok := <-u.inputCh + if !ok { + return val, io.EOF + } + return val, nil +} + +func (u *ReduceFnServerTest) Context() context.Context { + return u.ctx +} + +func TestService_ReduceFn(t *testing.T) { + + tests := []struct { + name string + handler func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) + input []*reducepb.ReduceRequest + expected []*reducepb.ReduceResponse + expectedErr bool + }{ + { + name: "reduce_fn_forward_msg_same_keys", + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) + }, + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "reduce_fn_forward_msg_multiple_keys", + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"}) + }, + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client3"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + { + Result: &reducepb.ReduceResponse_Result{ + Keys: []string{"client3_test"}, + Value: []byte(strconv.Itoa(60)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "reduce_fn_forward_msg_forward_to_all", + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- NewMessage([]byte(strconv.Itoa(sum))) + }, + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Value: []byte(strconv.Itoa(60)), + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + }, + { + name: "reduce_fn_forward_msg_drop_msg", + handler: func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { + sum := 0 + for val := range rch { + msgVal, _ := strconv.Atoi(string(val.Value())) + sum += msgVal + } + och <- MessageToDrop() + }, + input: []*reducepb.ReduceRequest{ + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_APPEND, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + { + Payload: &reducepb.ReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &reducepb.ReduceRequest_WindowOperation{ + Event: reducepb.ReduceRequest_WindowOperation_OPEN, + Windows: []*reducepb.Window{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + }, + }, + }, + }, + expected: []*reducepb.ReduceResponse{ + { + Result: &reducepb.ReduceResponse_Result{ + Tags: []string{DROP}, + Value: []byte{}, + }, + Window: &reducepb.Window{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + }, + EOF: false, + }, + }, + expectedErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := &Service{ + creatorHandle: SimpleCreatorWithReduceStreamFn(tt.handler), + } + // here's a trick for testing: + // because we are not using gRPC, we directly set a new incoming ctx + // instead of the regular outgoing context in the real gRPC connection. + ctx := grpcmd.NewIncomingContext(context.Background(), grpcmd.New(map[string]string{winStartTime: "60000", winEndTime: "120000"})) + + inputCh := make(chan *reducepb.ReduceRequest) + outputCh := make(chan *reducepb.ReduceResponse) + result := make([]*reducepb.ReduceResponse, 0) + + udfReduceFnStream := NewReduceFnServerTest(ctx, inputCh, outputCh) + + var wg sync.WaitGroup + var err error + + wg.Add(1) + go func() { + defer wg.Done() + err = fs.ReduceFn(udfReduceFnStream) + close(outputCh) + }() + + wg.Add(1) + go func() { + defer wg.Done() + for msg := range outputCh { + if !msg.EOF { + result = append(result, msg) + } + } + }() + + for _, val := range tt.input { + udfReduceFnStream.inputCh <- val + } + close(udfReduceFnStream.inputCh) + wg.Wait() + + if (err != nil) != tt.expectedErr { + t.Errorf("ReduceFn() error = %v, wantErr %v", err, tt.expectedErr) + return + } + + //sort and compare, since order of the output doesn't matter + sort.Slice(result, func(i, j int) bool { + return string(result[i].Result.Value) < string(result[j].Result.Value) + }) + + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("ReduceFn() got = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/reducestreamer/task_manager.go b/pkg/reducestreamer/task_manager.go new file mode 100644 index 00000000..a6d5feee --- /dev/null +++ b/pkg/reducestreamer/task_manager.go @@ -0,0 +1,177 @@ +package reducestreamer + +import ( + "context" + "fmt" + "strings" + "sync" + + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" +) + +// reduceStreamTask represents a task for a performing reduceStream operation. +type reduceStreamTask struct { + keys []string + window *v1.Window + reduceStreamer ReduceStreamer + inputCh chan Datum + outputCh chan Message + doneCh chan struct{} +} + +// buildReduceResponse builds the reduce response from the messages. +func (rt *reduceStreamTask) buildReduceResponse(message Message) *v1.ReduceResponse { + + response := &v1.ReduceResponse{ + Result: &v1.ReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + }, + Window: rt.window, + } + + return response +} + +func (rt *reduceStreamTask) buildEOFResponse() *v1.ReduceResponse { + response := &v1.ReduceResponse{ + Window: rt.window, + EOF: true, + } + + return response +} + +// uniqueKey returns the unique key for the reduceStream task to be used in the task manager to identify the task. +func (rt *reduceStreamTask) uniqueKey() string { + return fmt.Sprintf("%d:%d:%s", + rt.window.GetStart().AsTime().UnixMilli(), + rt.window.GetEnd().AsTime().UnixMilli(), + strings.Join(rt.keys, delimiter)) +} + +// reduceStreamTaskManager manages the reduceStream tasks. +type reduceStreamTaskManager struct { + creatorHandle ReduceStreamerCreator + tasks map[string]*reduceStreamTask + responseCh chan *v1.ReduceResponse +} + +func newReduceTaskManager(reduceStreamerCreator ReduceStreamerCreator) *reduceStreamTaskManager { + return &reduceStreamTaskManager{ + creatorHandle: reduceStreamerCreator, + tasks: make(map[string]*reduceStreamTask), + responseCh: make(chan *v1.ReduceResponse), + } +} + +// CreateTask creates a new reduceStream task and starts the reduceStream operation. +func (rtm *reduceStreamTaskManager) CreateTask(ctx context.Context, request *v1.ReduceRequest) error { + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("create operation error: invalid number of windows") + } + + md := NewMetadata(NewIntervalWindow(request.Operation.Windows[0].GetStart().AsTime(), + request.Operation.Windows[0].GetEnd().AsTime())) + + task := &reduceStreamTask{ + keys: request.GetPayload().GetKeys(), + window: request.Operation.Windows[0], + inputCh: make(chan Datum), + outputCh: make(chan Message), + doneCh: make(chan struct{}), + } + + key := task.uniqueKey() + rtm.tasks[key] = task + + go func() { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for message := range task.outputCh { + // write the output to the output channel, service will forward it to downstream + rtm.responseCh <- task.buildReduceResponse(message) + } + // send EOF + rtm.responseCh <- task.buildEOFResponse() + }() + + reduceStreamerHandle := rtm.creatorHandle.Create() + // invoke the reduceStream function + reduceStreamerHandle.ReduceStream(ctx, request.GetPayload().GetKeys(), task.inputCh, task.outputCh, md) + // close the output channel after the reduceStream function is done + close(task.outputCh) + // wait for the output to be forwarded + wg.Wait() + // send a done signal + close(task.doneCh) + }() + + // write the first message to the input channel + task.inputCh <- buildDatum(request) + return nil +} + +// AppendToTask writes the message to the reduceStream task. +// If the task is not found, it creates a new task and starts the reduceStream operation. +func (rtm *reduceStreamTaskManager) AppendToTask(ctx context.Context, request *v1.ReduceRequest) error { + if len(request.Operation.Windows) != 1 { + return fmt.Errorf("append operation error: invalid number of windows") + } + + gKey := generateKey(request.Operation.Windows[0], request.Payload.Keys) + task, ok := rtm.tasks[gKey] + + // if the task is not found, create a new task + if !ok { + return rtm.CreateTask(ctx, request) + } + + task.inputCh <- buildDatum(request) + return nil +} + +// OutputChannel returns the output channel for the reduceStream task manager to read the results. +func (rtm *reduceStreamTaskManager) OutputChannel() <-chan *v1.ReduceResponse { + return rtm.responseCh +} + +// WaitAll waits for all the reduceStream tasks to complete. +func (rtm *reduceStreamTaskManager) WaitAll() { + tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { + tasks = append(tasks, task) + } + + for _, task := range tasks { + <-task.doneCh + } + // after all the tasks are completed, close the output channel + close(rtm.responseCh) +} + +// CloseAll closes all the reduceStream tasks. +func (rtm *reduceStreamTaskManager) CloseAll() { + tasks := make([]*reduceStreamTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { + tasks = append(tasks, task) + } + + for _, task := range tasks { + close(task.inputCh) + } +} + +func generateKey(window *v1.Window, keys []string) string { + return fmt.Sprintf("%d:%d:%s", + window.GetStart().AsTime().UnixMilli(), + window.GetEnd().AsTime().UnixMilli(), + strings.Join(keys, delimiter)) +} + +func buildDatum(request *v1.ReduceRequest) Datum { + return NewHandlerDatum(request.Payload.GetValue(), request.Payload.EventTime.AsTime(), request.Payload.Watermark.AsTime()) +} diff --git a/pkg/reducestreamer/types.go b/pkg/reducestreamer/types.go new file mode 100644 index 00000000..1f19c27a --- /dev/null +++ b/pkg/reducestreamer/types.go @@ -0,0 +1,65 @@ +package reducestreamer + +import "time" + +// handlerDatum implements the Datum interface and is used in the reduceStream functions. +type handlerDatum struct { + value []byte + eventTime time.Time + watermark time.Time +} + +func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time) Datum { + return &handlerDatum{ + value: value, + eventTime: eventTime, + watermark: watermark, + } +} + +func (h *handlerDatum) Value() []byte { + return h.value +} + +func (h *handlerDatum) EventTime() time.Time { + return h.eventTime +} + +func (h *handlerDatum) Watermark() time.Time { + return h.watermark +} + +// intervalWindow implements IntervalWindow interface which will be passed as metadata +// to reduce handlers +type intervalWindow struct { + startTime time.Time + endTime time.Time +} + +func NewIntervalWindow(startTime time.Time, endTime time.Time) IntervalWindow { + return &intervalWindow{ + startTime: startTime, + endTime: endTime, + } +} + +func (i *intervalWindow) StartTime() time.Time { + return i.startTime +} + +func (i *intervalWindow) EndTime() time.Time { + return i.endTime +} + +// metadata implements Metadata interface which will be passed to reduceStream function. +type metadata struct { + intervalWindow IntervalWindow +} + +func NewMetadata(window IntervalWindow) Metadata { + return &metadata{intervalWindow: window} +} + +func (m *metadata) IntervalWindow() IntervalWindow { + return m.intervalWindow +} diff --git a/pkg/sessionreducer/doc.go b/pkg/sessionreducer/doc.go new file mode 100644 index 00000000..ddfbc457 --- /dev/null +++ b/pkg/sessionreducer/doc.go @@ -0,0 +1,5 @@ +// Package sessionreducer implements the server code for sessionReduce operation. + +// Examples: https://github.com/numaproj/numaflow-go/tree/main/pkg/sessionreducer/examples/ + +package sessionreducer diff --git a/pkg/sessionreducer/examples/counter/Dockerfile b/pkg/sessionreducer/examples/counter/Dockerfile new file mode 100644 index 00000000..7f1e3e5c --- /dev/null +++ b/pkg/sessionreducer/examples/counter/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/counter-example /bin/counter-example +RUN chmod +x /bin/counter-example + +#################################################################################################### +# counter +#################################################################################################### +FROM scratch as counter +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/counter-example /bin/counter-example +ENTRYPOINT [ "/bin/counter-example" ] diff --git a/pkg/sessionreducer/examples/counter/Makefile b/pkg/sessionreducer/examples/counter/Makefile new file mode 100644 index 00000000..eb4555da --- /dev/null +++ b/pkg/sessionreducer/examples/counter/Makefile @@ -0,0 +1,10 @@ +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/session-counter:v0.6.1" --target counter . + +clean: + -rm -rf ./dist diff --git a/pkg/sessionreducer/examples/counter/README.md b/pkg/sessionreducer/examples/counter/README.md new file mode 100644 index 00000000..5a72f269 --- /dev/null +++ b/pkg/sessionreducer/examples/counter/README.md @@ -0,0 +1,3 @@ +# Counter + +An example User Defined Function that counts the number of events. diff --git a/pkg/sessionreducer/examples/counter/go.mod b/pkg/sessionreducer/examples/counter/go.mod new file mode 100644 index 00000000..85215d54 --- /dev/null +++ b/pkg/sessionreducer/examples/counter/go.mod @@ -0,0 +1,19 @@ +module counter + +go 1.20 + +require ( + github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f + go.uber.org/atomic v1.11.0 +) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sessionreducer/examples/counter/go.sum b/pkg/sessionreducer/examples/counter/go.sum new file mode 100644 index 00000000..00ceacec --- /dev/null +++ b/pkg/sessionreducer/examples/counter/go.sum @@ -0,0 +1,32 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0 h1:aX6z3AIiJzA0XySqAZhP5ytZDZ3jcsQQnL81HP5mipU= +github.com/numaproj/numaflow-go v0.5.3-0.20231211071430-1231c4c278e0/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f h1:J43ekeRVzE6WGgkWl5oEQ+c4NT1i4VikMkygu4AeUYE= +github.com/numaproj/numaflow-go v0.6.1-0.20231219080635-d096c415a42f/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/sessionreducer/examples/counter/main.go b/pkg/sessionreducer/examples/counter/main.go new file mode 100644 index 00000000..8eafdffb --- /dev/null +++ b/pkg/sessionreducer/examples/counter/main.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "fmt" + "log" + "strconv" + + "go.uber.org/atomic" + + "github.com/numaproj/numaflow-go/pkg/sessionreducer" +) + +// Counter is a simple session reducer which counts the number of events in a session. +type Counter struct { + count *atomic.Int32 +} + +func (c *Counter) SessionReduce(ctx context.Context, keys []string, input <-chan sessionreducer.Datum, outputCh chan<- sessionreducer.Message) { + for range input { + c.count.Inc() + } + outputCh <- sessionreducer.NewMessage([]byte(fmt.Sprintf("%d", c.count.Load()))).WithKeys(keys) +} + +func (c *Counter) Accumulator(ctx context.Context) []byte { + return []byte(strconv.Itoa(int(c.count.Load()))) +} + +func (c *Counter) MergeAccumulator(ctx context.Context, accumulator []byte) { + val, err := strconv.Atoi(string(accumulator)) + if err != nil { + log.Println("unable to convert the accumulator value to int: ", err.Error()) + return + } + c.count.Add(int32(val)) +} + +func NewSessionCounter() sessionreducer.SessionReducer { + return &Counter{ + count: atomic.NewInt32(0), + } +} + +// SessionCounterCreator is the creator for the session reducer. +type SessionCounterCreator struct{} + +func (s *SessionCounterCreator) Create() sessionreducer.SessionReducer { + return NewSessionCounter() +} + +func main() { + sessionreducer.NewServer(&SessionCounterCreator{}).Start(context.Background()) +} diff --git a/pkg/sessionreducer/interface.go b/pkg/sessionreducer/interface.go new file mode 100644 index 00000000..905ae380 --- /dev/null +++ b/pkg/sessionreducer/interface.go @@ -0,0 +1,31 @@ +package sessionreducer + +import ( + "context" + "time" +) + +// Datum contains methods to get the payload information. +type Datum interface { + Value() []byte + EventTime() time.Time + Watermark() time.Time +} + +// SessionReducer is the interface which can be used to implement a session reduce operation. +type SessionReducer interface { + // SessionReduce applies a session reduce function to a request stream and streams the results. + SessionReduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message) + // Accumulator returns the accumulator for the session reducer, will be invoked when this session is merged + // with another session. + Accumulator(ctx context.Context) []byte + // MergeAccumulator merges the accumulator for the session reducer, will be invoked when another session is merged + // with this session. + MergeAccumulator(ctx context.Context, accumulator []byte) +} + +// SessionReducerCreator is the interface which can be used to create a session reducer. +type SessionReducerCreator interface { + // Create creates a session reducer, will be invoked once for every keyed window. + Create() SessionReducer +} diff --git a/pkg/sessionreducer/message.go b/pkg/sessionreducer/message.go new file mode 100644 index 00000000..8fcfe026 --- /dev/null +++ b/pkg/sessionreducer/message.go @@ -0,0 +1,52 @@ +package sessionreducer + +import "fmt" + +var ( + DROP = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__ +) + +// Message is used to wrap the data return by SessionReduce functions +type Message struct { + value []byte + keys []string + tags []string +} + +// NewMessage creates a Message with value +func NewMessage(value []byte) Message { + return Message{value: value} +} + +// MessageToDrop creates a Message to be dropped +func MessageToDrop() Message { + return Message{value: []byte{}, tags: []string{DROP}} +} + +// WithKeys is used to assign the keys to the message +func (m Message) WithKeys(keys []string) Message { + m.keys = keys + return m +} + +// WithTags is used to assign the tags to the message +// tags will be used for conditional forwarding +func (m Message) WithTags(tags []string) Message { + m.tags = tags + return m +} + +// Keys returns message keys +func (m Message) Keys() []string { + return m.keys +} + +// Value returns message value +func (m Message) Value() []byte { + return m.value +} + +// Tags returns message tags +func (m Message) Tags() []string { + return m.tags +} diff --git a/pkg/sessionreducer/options.go b/pkg/sessionreducer/options.go new file mode 100644 index 00000000..6e5da5b7 --- /dev/null +++ b/pkg/sessionreducer/options.go @@ -0,0 +1,43 @@ +package sessionreducer + +import ( + "github.com/numaproj/numaflow-go/pkg/info" +) + +type options struct { + sockAddr string + maxMessageSize int + serverInfoFilePath string +} + +// Option is the interface to apply options. +type Option func(*options) + +func DefaultOptions() *options { + return &options{ + sockAddr: address, + maxMessageSize: defaultMaxMessageSize, + serverInfoFilePath: info.ServerInfoFilePath, + } +} + +// WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. +func WithMaxMessageSize(size int) Option { + return func(opts *options) { + opts.maxMessageSize = size + } +} + +// WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. +func WithSockAddr(addr string) Option { + return func(opts *options) { + opts.sockAddr = addr + } +} + +// WithServerInfoFilePath sets the server info file path to the given path. +func WithServerInfoFilePath(f string) Option { + return func(opts *options) { + opts.serverInfoFilePath = f + } +} diff --git a/pkg/sessionreducer/options_test.go b/pkg/sessionreducer/options_test.go new file mode 100644 index 00000000..9dfbcfbf --- /dev/null +++ b/pkg/sessionreducer/options_test.go @@ -0,0 +1,18 @@ +package sessionreducer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithMaxMessageSize(t *testing.T) { + var ( + size = 1024 * 1024 * 10 + opts = &options{ + maxMessageSize: defaultMaxMessageSize, + } + ) + WithMaxMessageSize(1024 * 1024 * 10)(opts) + assert.Equal(t, size, opts.maxMessageSize) +} diff --git a/pkg/sessionreducer/server.go b/pkg/sessionreducer/server.go new file mode 100644 index 00000000..5b630bf3 --- /dev/null +++ b/pkg/sessionreducer/server.go @@ -0,0 +1,56 @@ +package sessionreducer + +import ( + "context" + "fmt" + "os/signal" + "syscall" + + "github.com/numaproj/numaflow-go/pkg" + sessionreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" + "github.com/numaproj/numaflow-go/pkg/shared" +) + +// server is a session reduce gRPC server. +type server struct { + svc *Service + opts *options +} + +// NewServer creates a new session reduce server. +func NewServer(r SessionReducerCreator, inputOptions ...Option) numaflow.Server { + opts := DefaultOptions() + for _, inputOption := range inputOptions { + inputOption(opts) + } + s := new(server) + s.svc = new(Service) + s.svc.creatorHandle = r + s.opts = opts + return s +} + +// Start starts the session reduce gRPC server. +func (r *server) Start(ctx context.Context) error { + ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // write server info to the file + // start listening on unix domain socket + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath) + if err != nil { + return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) + } + // close the listener + defer func() { _ = lis.Close() }() + + // create a grpc server + grpcServer := shared.CreateGRPCServer(r.opts.maxMessageSize) + defer grpcServer.GracefulStop() + + // register the sessionReduce service + sessionreducepb.RegisterSessionReduceServer(grpcServer, r.svc) + + // start the grpc server + return shared.StartGRPCServer(ctxWithSignal, grpcServer, lis) +} diff --git a/pkg/sessionreducer/server_test.go b/pkg/sessionreducer/server_test.go new file mode 100644 index 00000000..c020b6e2 --- /dev/null +++ b/pkg/sessionreducer/server_test.go @@ -0,0 +1,28 @@ +package sessionreducer + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestReduceServer_Start(t *testing.T) { + socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") + defer func() { + _ = os.RemoveAll(socketFile.Name()) + }() + + serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") + defer func() { + _ = os.RemoveAll(serverInfoFile.Name()) + }() + + // note: using actual uds connection + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + err := NewServer(&SessionSumCreator{}, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) + assert.NoError(t, err) +} diff --git a/pkg/sessionreducer/service.go b/pkg/sessionreducer/service.go new file mode 100644 index 00000000..b3ddd08b --- /dev/null +++ b/pkg/sessionreducer/service.go @@ -0,0 +1,113 @@ +package sessionreducer + +import ( + "context" + "io" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + sessionreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" +) + +const ( + uds = "unix" + defaultMaxMessageSize = 1024 * 1024 * 64 + address = "/var/run/numaflow/sessionreduce.sock" + delimiter = ":" +) + +// Service implements the proto gen server interface and contains the sesionreduce operation handler. +type Service struct { + sessionreducepb.UnimplementedSessionReduceServer + creatorHandle SessionReducerCreator +} + +// IsReady returns true to indicate the gRPC connection is ready. +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*sessionreducepb.ReadyResponse, error) { + return &sessionreducepb.ReadyResponse{Ready: true}, nil +} + +// SessionReduceFn applies a session reduce function to a request stream and streams the results. +func (fs *Service) SessionReduceFn(stream sessionreducepb.SessionReduce_SessionReduceFnServer) error { + + ctx := stream.Context() + taskManager := newReduceTaskManager(fs.creatorHandle) + // err group for the go routine which reads from the output channel and sends to the stream + var g errgroup.Group + + g.Go(func() error { + for output := range taskManager.OutputChannel() { + err := stream.Send(output) + if err != nil { + return err + } + } + return nil + }) + + for { + d, recvErr := stream.Recv() + + // if the stream is closed, break and wait for the tasks to return + if recvErr == io.EOF { + break + } + + if recvErr != nil { + statusErr := status.Errorf(codes.Internal, recvErr.Error()) + return statusErr + } + + // invoke the appropriate task manager method based on the operation + switch d.Operation.Event { + case sessionreducepb.SessionReduceRequest_WindowOperation_OPEN: + // create a new task and start the session reduce operation + // also append the datum to the task + err := taskManager.CreateTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE: + // close the task + taskManager.CloseTask(d) + case sessionreducepb.SessionReduceRequest_WindowOperation_APPEND: + // append the datum to the task + err := taskManager.AppendToTask(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case sessionreducepb.SessionReduceRequest_WindowOperation_MERGE: + // merge the tasks + err := taskManager.MergeTasks(ctx, d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + case sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND: + // expand the task + err := taskManager.ExpandTask(d) + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + } + + } + + // wait for all the tasks to return + taskManager.WaitAll() + + // wait for the go routine which reads from the output channel and sends to the stream to return + err := g.Wait() + if err != nil { + statusErr := status.Errorf(codes.Internal, err.Error()) + return statusErr + } + + return nil +} diff --git a/pkg/sessionreducer/service_test.go b/pkg/sessionreducer/service_test.go new file mode 100644 index 00000000..bcce002d --- /dev/null +++ b/pkg/sessionreducer/service_test.go @@ -0,0 +1,830 @@ +package sessionreducer + +import ( + "context" + "io" + "log" + "reflect" + "sort" + "strconv" + "sync" + "testing" + "time" + + "go.uber.org/atomic" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" + + sessionreducepb "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" +) + +type SessionReduceFnServerTest struct { + ctx context.Context + inputCh chan *sessionreducepb.SessionReduceRequest + outputCh chan *sessionreducepb.SessionReduceResponse + grpc.ServerStream +} + +func NewReduceFnServerTest(ctx context.Context, + inputCh chan *sessionreducepb.SessionReduceRequest, + outputCh chan *sessionreducepb.SessionReduceResponse) *SessionReduceFnServerTest { + return &SessionReduceFnServerTest{ + ctx: ctx, + inputCh: inputCh, + outputCh: outputCh, + } +} + +func (u *SessionReduceFnServerTest) Send(list *sessionreducepb.SessionReduceResponse) error { + u.outputCh <- list + return nil +} + +func (u *SessionReduceFnServerTest) Recv() (*sessionreducepb.SessionReduceRequest, error) { + val, ok := <-u.inputCh + if !ok { + return val, io.EOF + } + return val, nil +} + +func (u *SessionReduceFnServerTest) Context() context.Context { + return u.ctx +} + +type SessionSum struct { + sum *atomic.Int32 +} + +func (s *SessionSum) SessionReduce(ctx context.Context, keys []string, inputCh <-chan Datum, outputCh chan<- Message) { + for val := range inputCh { + msgVal, _ := strconv.Atoi(string(val.Value())) + s.sum.Add(int32(msgVal)) + } + outputCh <- NewMessage([]byte(strconv.Itoa(int(s.sum.Load())))).WithKeys([]string{keys[0] + "_test"}) +} + +func (s *SessionSum) Accumulator(ctx context.Context) []byte { + return []byte(strconv.Itoa(int(s.sum.Load()))) +} + +func (s *SessionSum) MergeAccumulator(ctx context.Context, accumulator []byte) { + val, err := strconv.Atoi(string(accumulator)) + if err != nil { + log.Println("unable to convert the accumulator value to int: ", err.Error()) + return + } + s.sum.Add(int32(val)) +} + +type SessionSumCreator struct { +} + +func (s *SessionSumCreator) Create() SessionReducer { + return NewSessionSum() +} + +func NewSessionSum() SessionReducer { + return &SessionSum{ + sum: atomic.NewInt32(0), + } +} + +func TestService_SessionReduceFn(t *testing.T) { + + tests := []struct { + name string + handler SessionReducerCreator + input []*sessionreducepb.SessionReduceRequest + expected []*sessionreducepb.SessionReduceResponse + expectedErr bool + }{ + { + name: "open_append_close", + handler: &SessionSumCreator{}, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client_test"}, + Value: []byte(strconv.Itoa(60)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(120000)), + Slot: "slot-0", + Keys: []string{"client"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "open_expand_close", + handler: &SessionSumCreator{}, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(75000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(79000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(75000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(79000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(75000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + EOF: false, + }, + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(79000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "open_merge_close", + handler: &SessionSumCreator{}, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(20)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + EOF: false, + }, + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(40)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + { + name: "open_expand_append_merge_close", + handler: &SessionSumCreator{}, + input: []*sessionreducepb.SessionReduceRequest{ + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(85000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_EXPAND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(88000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client1"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Payload: &sessionreducepb.SessionReduceRequest_Payload{ + Keys: []string{"client2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_APPEND, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(75000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_MERGE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(70000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + { + Start: timestamppb.New(time.UnixMilli(78000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + { + Operation: &sessionreducepb.SessionReduceRequest_WindowOperation{ + Event: sessionreducepb.SessionReduceRequest_WindowOperation_CLOSE, + KeyedWindows: []*sessionreducepb.KeyedWindow{ + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + { + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + }, + }, + }, + }, + expected: []*sessionreducepb.SessionReduceResponse{ + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client1_test"}, + Value: []byte(strconv.Itoa(40)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(95000)), + Slot: "slot-0", + Keys: []string{"client1"}, + }, + EOF: false, + }, + { + Result: &sessionreducepb.SessionReduceResponse_Result{ + Keys: []string{"client2_test"}, + Value: []byte(strconv.Itoa(80)), + }, + KeyedWindow: &sessionreducepb.KeyedWindow{ + Start: timestamppb.New(time.UnixMilli(60000)), + End: timestamppb.New(time.UnixMilli(98000)), + Slot: "slot-0", + Keys: []string{"client2"}, + }, + EOF: false, + }, + }, + expectedErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := &Service{ + creatorHandle: tt.handler, + } + // here's a trick for testing: + // because we are not using gRPC, we directly set a new incoming ctx + // instead of the regular outgoing context in the real gRPC connection. + + inputCh := make(chan *sessionreducepb.SessionReduceRequest) + outputCh := make(chan *sessionreducepb.SessionReduceResponse) + result := make([]*sessionreducepb.SessionReduceResponse, 0) + + udfReduceFnStream := NewReduceFnServerTest(context.Background(), inputCh, outputCh) + + var wg sync.WaitGroup + var err error + + wg.Add(1) + go func() { + defer wg.Done() + err = fs.SessionReduceFn(udfReduceFnStream) + close(outputCh) + }() + + wg.Add(1) + go func() { + defer wg.Done() + for msg := range outputCh { + if !msg.EOF { + result = append(result, msg) + } + } + }() + + for _, val := range tt.input { + udfReduceFnStream.inputCh <- val + } + close(udfReduceFnStream.inputCh) + wg.Wait() + + if (err != nil) != tt.expectedErr { + t.Errorf("ReduceFn() error = %v, wantErr %v", err, tt.expectedErr) + return + } + + //sort and compare, since order of the output doesn't matter + sort.Slice(result, func(i, j int) bool { + return string(result[i].Result.Value) < string(result[j].Result.Value) + }) + + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("SessionReduceFn() got = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go new file mode 100644 index 00000000..2abd58c6 --- /dev/null +++ b/pkg/sessionreducer/task_manager.go @@ -0,0 +1,307 @@ +package sessionreducer + +import ( + "context" + "fmt" + "strings" + "sync" + + "go.uber.org/atomic" + + v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1" +) + +// sessionReduceTask represents a task for a performing session reduce operation. +type sessionReduceTask struct { + keyedWindow *v1.KeyedWindow + sessionReducer SessionReducer + inputCh chan Datum + outputCh chan Message + doneCh chan struct{} + merged *atomic.Bool +} + +// buildSessionReduceResponse builds the session reduce response from the messages. +func (rt *sessionReduceTask) buildSessionReduceResponse(message Message) *v1.SessionReduceResponse { + + response := &v1.SessionReduceResponse{ + Result: &v1.SessionReduceResponse_Result{ + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + }, + KeyedWindow: rt.keyedWindow, + } + + return response +} + +// buildEOFResponse builds the EOF response for the session reduce task. +func (rt *sessionReduceTask) buildEOFResponse() *v1.SessionReduceResponse { + response := &v1.SessionReduceResponse{ + KeyedWindow: rt.keyedWindow, + EOF: true, + } + + return response +} + +// uniqueKey returns the unique key for the session reduce task to be used in the task manager to identify the task. +func (rt *sessionReduceTask) uniqueKey() string { + return fmt.Sprintf("%d:%d:%s", + rt.keyedWindow.GetStart().AsTime().UnixMilli(), + rt.keyedWindow.GetEnd().AsTime().UnixMilli(), + strings.Join(rt.keyedWindow.GetKeys(), delimiter)) +} + +// sessionReduceTaskManager manages the tasks for a session reduce operation. +type sessionReduceTaskManager struct { + creatorHandle SessionReducerCreator + tasks map[string]*sessionReduceTask + responseCh chan *v1.SessionReduceResponse + rw sync.RWMutex +} + +func newReduceTaskManager(sessionReducerFactory SessionReducerCreator) *sessionReduceTaskManager { + return &sessionReduceTaskManager{ + creatorHandle: sessionReducerFactory, + tasks: make(map[string]*sessionReduceTask), + responseCh: make(chan *v1.SessionReduceResponse), + } +} + +// CreateTask creates a new task and starts the session reduce operation. +func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1.SessionReduceRequest) error { + rtm.rw.Lock() + + // for create operation, there should be exactly one keyedWindow + if len(request.Operation.KeyedWindows) != 1 { + return fmt.Errorf("create operation error: invalid number of windows in the request - %d", len(request.Operation.KeyedWindows)) + } + + task := &sessionReduceTask{ + keyedWindow: request.Operation.KeyedWindows[0], + sessionReducer: rtm.creatorHandle.Create(), + inputCh: make(chan Datum), + outputCh: make(chan Message), + doneCh: make(chan struct{}), + merged: atomic.NewBool(false), + } + + // add the task to the tasks list + key := task.uniqueKey() + rtm.tasks[key] = task + + rtm.rw.Unlock() + + go func() { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for message := range task.outputCh { + if !task.merged.Load() { + // write the output to the output channel, service will forward it to downstream + // if the task is merged to another task, we don't need to send the response + rtm.responseCh <- task.buildSessionReduceResponse(message) + } + } + if !task.merged.Load() { + // send EOF + rtm.responseCh <- task.buildEOFResponse() + } + }() + + task.sessionReducer.SessionReduce(ctx, task.keyedWindow.GetKeys(), task.inputCh, task.outputCh) + // close the output channel and wait for the response to be forwarded + close(task.outputCh) + wg.Wait() + // send a done signal + close(task.doneCh) + // delete the task from the tasks list + rtm.rw.Lock() + delete(rtm.tasks, key) + rtm.rw.Unlock() + }() + + // send the datum to the task if the payload is not nil + if request.Payload != nil { + task.inputCh <- buildDatum(request.Payload) + } + + return nil +} + +// AppendToTask writes the message to the reduce task. +// If the task is not found, it will create a new task and starts the session reduce operation. +func (rtm *sessionReduceTaskManager) AppendToTask(ctx context.Context, request *v1.SessionReduceRequest) error { + + // for append operation, there should be exactly one keyedWindow + if len(request.Operation.KeyedWindows) != 1 { + return fmt.Errorf("append operation error: invalid number of windows in the request - %d", len(request.Operation.KeyedWindows)) + } + + rtm.rw.RLock() + task, ok := rtm.tasks[generateKey(request.Operation.KeyedWindows[0])] + rtm.rw.RUnlock() + + // if the task is not found, create a new task and start the session reduce operation + if !ok { + return rtm.CreateTask(ctx, request) + } + + // send the datum to the task if the payload is not nil + if request.Payload != nil { + task.inputCh <- buildDatum(request.Payload) + } + return nil +} + +// CloseTask closes the input channel of the session reduce tasks. +func (rtm *sessionReduceTaskManager) CloseTask(request *v1.SessionReduceRequest) { + rtm.rw.RLock() + tasksToBeClosed := make([]*sessionReduceTask, 0, len(request.Operation.KeyedWindows)) + for _, window := range request.Operation.KeyedWindows { + key := generateKey(window) + task, ok := rtm.tasks[key] + if ok { + tasksToBeClosed = append(tasksToBeClosed, task) + } + } + rtm.rw.RUnlock() + + for _, task := range tasksToBeClosed { + close(task.inputCh) + } +} + +// MergeTasks merges the session reduce tasks. It will create a new task with the merged window and +// merges the accumulators from the other tasks to the merged task. +func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1.SessionReduceRequest) error { + rtm.rw.Lock() + mergedWindow := request.Operation.KeyedWindows[0] + + tasks := make([]*sessionReduceTask, 0, len(request.Operation.KeyedWindows)) + + // merge the aggregators from the other tasks + for _, window := range request.Operation.KeyedWindows { + key := generateKey(window) + task, ok := rtm.tasks[key] + if !ok { + rtm.rw.Unlock() + return fmt.Errorf("merge operation error: task not found for %s", key) + } + task.merged.Store(true) + tasks = append(tasks, task) + + // mergedWindow will be the largest window which contains all the windows + if window.GetStart().AsTime().Before(mergedWindow.GetStart().AsTime()) { + mergedWindow.Start = window.Start + } + + if window.GetEnd().AsTime().After(mergedWindow.GetEnd().AsTime()) { + mergedWindow.End = window.End + } + } + + rtm.rw.Unlock() + + accumulators := make([][]byte, 0, len(tasks)) + // close all the tasks and collect the accumulators + for _, task := range tasks { + close(task.inputCh) + // wait for the task to complete + <-task.doneCh + accumulators = append(accumulators, task.sessionReducer.Accumulator(ctx)) + } + + // create a new task with the merged keyedWindow + err := rtm.CreateTask(ctx, &v1.SessionReduceRequest{ + Payload: nil, + Operation: &v1.SessionReduceRequest_WindowOperation{ + Event: v1.SessionReduceRequest_WindowOperation_OPEN, + KeyedWindows: []*v1.KeyedWindow{mergedWindow}, + }, + }) + if err != nil { + return err + } + + rtm.rw.RLock() + mergedTask, ok := rtm.tasks[generateKey(mergedWindow)] + rtm.rw.RUnlock() + if !ok { + return fmt.Errorf("merge operation error: merged task not found for key %s", mergedWindow.String()) + } + // merge the accumulators using the merged task + for _, aggregator := range accumulators { + mergedTask.sessionReducer.MergeAccumulator(ctx, aggregator) + } + + return nil +} + +// ExpandTask expands session reduce task. It will update the keyedWindow of the task +// expects request.Operation.KeyedWindows to have exactly two windows. The first is the old window and the second +// is the new expanded window. +func (rtm *sessionReduceTaskManager) ExpandTask(request *v1.SessionReduceRequest) error { + // for expand operation, there should be exactly two windows + if len(request.Operation.KeyedWindows) != 2 { + return fmt.Errorf("expand operation error: expected exactly two windows") + } + + rtm.rw.Lock() + key := generateKey(request.Operation.KeyedWindows[0]) + task, ok := rtm.tasks[key] + if !ok { + rtm.rw.Unlock() + return fmt.Errorf("expand operation error: task not found for key - %s", key) + } + + // assign the new keyedWindow to the task + task.keyedWindow = request.Operation.KeyedWindows[1] + + // delete the old entry from the tasks map and add the new entry + delete(rtm.tasks, key) + rtm.tasks[task.uniqueKey()] = task + rtm.rw.Unlock() + + // send the datum to the task if the payload is not nil + if request.Payload != nil { + task.inputCh <- buildDatum(request.GetPayload()) + } + + return nil +} + +// OutputChannel returns the output channel of the task manager to read the results. +func (rtm *sessionReduceTaskManager) OutputChannel() <-chan *v1.SessionReduceResponse { + return rtm.responseCh +} + +// WaitAll waits for all the pending reduce tasks to complete. +func (rtm *sessionReduceTaskManager) WaitAll() { + rtm.rw.RLock() + tasks := make([]*sessionReduceTask, 0, len(rtm.tasks)) + for _, task := range rtm.tasks { + tasks = append(tasks, task) + } + rtm.rw.RUnlock() + + for _, task := range tasks { + <-task.doneCh + } + // after all the tasks are completed, close the output channel + close(rtm.responseCh) +} + +func generateKey(keyedWindows *v1.KeyedWindow) string { + return fmt.Sprintf("%d:%d:%s", + keyedWindows.GetStart().AsTime().UnixMilli(), + keyedWindows.GetEnd().AsTime().UnixMilli(), + strings.Join(keyedWindows.GetKeys(), delimiter)) +} + +func buildDatum(payload *v1.SessionReduceRequest_Payload) Datum { + return NewHandlerDatum(payload.GetValue(), payload.EventTime.AsTime(), payload.Watermark.AsTime()) +} diff --git a/pkg/sessionreducer/types.go b/pkg/sessionreducer/types.go new file mode 100644 index 00000000..ee5c4966 --- /dev/null +++ b/pkg/sessionreducer/types.go @@ -0,0 +1,30 @@ +package sessionreducer + +import "time" + +// handlerDatum implements the Datum interface and is used in the SessionReduce functions. +type handlerDatum struct { + value []byte + eventTime time.Time + watermark time.Time +} + +func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time) Datum { + return &handlerDatum{ + value: value, + eventTime: eventTime, + watermark: watermark, + } +} + +func (h *handlerDatum) Value() []byte { + return h.value +} + +func (h *handlerDatum) EventTime() time.Time { + return h.eventTime +} + +func (h *handlerDatum) Watermark() time.Time { + return h.watermark +}