From e7d08546f7dd0c13d84f40ea2eed8436d7502d64 Mon Sep 17 00:00:00 2001 From: a3hadi Date: Mon, 11 Mar 2024 15:13:34 -0400 Subject: [PATCH] modify makefiles Signed-off-by: a3hadi --- development.md | 4 ++++ pkg/mapper/examples/even_odd/Makefile | 10 +++++++++- pkg/mapper/examples/flatmap/Makefile | 10 +++++++++- pkg/mapper/examples/forward_message/Makefile | 10 +++++++++- pkg/mapper/examples/retry/Makefile | 10 +++++++++- pkg/mapper/examples/tickgen/Makefile | 10 +++++++++- pkg/mapstreamer/examples/flatmap_stream/Makefile | 10 +++++++++- pkg/reducer/examples/counter/Makefile | 10 +++++++++- pkg/reducer/examples/sum/Makefile | 10 +++++++++- pkg/reducestreamer/examples/counter/Makefile | 10 +++++++++- pkg/reducestreamer/examples/sum/Makefile | 10 +++++++++- pkg/sessionreducer/examples/counter/Makefile | 10 +++++++++- pkg/sideinput/examples/simple-sideinput/Makefile | 3 +-- pkg/sideinput/examples/simple-sideinput/udf/Makefile | 1 - pkg/sinker/examples/log/Makefile | 10 +++++++++- pkg/sourcer/examples/simple_source/Makefile | 10 +++++++++- .../examples/assign_event_time/Makefile | 10 +++++++++- .../examples/event_time_filter/Makefile | 10 +++++++++- 18 files changed, 140 insertions(+), 18 deletions(-) diff --git a/development.md b/development.md index f05ac822..d76ed24d 100644 --- a/development.md +++ b/development.md @@ -9,6 +9,10 @@ The SDK uses local references in the `go.mod` file, i.e. the `github.com/numapro `numaflow-go` directory. Thus, you can automatically test your SDK changes without the need to push to your forked repository or modify the `go.mod` file. Simply make your changes, build and push the desired example image (with an appropriate tag, such as `test`), and you are ready to use it in any pipeline. +Each example directory has a Makefile which can be used to build, tag, and push images. In most cases, the `image` target should be used. +However, `buildx`, which is used to support multiple architectures, does not make a built image available locally. In the case that a developer +wants this functionality, they can use the `test-image` target. + ### Deploying 1. Create a PR for your changes. Once merged, it will trigger a workflow to build and push the images for all the examples, diff --git a/pkg/mapper/examples/even_odd/Makefile b/pkg/mapper/examples/even_odd/Makefile index 87609b7d..26f58970 100644 --- a/pkg/mapper/examples/even_odd/Makefile +++ b/pkg/mapper/examples/even_odd/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/even-odd-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-even-odd:stable" --target even-odd . + docker buildx build -t "quay.io/numaio/numaflow-go/map-even-odd:${TAG}" --platform linux/amd64,linux/arm64 --target even-odd . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/map-even-odd:${TAG}" --target even-odd . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-even-odd:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/flatmap/Makefile b/pkg/mapper/examples/flatmap/Makefile index 87a75e8a..df27a2a2 100644 --- a/pkg/mapper/examples/flatmap/Makefile +++ b/pkg/mapper/examples/flatmap/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-flatmap:stable" --target flatmap . + docker buildx build -t "quay.io/numaio/numaflow-go/map-flatmap:${TAG}" --platform linux/amd64,linux/arm64 --target flatmap . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/map-flatmap:${TAG}" --target flatmap . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-flatmap:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/forward_message/Makefile b/pkg/mapper/examples/forward_message/Makefile index 67ee2531..80549c26 100644 --- a/pkg/mapper/examples/forward_message/Makefile +++ b/pkg/mapper/examples/forward_message/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/forward-message-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-forward-message:stable" --target forward-message . + docker buildx build -t "quay.io/numaio/numaflow-go/map-forward-message:${TAG}" --platform linux/amd64,linux/arm64 --target forward-message . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/map-forward-message:${TAG}" --target forward-message . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-forward-message:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/retry/Makefile b/pkg/mapper/examples/retry/Makefile index bdd63e02..17f610a4 100644 --- a/pkg/mapper/examples/retry/Makefile +++ b/pkg/mapper/examples/retry/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/retry main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-retry:stable" --target retry . + docker buildx build -t "quay.io/numaio/numaflow-go/map-retry:${TAG}" --platform linux/amd64,linux/arm64 --target retry . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/map-retry:${TAG}" --target map-retry . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-retry:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/tickgen/Makefile b/pkg/mapper/examples/tickgen/Makefile index 34580b90..bfc9f1db 100644 --- a/pkg/mapper/examples/tickgen/Makefile +++ b/pkg/mapper/examples/tickgen/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/tickgen-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-tickgen:stable" --target tickgen . + docker buildx build -t "quay.io/numaio/numaflow-go/map-tickgen:${TAG}" --platform linux/amd64,linux/arm64 --target tickgen . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/map-tickgen:${TAG}" --target tickgen . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-tickgen:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapstreamer/examples/flatmap_stream/Makefile b/pkg/mapstreamer/examples/flatmap_stream/Makefile index 8f7b8966..1f4649f9 100644 --- a/pkg/mapstreamer/examples/flatmap_stream/Makefile +++ b/pkg/mapstreamer/examples/flatmap_stream/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-stream-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:stable" --target flatmap_stream . + docker buildx build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}" --platform linux/amd64,linux/arm64 --target flatmap_stream . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}" --target flatmap_stream . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/counter/Makefile b/pkg/reducer/examples/counter/Makefile index b04f0ace..bda436aa 100644 --- a/pkg/reducer/examples/counter/Makefile +++ b/pkg/reducer/examples/counter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-counter:stable" --target counter . + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-counter:${TAG}" --target counter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-counter:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/sum/Makefile b/pkg/reducer/examples/sum/Makefile index f3e9e8d1..6c9cc0fd 100644 --- a/pkg/reducer/examples/sum/Makefile +++ b/pkg/reducer/examples/sum/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-sum:stable" --target sum . + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sum:${TAG}" --platform linux/amd64,linux/arm64 --target sum . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-sum:${TAG}" --target sum . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sum:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducestreamer/examples/counter/Makefile b/pkg/reducestreamer/examples/counter/Makefile index bc6e4c4d..7686d8ff 100644 --- a/pkg/reducestreamer/examples/counter/Makefile +++ b/pkg/reducestreamer/examples/counter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:stable" --target counter . + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}" --target counter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducestreamer/examples/sum/Makefile b/pkg/reducestreamer/examples/sum/Makefile index 9cf335b6..c51575ea 100644 --- a/pkg/reducestreamer/examples/sum/Makefile +++ b/pkg/reducestreamer/examples/sum/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:stable" --target sum . + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}" --platform linux/amd64,linux/arm64 --target sum . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}" --target sum . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sessionreducer/examples/counter/Makefile b/pkg/sessionreducer/examples/counter/Makefile index 19c37944..3ef9fe9d 100644 --- a/pkg/sessionreducer/examples/counter/Makefile +++ b/pkg/sessionreducer/examples/counter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/session-counter:stable" --target counter . + docker buildx build -t "quay.io/numaio/numaflow-go/session-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/session-counter:${TAG}" --target counter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/session-counter:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sideinput/examples/simple-sideinput/Makefile b/pkg/sideinput/examples/simple-sideinput/Makefile index 0bf98d12..0b7c220f 100644 --- a/pkg/sideinput/examples/simple-sideinput/Makefile +++ b/pkg/sideinput/examples/simple-sideinput/Makefile @@ -9,10 +9,9 @@ build: image: build docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push -# If testing locally, there isn't a need for multi-arch images .PHONY: test_image test_image: build - docker build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --target udf-sideinput . + docker build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --target sideinput . @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sideinput-example:${TAG}"; fi clean: diff --git a/pkg/sideinput/examples/simple-sideinput/udf/Makefile b/pkg/sideinput/examples/simple-sideinput/udf/Makefile index 8eb24fa6..7110a3e3 100644 --- a/pkg/sideinput/examples/simple-sideinput/udf/Makefile +++ b/pkg/sideinput/examples/simple-sideinput/udf/Makefile @@ -9,7 +9,6 @@ build: image: build docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target udf-sideinput . --push -# If testing locally, there isn't a need for multi-arch images .PHONY: test_image test_image: build docker build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --target udf-sideinput . diff --git a/pkg/sinker/examples/log/Makefile b/pkg/sinker/examples/log/Makefile index 37e42e94..03393287 100644 --- a/pkg/sinker/examples/log/Makefile +++ b/pkg/sinker/examples/log/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/log-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/sink-log:stable" --target log . + docker buildx build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --platform linux/amd64,linux/arm64 --target log . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --target log . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sink-log:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sourcer/examples/simple_source/Makefile b/pkg/sourcer/examples/simple_source/Makefile index 61c35287..fdad0de4 100644 --- a/pkg/sourcer/examples/simple_source/Makefile +++ b/pkg/sourcer/examples/simple_source/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/simple-source-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/source-simple-source:stable" --target simple-source . + docker buildx build -t "quay.io/numaio/numaflow-go/source-simple-source:${TAG}" --platform linux/amd64,linux/arm64 --target simple-source . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/source-simple-source:${TAG}" --target simple-source . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/source-simple-source:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sourcetransformer/examples/assign_event_time/Makefile b/pkg/sourcetransformer/examples/assign_event_time/Makefile index 34a183ef..47ed8352 100644 --- a/pkg/sourcetransformer/examples/assign_event_time/Makefile +++ b/pkg/sourcetransformer/examples/assign_event_time/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/assign-event-time-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/mapt-assign-event-time:stable" --target assign-event-time . + docker buildx build -t "quay.io/numaio/numaflow-go/mapt-assign-event-time:${TAG}" --platform linux/amd64,linux/arm64 --target assign-event-time . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/mapt-assign-event-time:${TAG}" --target assign-event-time . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/mapt-assign-event-time:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sourcetransformer/examples/event_time_filter/Makefile b/pkg/sourcetransformer/examples/event_time_filter/Makefile index b94f3144..8a0f464d 100644 --- a/pkg/sourcetransformer/examples/event_time_filter/Makefile +++ b/pkg/sourcetransformer/examples/event_time_filter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/event-time-filter-example main.go .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/mapt-event-time-filter:stable" --target event-time-filter . + docker buildx build -t "quay.io/numaio/numaflow-go/mapt-event-time-filter:${TAG}" --platform linux/amd64,linux/arm64 --target event-time-filter . --push + +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/mapt-event-time-filter:${TAG}" --target event-time-filter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/mapt-event-time-filter:${TAG}"; fi clean: -rm -rf ./dist