refactor

Signed-off-by: a3hadi <[email protected]>

.github/workflows/build-push.yaml

@@ -15,11 +15,12 @@ jobs:
       max-parallel: 6
       matrix:
         repository: [
-          "map-even-odd",
-          "map-flatmap", "map-forward-message", "map-retry", "map-tickgen",
-          "map-flatmap-stream", "reduce-counter", "reduce-sum", "reduce-stream-counter",
-          "reduce-stream-sum", "session-counter", "sideinput-example", "udf-sideinput-example",
-          "sink-log", "source-simple-source", "mapt-assign-event-time", "mapt-event-time-filter"
+          "sideinput-example", "udf-sideinput-example"
+#          "map-even-odd",
+#          "map-flatmap", "map-forward-message", "map-retry", "map-tickgen",
+#          "map-flatmap-stream", "reduce-counter", "reduce-sum", "reduce-stream-counter",
+#          "reduce-stream-sum", "session-counter", "sideinput-example", "udf-sideinput-example",
+#          "sink-log", "source-simple-source", "mapt-assign-event-time", "mapt-event-time-filter"
         ]
 
     steps:
@@ -35,11 +36,5 @@ jobs:
           registry: quay.io
           username: ${{ secrets.NUMAIO_USERNAME }}
           password: ${{ secrets.NUMAIO_PASSWORD }}
-      - name: Build and tag images
-        run: ./update_examples.sh --build
-      - name: Push images
-        env:
-          IMAGE_PREFIX: "quay.io/numaio/numaflow-go"
-          TAG: "stable"
-        run: |
-          docker push ${{ env.IMAGE_PREFIX }}/${{ matrix.repository }}:${{ env.TAG }}
+      - name: Build, tag, and push images
+        run: ./update_examples.sh --build-push

development.md

@@ -7,15 +7,15 @@ This document explains the development process for the Numaflow Go SDK.
 
 The SDK uses local references in the `go.mod` file, i.e. the `github.com/numaproj/numaflow-go` dependency is pointing to your local
 `numaflow-go` directory. Thus, you can automatically test your SDK changes without the need to push to your forked repository or modify the `go.mod` file.
-Simply make your changes, build and push the desired example image, and you are ready to use it in any pipeline.
+Simply make your changes, build and push the desired example image (with an appropriate tag, such as `test`), and you are ready to use it in any pipeline.
 
 ### Deploying
 
 1. Create a PR for your changes. Once merged, it will trigger a workflow to build and push the images for all the examples, 
 with the tag `stable`. This consistent tag name is used so that the tags in the [E2E test pipelines](https://github.com/numaproj/numaflow/tree/main/test) do not need to be 
 updated each time a new change is made. 
 2. After the changes have been merged it is encouraged to update the dependency management files so that the version 
-displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done simply by getting the
+displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done by getting the
 commit SHA of the merged changes and using it with the update script: 
 ```shell
 ./update_examples -c <commit-sha>

pkg/sideinput/examples/simple-sideinput/Makefile

@@ -1,10 +1,19 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sideinput-example main.go
 
 .PHONY: image
 image: build
-	docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:stable" --platform linux/amd64,linux/arm64 --output type=local --target sideinput .
+	docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push
+
+# If testing locally, there isn't a need for multi-arch images
+.PHONY: test_image
+test_image: build
+	docker build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --target udf-sideinput .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sideinput-example:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/sideinput/examples/simple-sideinput/udf/Makefile

@@ -1,13 +1,19 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/udf-sideinput-example main.go
 
 .PHONY: image
-#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command
 image: build
-	docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:stable" --platform linux/amd64,linux/arm64 --output type=local --target udf-sideinput .
-
+	docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target udf-sideinput . --push
 
+# If testing locally, there isn't a need for multi-arch images
+.PHONY: test_image
+test_image: build
+	docker build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --target udf-sideinput .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

update_examples.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 function show_help () {
-    echo "Usage: $0 [-h|--help] (-b|--build | -c|--commit-sha <commit_sha>)"
-    echo "  -h, --help         Display help message and exit"
-    echo "  -b, --build        Build the docker images of all the examples"
-    echo "  -c, --commit-sha   Update all of the examples to depend on the numaflow-go version with the specified SHA or version"
+    echo "Usage: $0 [-h|--help] (-bp|--build-push | -u|--update <SDK-version>)"
+    echo "  -h, --help          Display help message and exit"
+    echo "  -bp, --build-push   Build the docker images of all the examples and push them to the quay.io registry (with tag: stable)"
+    echo "  -u, --update        Update all of the examples to depend on the numaflow-go version with the specified SHA or version"
 }
 
 function traverse_examples () {
@@ -32,28 +32,28 @@ if [ $# -eq 0 ]; then
 fi
 
 usingHelp=0
-usingBuild=0
-usingSHA=0
-commit_sha=""
+usingBuildPush=0
+usingVersion=0
+version=""
 
 function handle_options () {
   while [ $# -gt 0 ]; do
     case "$1" in
       -h | --help)
         usingHelp=1
         ;;
-      -b | --build)
-        usingBuild=1
+      -bp | --build-push)
+        usingBuildPush=1
         ;;
-      -c | --commit-sha)
-        usingSHA=1
+      -u | --update)
+        usingVersion=1
         if [ -z "$2" ]; then
           echo "Commit SHA or version not specified." >&2
           show_help
           exit 1
         fi
 
-        commit_sha=$2
+        version=$2
         shift
         ;;
       *)
@@ -68,20 +68,20 @@ function handle_options () {
 
 handle_options "$@"
 
-if (( usingBuild + usingSHA + usingHelp > 1 )); then
-  echo "Only one of '-h', '-b', '-c' is allowed at a time" >&2
+if (( usingBuildPush + usingVersion + usingHelp > 1 )); then
+  echo "Only one of '-h', '-bp', '-u' is allowed at a time" >&2
   show_help
   exit 1
 fi
 
-if [ -n "$commit_sha" ]; then
- echo "Will update to: $commit_sha"
+if [ -n "$version" ]; then
+ echo "Will update to: $version"
 fi
 
-if (( usingBuild )); then
+if (( usingBuildPush )); then
   traverse_examples "make image"
-elif (( usingSHA )); then
-  traverse_examples "go get github.com/numaproj/numaflow-go@$commit_sha" "go mod tidy"
+elif (( usingVersion )); then
+  traverse_examples "go get github.com/numaproj/numaflow-go@$version" "go mod tidy"
 elif (( usingHelp )); then
   show_help
 fi