diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index fe0fdfc2..d0ea7e06 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -15,11 +15,12 @@ jobs: max-parallel: 6 matrix: repository: [ - "map-even-odd", - "map-flatmap", "map-forward-message", "map-retry", "map-tickgen", - "map-flatmap-stream", "reduce-counter", "reduce-sum", "reduce-stream-counter", - "reduce-stream-sum", "session-counter", "sideinput-example", "udf-sideinput-example", - "sink-log", "source-simple-source", "mapt-assign-event-time", "mapt-event-time-filter" + "sideinput-example", "udf-sideinput-example" +# "map-even-odd", +# "map-flatmap", "map-forward-message", "map-retry", "map-tickgen", +# "map-flatmap-stream", "reduce-counter", "reduce-sum", "reduce-stream-counter", +# "reduce-stream-sum", "session-counter", "sideinput-example", "udf-sideinput-example", +# "sink-log", "source-simple-source", "mapt-assign-event-time", "mapt-event-time-filter" ] steps: @@ -35,11 +36,5 @@ jobs: registry: quay.io username: ${{ secrets.NUMAIO_USERNAME }} password: ${{ secrets.NUMAIO_PASSWORD }} - - name: Build and tag images - run: ./update_examples.sh --build - - name: Push images - env: - IMAGE_PREFIX: "quay.io/numaio/numaflow-go" - TAG: "stable" - run: | - docker push ${{ env.IMAGE_PREFIX }}/${{ matrix.repository }}:${{ env.TAG }} + - name: Build, tag, and push images + run: ./update_examples.sh --build-push diff --git a/development.md b/development.md index a8cae1ab..f05ac822 100644 --- a/development.md +++ b/development.md @@ -7,7 +7,7 @@ This document explains the development process for the Numaflow Go SDK. The SDK uses local references in the `go.mod` file, i.e. the `github.com/numaproj/numaflow-go` dependency is pointing to your local `numaflow-go` directory. Thus, you can automatically test your SDK changes without the need to push to your forked repository or modify the `go.mod` file. -Simply make your changes, build and push the desired example image, and you are ready to use it in any pipeline. +Simply make your changes, build and push the desired example image (with an appropriate tag, such as `test`), and you are ready to use it in any pipeline. ### Deploying @@ -15,7 +15,7 @@ Simply make your changes, build and push the desired example image, and you are with the tag `stable`. This consistent tag name is used so that the tags in the [E2E test pipelines](https://github.com/numaproj/numaflow/tree/main/test) do not need to be updated each time a new change is made. 2. After the changes have been merged it is encouraged to update the dependency management files so that the version -displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done simply by getting the +displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done by getting the commit SHA of the merged changes and using it with the update script: ```shell ./update_examples -c diff --git a/pkg/sideinput/examples/simple-sideinput/Makefile b/pkg/sideinput/examples/simple-sideinput/Makefile index 97828e12..0bf98d12 100644 --- a/pkg/sideinput/examples/simple-sideinput/Makefile +++ b/pkg/sideinput/examples/simple-sideinput/Makefile @@ -1,10 +1,19 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sideinput-example main.go .PHONY: image image: build - docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:stable" --platform linux/amd64,linux/arm64 --output type=local --target sideinput . + docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +# If testing locally, there isn't a need for multi-arch images +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --target udf-sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sideinput-example:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sideinput/examples/simple-sideinput/udf/Makefile b/pkg/sideinput/examples/simple-sideinput/udf/Makefile index de9e50dd..8eb24fa6 100644 --- a/pkg/sideinput/examples/simple-sideinput/udf/Makefile +++ b/pkg/sideinput/examples/simple-sideinput/udf/Makefile @@ -1,13 +1,19 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/udf-sideinput-example main.go .PHONY: image -#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command image: build - docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:stable" --platform linux/amd64,linux/arm64 --output type=local --target udf-sideinput . - + docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target udf-sideinput . --push +# If testing locally, there isn't a need for multi-arch images +.PHONY: test_image +test_image: build + docker build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --target udf-sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}"; fi clean: -rm -rf ./dist diff --git a/update_examples.sh b/update_examples.sh index 09b9fdbe..893bd68a 100755 --- a/update_examples.sh +++ b/update_examples.sh @@ -1,10 +1,10 @@ #!/bin/bash function show_help () { - echo "Usage: $0 [-h|--help] (-b|--build | -c|--commit-sha )" - echo " -h, --help Display help message and exit" - echo " -b, --build Build the docker images of all the examples" - echo " -c, --commit-sha Update all of the examples to depend on the numaflow-go version with the specified SHA or version" + echo "Usage: $0 [-h|--help] (-bp|--build-push | -u|--update )" + echo " -h, --help Display help message and exit" + echo " -bp, --build-push Build the docker images of all the examples and push them to the quay.io registry (with tag: stable)" + echo " -u, --update Update all of the examples to depend on the numaflow-go version with the specified SHA or version" } function traverse_examples () { @@ -32,9 +32,9 @@ if [ $# -eq 0 ]; then fi usingHelp=0 -usingBuild=0 -usingSHA=0 -commit_sha="" +usingBuildPush=0 +usingVersion=0 +version="" function handle_options () { while [ $# -gt 0 ]; do @@ -42,18 +42,18 @@ function handle_options () { -h | --help) usingHelp=1 ;; - -b | --build) - usingBuild=1 + -bp | --build-push) + usingBuildPush=1 ;; - -c | --commit-sha) - usingSHA=1 + -u | --update) + usingVersion=1 if [ -z "$2" ]; then echo "Commit SHA or version not specified." >&2 show_help exit 1 fi - commit_sha=$2 + version=$2 shift ;; *) @@ -68,20 +68,20 @@ function handle_options () { handle_options "$@" -if (( usingBuild + usingSHA + usingHelp > 1 )); then - echo "Only one of '-h', '-b', '-c' is allowed at a time" >&2 +if (( usingBuildPush + usingVersion + usingHelp > 1 )); then + echo "Only one of '-h', '-bp', '-u' is allowed at a time" >&2 show_help exit 1 fi -if [ -n "$commit_sha" ]; then - echo "Will update to: $commit_sha" +if [ -n "$version" ]; then + echo "Will update to: $version" fi -if (( usingBuild )); then +if (( usingBuildPush )); then traverse_examples "make image" -elif (( usingSHA )); then - traverse_examples "go get github.com/numaproj/numaflow-go@$commit_sha" "go mod tidy" +elif (( usingVersion )); then + traverse_examples "go get github.com/numaproj/numaflow-go@$version" "go mod tidy" elif (( usingHelp )); then show_help fi