From 502262f3e13ef144f8ed9fa1bf95b701334ac0d0 Mon Sep 17 00:00:00 2001 From: Abdullah Yildirim Date: Mon, 25 Mar 2024 10:14:54 -0400 Subject: [PATCH] chore: refactor developer guide (#101) Signed-off-by: a3hadi --- .github/workflows/build-push.yaml | 2 +- development.md | 73 ------------------- examples/README.md | 63 ++++++++++++++++ update_examples.sh => hack/update_examples.sh | 20 ++--- 4 files changed, 70 insertions(+), 88 deletions(-) delete mode 100644 development.md create mode 100644 examples/README.md rename update_examples.sh => hack/update_examples.sh (89%) diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index 1d98487..2dc11fc 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -39,4 +39,4 @@ jobs: username: ${{ secrets.NUMAIO_USERNAME }} password: ${{ secrets.NUMAIO_PASSWORD }} - name: Build, tag, and push images - run: ./update_examples.sh --build-push-example ${{ matrix.execution_ids }} + run: ./hack/update_examples.sh --build-push-example ${{ matrix.execution_ids }} diff --git a/development.md b/development.md deleted file mode 100644 index ad3abf8..0000000 --- a/development.md +++ /dev/null @@ -1,73 +0,0 @@ -# Development - -This document explains the development process for the Numaflow Java SDK. - -### Testing - -After you make certain changes to the Java SDK implementation, you would like to use it to build a -brand new UDF image, and run it on a pipeline to verify the image continues working well with the Numaflow platform. -In order to do this you can follow these steps: - -1. Make the changes. -2. Update your local repository using maven, to reflect your latest changes. This can be done in the root -directory by running: `mvn clean install` (make sure that the version in the `pom.xml in the root and examples -directories match). -3. Build your test images, and test them in a pipeline. - -There exists a script `update_examples.sh`, which you can also use while testing, for your convenience. - -If you would like to build and push a specific example, you can run: - -```shell -./update_examples -bpe -t - ``` -The -bpe flag builds and pushes to the remote registry. must be -the id tag used within the execution element of the desired example. It is recommended to specify a tag, -like `test`, as the default is `stable`, which is used by the Github Actions CI. For example, in order to push -the example, for the reducer SDK, specifically the sum function, you would run: -```shell -./update_examples -bpe reduce-sum -t test -``` -If you would like to build and push all the examples at once, you can run: -```shell -./update_examples -bp -t test -``` -Both `bpe` and `bp` first build a local image with the naming convention -`numaflow-java-examples/:`, which then gets pushed as -`quay.io/numaio/numaflow-java/:`. If while testing, you would like -to build your image locally, without pushing to quay.io, you can run the following in the -examples directory: - -```shell -mvn jib:dockerBuild@ -Ddocker.tag= -``` - -If you would like to build all example images without pushing, in the examples directory you can also run: -```shell -mvn clean install -Ddocker.tag= -``` - -### Deploying - -After confirming that your changes pass local testing: - -1. Clean up testing artifacts. -2. Create a PR. Once your PR has been merged a Github Actions workflow will be triggered to build, tag (with `stable`), and push -all example images, so that they are using the most up-to-date version of the SDK, i.e. the one including your -changes. - -### Before Release - -Before releasing a new SDK version, make sure to update all references from the old version to the new one. -For example, the version in the `pom.xml` in the root and example directories. For [reference -](https://github.com/numaproj/numaflow-java/pull/89/files#diff-9c5fb3d1b7e3b0f54bc5c4182965c4fe1f9023d449017cece3005d3f90e8e4d8). - -### Adding a New Example - -If you add a new example, there are a few steps to follow in order for it to be used by the update script and the Docker -Publish workflow: - -1. Add the example to the `pom.xml` file in the examples directory, within an execution element. Note that the -id tag you specify must be exactly the same as the quay.io repository name for the example. -2. Add the id tag you specified in step 1 to the executionIDs array in `update_examples.sh`. -3. Add the id tag you specified in step 1 to the execution_ids matrix in `build-push.yaml`. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..df3955e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,63 @@ +# Developer Guide + +This document explains the development process for the Numaflow Java SDK. + +### Testing + +After you make certain changes to the Java SDK implementation, you would like to use it to build a +brand new UDF image, and run it on a pipeline to verify the image continues working well with the Numaflow platform. +In order to do this you can follow these steps: + +1. Make the changes +2. Update your local repository using maven, to reflect your latest changes. This can be done in the root +directory by running: `mvn clean install` (make sure that the version in the `pom.xml` in the root and examples +directories match) +3. In the examples directory you can then run `mvn clean install -Ddocker.tag=`, which will build `numaflow-java-examples`, run unit tests, +and then build all example images. Now that the images are available locally, +you can take the desired image and test it in a pipeline + +If you want to build and push all the example images at once, you can run: +```shell +./hack/update_examples -bp -t +``` +The default tag is `stable`, but it is recommended you specify your own for testing purposes, as the Github Actions CI uses the `stable` tag. +This consistent tag name is used so that the tags in the [E2E test pipelines](https://github.com/numaproj/numaflow/tree/main/test) do not need to be +updated each time an SDK change is made. + +You can alternatively build and push a specific example image by running the following: +```shell +./hack/update_examples -bpe -t + ``` +Both `-bpe` and `-bp` first build a local image with the naming convention +`numaflow-java-examples/:`, which then gets pushed as +`quay.io/numaio/numaflow-java/:`. If you want to build and tag all images locally, +without pushing to quay.io, as mentioned in step 3, run: `mvn clean install -Ddocker.tag=`. + +Note: before running the script, ensure that through the CLI, you are logged into quay.io. + +### Deploying + +After confirming that your changes pass local testing: + +1. Clean up testing artifacts +2. Create a PR. Once your PR has been merged, a Github Actions workflow (Docker Publish) will be triggered, to build, tag (with `stable`), and push +all example images. This ensures that all example images are using the most up-to-date version of the SDK, i.e. the one including your +changes + +### Before Release + +Before releasing a new SDK version, make sure to update all references from the old version to the new one. +For example, the version in the `pom.xml` in the root and example directories should be updated (for [reference +](https://github.com/numaproj/numaflow-java/pull/89/files#diff-9c5fb3d1b7e3b0f54bc5c4182965c4fe1f9023d449017cece3005d3f90e8e4d8)). After making these changes, create a PR. Once merged, it will trigger the Docker Publish workflow, and should be included in the release. +As a result, the correct SDK version will always be printed in the server information logs, and the example images will +always be using the latest changes (due to pointing to the local maven repository that is built). + +### Adding a New Example + +If you add a new example, there are a few steps to follow in order for it to be used by the update script and the Docker +Publish workflow: + +1. Add the example to the `pom.xml` file in the examples directory, within an execution element. Note that the +`id` tag you specify must be exactly the same as the quay.io repository name for the example +2. Add the `id` tag you specified in step 1 to the `executionIDs` array in `update_examples.sh` +3. Add the `id` tag you specified in step 1 to the `execution_ids` matrix in `build-push.yaml` diff --git a/update_examples.sh b/hack/update_examples.sh similarity index 89% rename from update_examples.sh rename to hack/update_examples.sh index e4cb78e..ea52461 100755 --- a/update_examples.sh +++ b/hack/update_examples.sh @@ -76,10 +76,6 @@ if (( (usingTag + usingHelp > 1) || (usingTag && usingBuildPush + usingBuildPush exit 1 fi -if [ -n "$version" ]; then - echo "Using version: $version" -fi - if [ -n "$executionID" ]; then echo "Updating example: $executionID" fi @@ -106,7 +102,7 @@ function dockerPublish () { fi } -if (( usingBuildPush )); then +function build () { if ! mvn clean install; then echo "Error: failed to mvn clean install in root directory" >&2 exit 1 @@ -116,20 +112,16 @@ if (( usingBuildPush )); then echo "Error: failed to build images in examples directory" >&2 exit 1 fi +} + +if (( usingBuildPush )); then + build for id in "${executionIDs[@]}" do dockerPublish "$id" done elif (( usingBuildPushExample )); then - if ! mvn clean install; then - echo "Error: failed to mvn clean install in root directory" >&2 - exit 1 - fi - cd examples || exit - if ! mvn clean install jib:dockerBuild@"$executionID" -Ddocker.tag="$tag"; then - echo "Error: failed to build example image $executionID" >&2 - exit 1 - fi + build dockerPublish "$executionID" elif (( usingHelp )); then show_help