From cfe40bee31f38e47d0bc8e769ad8ea2d742d682e Mon Sep 17 00:00:00 2001 From: a3hadi Date: Wed, 20 Mar 2024 16:47:30 -0400 Subject: [PATCH] feat: implement update script and build/push workflow Signed-off-by: a3hadi --- .github/workflows/build-push.yaml | 37 ++++ .github/workflows/{ci.yaml => run-tests.yaml} | 0 development.md | 65 +++++++ examples/pom.xml | 91 +++++++++- update_examples.sh | 161 ++++++++++++++++++ 5 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build-push.yaml rename .github/workflows/{ci.yaml => run-tests.yaml} (100%) create mode 100644 development.md create mode 100755 update_examples.sh diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml new file mode 100644 index 00000000..fd6b9cd2 --- /dev/null +++ b/.github/workflows/build-push.yaml @@ -0,0 +1,37 @@ +name: Docker Publish + +on: + push: + branches: [ main ] + tags: + - '*' + +jobs: + docker_publish: + name: Build, Tag, and Push Image + runs-on: ubuntu-latest + + strategy: + matrix: + execution_ids: [ + "mapt-event-time-filter-function", "flat-map-stream", "map-flatmap", + "even-odd", "simple-sink", "reduce-sum", "reduce-stream-sum", + "map-forward-message", "reduce-counter", "sideinput-example", + "udf-sideinput-example", "source-simple-source" + ] + + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Quay.io registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.NUMAIO_USERNAME }} + password: ${{ secrets.NUMAIO_PASSWORD }} + - name: Build, tag, and push images + run: ./update_examples.sh --build-push-example ${{ matrix.execution_ids }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/run-tests.yaml similarity index 100% rename from .github/workflows/ci.yaml rename to .github/workflows/run-tests.yaml diff --git a/development.md b/development.md new file mode 100644 index 00000000..2be0be52 --- /dev/null +++ b/development.md @@ -0,0 +1,65 @@ +# Development + +This document explains the development process for the Numaflow Java SDK. + +### Testing + +After you make certain changes to the Java SDK implementation, you would like to use it to build a +brand new UDF image, and run it on a pipeline to verify the image continues working well with the Numaflow platform. +In order to do this you can follow these steps: + +1. Make the changes. +2. First increase the version in the root `pom.xml` file (increasing the PATCH version by 1 is suggested, example: X.X.0 -> X.X.1). +This can be achieved by running the following at the root directory: + ```shell + ./update_examples -u + ``` +3. You can now build the test UDF image and push it to a remote repository. You can do this by simply running: + ```shell + ./update_examples -bpe -t + ``` + the -bpe flag builds and pushes the given example to the remote registry, must be + the id tag used within the execution element of the desired example. It is recommended to specify a tag, + like `test`, as the default is `stable` which is used by the Github Actions CI. For example in order to push + the example for the reducer SDK, specifically the sum function, we would run: + ```shell + ./update_examples -bpe reduce-sum -t test + ``` + If you would like to build and push all the examples at once, you can run: + ```shell + ./update_examples -bp -t test + ``` +4. Once the version is updated and the desired image is built, tagged, and pushed, you can run a pipeline +using the image, i.e. we can use Numaflow e2e test cases to verify the changes. + - Checkout the latest Numaflow repository + - Find the test case and update the image and path in the test pipeline yaml file + - Run `make Test*` + +### Deploying + +After confiring that your changes pass local testing, you should roll back the version updates before +publishing the PR. The real version updates can be raised in a separate PR, which is discussed +in the [After Release](#after-release) section. + +1. Revert both the `pom.xml` changes by re-running the update script with the version flag: +```shell +./update_examples -u +``` +2. Clean up testing artifacts, i.e. delete any images that are no longer used on the remote registry repositories, +so that they are not flooded with testing images. +3. Create a PR. Once your PR has been merged a Github Actions workflow will be triggered to build, tag, and push +all example images, so that they are using the most up-to-date version of the SDK, i.e. the one including your +changes. + +### After Release + +Once a new release has been made, and its corresponding version tag exists on the remote repo, we want to update the dependency +management files to reflect this new version: + +```shell +./update_examples.sh -u + ``` + +Similar to the deployment step, create a PR for the changes created (which should just be the version bump +in the `pom.xml` files in root and example directories), and a Github Actions workflow will be triggered to +build, tag, and push all example images. diff --git a/examples/pom.xml b/examples/pom.xml index 63df86f2..7cdaeb86 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -34,7 +34,7 @@ 3.3.0 - event-time-filter-function + mapt-event-time-filter-function package dockerBuild @@ -46,7 +46,7 @@ - numaflow-java-examples/event-time-filter + numaflow-java-examples/mapt-event-time-filter-function @@ -68,7 +68,7 @@ - flat-map + map-flatmap package dockerBuild @@ -151,6 +151,91 @@ + + map-forward-message + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.map.forward.ForwardFunction + + + + numaflow-java-examples/map-forward-message + + + + + reduce-counter + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.reduce.count.CounterFactory + + + + numaflow-java-examples/reduce-counter + + + + + sideinput-example + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.sideinput.simple.SimpleSideInput + + + + numaflow-java-examples/sideinput-example + + + + + udf-sideinput-example + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.sideinput.udf.SimpleMapWithSideInput + + + + numaflow-java-examples/udf-sideinput-example + + + + + source-simple-source + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.source.simple.SimpleSource + + + + numaflow-java-examples/source-simple-source + + + diff --git a/update_examples.sh b/update_examples.sh new file mode 100755 index 00000000..7670da28 --- /dev/null +++ b/update_examples.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +function show_help () { + echo "Usage: $0 [-h|--help | -t|--tag ] (-bp|--build-push | -bpe|--build-push-example | -u|--update )" + echo " -h, --help Display help message and exit" + echo " -bp, --build-push Build all the examples and push them to the quay.io registry" + echo " -bpe, --build-push-example Build the given example id (found in examples/pom.xml), and push it to the quay.io registry" + echo " -t, --tag To be optionally used with -bpe or -bp. Specify the tag to build with. Default tag: stable" + echo " -u, --update Update the pom.xml files in the root and example directories to the specified version" +} + +if [ $# -eq 0 ]; then + echo "Error: provide at least one argument" >&2 + show_help + exit 1 +fi + +usingHelp=0 +usingBuildPush=0 +usingBuildPushExample=0 +usingVersion=0 +usingTag=0 +version="" +executionID="" +tag="stable" + +function handle_options () { + while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usingHelp=1 + ;; + -bp | --build-push) + usingBuildPush=1 + ;; + -bpe | --build-push-example) + if [ -z "$2" ]; then + echo "execution ID of example not specified." >&2 + show_help + exit 1 + fi + + usingBuildPushExample=1 + executionID=$2 + shift + ;; + -t | --tag) + if [ -z "$2" ]; then + echo "Tag not specified." >&2 + show_help + exit 1 + fi + + usingTag=1 + tag=$2 + shift + ;; + -u | --update) + if [ -z "$2" ]; then + echo "Version not specified." >&2 + show_help + exit 1 + fi + + usingVersion=1 + version=$2 + shift + ;; + *) + echo "Invalid option: $1" >&2 + show_help + exit 1 + ;; + esac + shift + done +} + +handle_options "$@" + +if (( usingBuildPush + usingBuildPushExample + usingHelp + usingVersion > 1 )); then + echo "Only one of '-h', '-bp', '-bpe', or '-u' is allowed at a time" >&2 + show_help + exit 1 +fi + +if (( (usingTag + usingHelp + usingVersion > 1) || (usingTag && usingBuildPush + usingBuildPushExample == 0) )); then + echo "Can only use -t with -bp or -bpe" >&2 + show_help + exit 1 +fi + +if [ -n "$version" ]; then + echo "Using version: $version" +fi + +if [ -n "$executionID" ]; then + echo "Updating example: $executionID" +fi + +if [ -n "$tag" ] && (( ! usingHelp )) && (( ! usingVersion )); then + echo "Using tag: $tag" +fi + +executionIDs=("mapt-event-time-filter-function" "flat-map-stream" "map-flatmap" \ + "even-odd" "simple-sink" "reduce-sum" "reduce-stream-sum" \ + "map-forward-message" "reduce-counter" "sideinput-example" \ + "udf-sideinput-example" "source-simple-source" + ) + +function dockerPublish () { + echo "Docker publish for example: $1" + if ! docker tag numaflow-java-examples/"$1":latest quay.io/numaio/numaflow-java/"$1":"$2"; then + echo "Error: failed to tag example $1 with tag $2" >&2 + exit 1 + fi + if ! docker push quay.io/numaio/numaflow-java/reduce-sum:"$2"; then + echo "Error: failed to push example $1 with tag $2" >&2 + exit 1 + fi +} + +if (( usingBuildPush )); then + if ! mvn clean install; then + echo "Error: failed to mvn clean install in root directory" >&2 + exit 1 + fi + cd examples || exit + if ! mvn clean install; then + echo "Error: failed to build images in examples directory" >&2 + exit 1 + fi + for id in "${executionIDs[@]}" + do + dockerPublish "$id" "$tag" + done +elif (( usingBuildPushExample )); then + if ! mvn clean install; then + echo "Error: failed to mvn clean install in root directory" >&2 + exit 1 + fi + cd examples || exit + if ! mvn jib:dockerBuild@"$executionID"; then + echo "Error: failed to build example image $executionID" >&2 + exit 1 + fi + dockerPublish "$executionID" "$tag" +elif (( usingVersion )); then + if ! mvn versions:set -DnewVersion="$version"; then + echo "Error: failed to update version in pom.xml file in root directory" >&2 + exit 1 + fi + + cd examples || exit + if ! mvn versions:use-dep-version -Dincludes=io.numaproj.numaflow -DdepVersion="$version" -DforceVersion=true; then + echo "Error: failed to update version in pom.xml file in examples directory" >&2 + exit 1 + fi +elif (( usingHelp )); then + show_help +fi