feat: implement update script #102 (#137)

Signed-off-by: a3hadi <[email protected]>

.hack/update_examples.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+
+function show_help () {
+    echo "Usage: $0 [-h|--help | -t|--tag <tag>] (-bp|--build-push | -bpe|--build-push-example <path> | -us|--update-sha <commit-sha> | -uv|--update-version <version>)"
+    echo "  -h, --help                   Display help message and exit"
+    echo "  -bp, --build-push            Build the Dockerfiles of all the examples and push them to the quay.io registry"
+    echo "  -bpe, --build-push-example   Build the Dockerfile of the given example directory path, and push it to the quay.io registry"
+    echo "  -t, --tag                    To be optionally used with -bpe or -bp. Specify the tag to build with. Default tag: stable"
+    echo "  -us, --update-sha            Update all of the examples to depend on the specified commit SHA"
+    echo "  -uv, --update-version        Update all of the examples to depend on the specified version"
+}
+#update sha update version
+
+function traverse_examples () {
+  find examples -name "pyproject.toml"  | while read -r line;
+  do
+      dir="$(dirname "${line}")"
+      cd "$dir" || exit
+      # TODO: rewrite asyncio-reduce example using latest SDK version, as it is currently using old methods
+      if [ "$dir" == "examples/developer_guide" ] || [ "$dir" == examples/reduce/asyncio-reduce ]; then
+          cd ~- || exit
+          continue
+      fi
+
+      for command in "$@"
+      do
+        if ! $command; then
+          echo "Error: failed $command in $dir" >&2
+          exit 1
+        fi
+      done
+
+      cd ~- || exit
+  done
+}
+
+if [ $# -eq 0 ]; then
+  echo "Error: provide at least one argument" >&2
+  show_help
+  exit 1
+fi
+
+usingHelp=0
+usingBuildPush=0
+usingBuildPushExample=0
+usingSHA=0
+usingVersion=0
+usingTag=0
+sha=""
+version=""
+directoryPath=""
+tag="stable"
+
+function handle_options () {
+  while [ $# -gt 0 ]; do
+    case "$1" in
+      -h | --help)
+        usingHelp=1
+        ;;
+      -bp | --build-push)
+        usingBuildPush=1
+        ;;
+      -bpe | --build-push-example)
+        if [ -z "$2" ]; then
+          echo "Directory path not specified." >&2
+          show_help
+          exit 1
+        fi
+
+        usingBuildPushExample=1
+        directoryPath=$2
+        shift
+        ;;
+      -t | --tag)
+        if [ -z "$2" ]; then
+          echo "Tag not specified." >&2
+          show_help
+          exit 1
+        fi
+
+        usingTag=1
+        tag=$2
+        shift
+        ;;
+      -us | --update-sha)
+        if [ -z "$2" ]; then
+          echo "Commit SHA not specified." >&2
+          show_help
+          exit 1
+        fi
+
+        usingSHA=1
+        sha=$2
+        shift
+        ;;
+      -uv | --update-version)
+        if [ -z "$2" ]; then
+          echo "Version not specified." >&2
+          show_help
+          exit 1
+        fi
+
+        usingVersion=1
+        version=$2
+        shift
+        ;;
+      *)
+        echo "Invalid option: $1" >&2
+        show_help
+        exit 1
+        ;;
+    esac
+    shift
+  done
+}
+
+handle_options "$@"
+
+if (( usingBuildPush + usingBuildPushExample + usingSHA + usingHelp + usingVersion > 1 )); then
+  echo "Only one of '-h', '-bp', '-bpe', '-us', or '-uv' is allowed at a time" >&2
+  show_help
+  exit 1
+fi
+
+if (( (usingTag + usingSHA + usingHelp + usingVersion > 1) || (usingTag && usingBuildPush + usingBuildPushExample == 0) )); then
+  echo "Can only use -t with -bp or -bpe" >&2
+  show_help
+  exit 1
+fi
+
+if [ -n "$sha" ]; then
+ echo "Using SHA: $sha"
+fi
+
+if [ -n "$version" ]; then
+ echo "Using version: $version"
+fi
+
+if [ -n "$directoryPath" ]; then
+ echo "Dockerfile path to use: $directoryPath"
+fi
+
+if [ -n "$tag" ] && (( ! usingSHA )) && (( ! usingHelp )) && (( ! usingVersion )); then
+ echo "Using tag: $tag"
+fi
+
+if (( usingBuildPush )); then
+  traverse_examples "make image-push TAG=$tag"
+elif (( usingBuildPushExample )); then
+   cd "./$directoryPath" || exit
+   if ! make image-push TAG="$tag"; then
+     echo "Error: failed to run make image-push in $directoryPath" >&2
+     exit 1
+   fi
+elif (( usingSHA )); then
+  traverse_examples "poetry add git+https://github.com/numaproj/numaflow-python.git@$sha"
+elif (( usingVersion )); then
+  poetry version "$version"
+  traverse_examples "poetry add pynumaflow@~$version"
+elif (( usingHelp )); then
+  show_help
+fi

examples/developer_guide/README.md

@@ -1,11 +1,75 @@
 # Developer Guide
 
-This example is for numaflow-python contributors/developers. The example includes how to use your branch to build the UDF image to test your code change before submitting a PR. 
+This example is for numaflow-python contributors/developers. The example includes how to use your branch to build the UDF image to test your code change before submitting a PR.
 
-1. Install [Poetry](https://python-poetry.org/docs/) before starting your test. Make sure you have the correct python version.
+### Testing
+
+1. Install [Poetry](https://python-poetry.org/docs/) before starting your test. Make sure you have the correct Python version.
 2. Push your code change to your branch.
-3. Update the `pynumaflow` in `pyproject.tomal` file with your (forked) repo url and your branch name. For example, `pynumaflow = {git = "https://github.com/chromevoid/numaflow-python", rev = "test-branch"}`
+3. Update the `pynumaflow` dependency in the `pyproject.toml` file with your (forked) repo url and your branch name. For example, `pynumaflow = {git = "https://github.com/chromevoid/numaflow-python", rev = "test-branch"}`
 4. Run `poetry update -vv` from this `developer_guide` folder. You should get a `poetry.lock` file.
-5. Update your `example.py` and the `image name` in `Makefile` as needed.
+5. Update your `example.py` as/if needed.
 6. Run `make image` to build your image.
-7. Now you have the image with your customized example and your code change to test in the numaflow pipeline. Example pipeline `pipeline-numaflow.yaml` is also provided in this `developer_guide` folder. Please check [numaflow](https://numaflow.numaproj.io/) for more details.
+7. Now you have the image with your customized example and your code change to test in a numaflow pipeline. Example pipeline `pipeline-numaflow.yaml` is also provided in this `developer_guide` folder. Please check [numaflow](https://numaflow.numaproj.io/) for more details.
+
+Each example directory has a Makefile which can be used to build, tag, and push images.
+If you want to build the image and immediately push it to quay.io, use the `image-push` target.
+If you want to build a local image without pushing, use the `image` target.
+
+After making changes to the SDK, if you want to build all the example images at once, in the root directory you can run:
+```shell
+./hack/update_examples.sh -bp -t <tag>
+```
+The default tag is `stable`, but it is recommended you specify your own for testing purposes, as the Github Actions CI uses the `stable` tag. Note: do not forget to clean up testing tags
+in the registry, i.e. delete them, once you are done testing.
+
+You can alternatively build a specific example image by running the following in the root directory and providing the path to the Dockerfile:
+```shell
+./hack/update_examples.sh -bpe <path> -t <tag>
+```
+This is essentially equivalent to running `make image-push TAG=<tag>` in the example directory itself.
+
+### Deploying
+
+Once you have confirmed that your changes pass local testing:
+1. Revert the `pyproject.toml` file to its previous state, i.e. before you updated it with your forked repo and branch
+2. Create a PR for your changes
+
+Once the PR has been merged it is important that the pynumaflow dependency of the example images use the merged commit SHA
+as reference. Thus, before you delete/leave your branch, run:
+```shell
+./hack/update_examples.sh -u <commit-sha>
+./hack/update_examples.sh -bp
+```
+
+The above commands will update the pynumaflow dependency to the specified commit SHA, and build, tag, and push the image, respectively,
+across all example directories. Since we do not want to flood the commit history with dependency updates, it is not necessary
+to create a second PR with these changes.
+
+It is not necessary as due to the commands above, the images will be running with the latest commit SHA, i.e. their
+pynumaflow dependency in the `pyproject.toml` will be
+`pynumaflow = {git = "https://github.com/numaproj/numaflow-python.git", rev = "latest-sha"}` while the repo itself will show
+`pynumaflow = "~<latest-version>"`. As a result, the server information will always print the correct SDK version in the logs, and
+the example images will always be using the latest commit SHA.
+
+
+### After Release
+
+Once a new release has been made, and its corresponding version tag exists on the remote repo, we want to update the dependency
+management files to reflect this new version:
+
+```shell
+./hack/update_examples.sh -r <version>
+  ```
+
+This will update the `pyproject.toml` files in all the example directories as well as the one in the root directory,
+to depend on the specified version, i.e. the one just released. After running the above, create a PR for the changes
+that the script made.
+
+Once your changes have been merged, similar to the deployment steps above, before deleting/leaving your branch, update
+the example images to use the merged commit SHA:
+
+```shell
+./hack/update_examples.sh -u <commit-sha>
+./hack/update_examples.sh -bp
+```

examples/developer_guide/pipeline-numaflow.yaml

@@ -14,17 +14,17 @@ spec:
       udf:
         container:
           # split the incoming message by comma
-          image: quay.io/numaio/numaflow-python/map-flatmap:latest
+          image: quay.io/numaio/numaflow-python/map-flatmap:stable
     - name: cat
       udf:
         container:
           # simply forward the message
-          image: quay.io/numaio/numaflow-python/map-forward-message:latest
+          image: quay.io/numaio/numaflow-python/map-forward-message:stable
     - name: counter
       udf:
         container:
           # count the number of incoming messages
-          image: quay.io/numaio/numaflow-python/reduce-counter:latest
+          image: quay.io/numaio/numaflow-python/reduce-counter:stable
         groupBy:
           window:
             fixed:
@@ -36,7 +36,7 @@ spec:
       sink:
         udsink:
           container:
-            image: quay.io/numaio/numaflow-python/sink-log:latest
+            image: quay.io/numaio/numaflow-python/sink-log:stable
   edges:
     - from: in
       to: flatmap
@@ -46,4 +46,4 @@ spec:
       to: counter
       parallelism: 1
     - from: counter
-      to: sink
+      to: sink

examples/developer_guide/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = ">=3.9, <3.12"
-aiorun = "^2022.11.1"
+aiorun = "^2023.7"
 pynumaflow = {git = "https://github.com/<YOUR_ID>/numaflow-python", rev = "<YOUR_BRANCH>"}
 
 [tool.poetry.dev-dependencies]

examples/map/even_odd/Makefile

@@ -1,8 +1,11 @@
+TAG ?= stable
+PUSH ?= false
+
+.PHONY: image-push
+image-push:
+	docker buildx build -t "quay.io/numaio/numaflow-python/even-odd:${TAG}" --platform linux/amd64,linux/arm64 . --push
+
 .PHONY: image
 image:
-	docker build -t "quay.io/numaio/numaflow-python/even-odd:v0.5.0" .
-# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work
-# under the CI E2E test environment.
-# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command
-# docker buildx build -t "quay.io/numaio/numaflow-python/even-odd:latest" --platform linux/amd64,linux/arm64 . --push
-# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix
+	docker build -t "quay.io/numaio/numaflow-python/even-odd:${TAG}" .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/even-odd:${TAG}"; fi

examples/map/even_odd/pipeline-numaflow.yaml

@@ -17,7 +17,7 @@ spec:
         args:
         - python
         - example.py
-        image: "quay.io/numaio/numaflow-python/even-odd:v0.5.0"
+        image: "quay.io/numaio/numaflow-python/even-odd:stable"
   - name: even-sink
     scale:
       min: 1

examples/map/even_odd/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-pynumaflow = "~0.7.0"
+pynumaflow = "~0.6.0"
 
 [tool.poetry.dev-dependencies]
 

examples/map/flatmap/Makefile

@@ -1,8 +1,11 @@
+TAG ?= stable
+PUSH ?= false
+
+.PHONY: image-push
+image-push:
+	docker buildx build -t "quay.io/numaio/numaflow-python/map-flatmap:${TAG}" --platform linux/amd64,linux/arm64 . --push
+
 .PHONY: image
 image:
-	docker build -t "quay.io/numaio/numaflow-python/map-flatmap:v0.6.0" .
-# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work
-# under the CI E2E test environment.
-# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command
-# docker buildx build -t "quay.io/numaio/numaflow-python/map-flatmap:latest" --platform linux/amd64,linux/arm64 . --push
-# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix
+	docker build -t "quay.io/numaio/numaflow-python/map-flatmap:${TAG}" .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/map-flatmap:${TAG}"; fi

examples/map/flatmap/pipeline.yaml

@@ -15,7 +15,7 @@ spec:
     - name: flatmap
       udf:
         container:
-          image: "quay.io/numaio/numaflow-python/map-flatmap:v0.6.0"
+          image: "quay.io/numaio/numaflow-python/map-flatmap:stable"
           env:
             - name: PYTHONDEBUG
               value: "true"

examples/map/flatmap/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-pynumaflow = { git = "https://github.com/numaproj/numaflow-python.git", rev = "897ebc49ca3db21bd9eeb91b09c9607e910d6776" }
+pynumaflow = "~0.6.0"
 
 [tool.poetry.dev-dependencies]
 

examples/map/forward_message/Makefile

@@ -1,8 +1,11 @@
+TAG ?= stable
+PUSH ?= false
+
+.PHONY: image-push
+image-push:
+	docker buildx build -t "quay.io/numaio/numaflow-python/map-forward-message:${TAG}" --platform linux/amd64,linux/arm64 . --push
+
 .PHONY: image
 image:
-	docker build -t "quay.io/numaio/numaflow-python/map-forward-message:v0.7.0" .
-# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work
-# under the CI E2E test environment.
-# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command
-# docker buildx build -t "quay.io/numaio/numaflow-python/map-forward-message:latest" --platform linux/amd64,linux/arm64 . --push
-# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix
+	docker build -t "quay.io/numaio/numaflow-python/map-forward-message:${TAG}" .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/map-forward-message:${TAG}"; fi

examples/map/forward_message/pipeline-numaflow.yaml

@@ -17,7 +17,7 @@ spec:
         args:
         - python
         - example.py
-        image: docker.io/library/test-python-udf:v1
+        image: "quay.io/numaio/numaflow-python/map-forward-message:stable"
   - name: log-output
     sink:
       log: {}

examples/map/forward_message/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-pynumaflow = "~0.7.0"
+pynumaflow = "~0.6.0"
 
 [tool.poetry.dev-dependencies]