diff --git a/.github/workflows/ci.yml b/.github/workflows/run-tests.yml similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/run-tests.yml diff --git a/.hack/update_examples.sh b/.hack/update_examples.sh new file mode 100755 index 00000000..22320410 --- /dev/null +++ b/.hack/update_examples.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +function show_help () { + echo "Usage: $0 [-h|--help | -t|--tag ] (-bp|--build-push | -bpe|--build-push-example | -us|--update-sha | -uv|--update-version )" + echo " -h, --help Display help message and exit" + echo " -bp, --build-push Build the Dockerfiles of all the examples and push them to the quay.io registry" + echo " -bpe, --build-push-example Build the Dockerfile of the given example directory path, and push it to the quay.io registry" + echo " -t, --tag To be optionally used with -bpe or -bp. Specify the tag to build with. Default tag: stable" + echo " -us, --update-sha Update all of the examples to depend on the specified commit SHA" + echo " -uv, --update-version Update all of the examples to depend on the specified version" +} +#update sha update version + +function traverse_examples () { + find examples -name "pyproject.toml" | while read -r line; + do + dir="$(dirname "${line}")" + cd "$dir" || exit + # TODO: rewrite asyncio-reduce example using latest SDK version, as it is currently using old methods + if [ "$dir" == "examples/developer_guide" ] || [ "$dir" == examples/reduce/asyncio-reduce ]; then + cd ~- || exit + continue + fi + + for command in "$@" + do + if ! $command; then + echo "Error: failed $command in $dir" >&2 + exit 1 + fi + done + + cd ~- || exit + done +} + +if [ $# -eq 0 ]; then + echo "Error: provide at least one argument" >&2 + show_help + exit 1 +fi + +usingHelp=0 +usingBuildPush=0 +usingBuildPushExample=0 +usingSHA=0 +usingVersion=0 +usingTag=0 +sha="" +version="" +directoryPath="" +tag="stable" + +function handle_options () { + while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usingHelp=1 + ;; + -bp | --build-push) + usingBuildPush=1 + ;; + -bpe | --build-push-example) + if [ -z "$2" ]; then + echo "Directory path not specified." >&2 + show_help + exit 1 + fi + + usingBuildPushExample=1 + directoryPath=$2 + shift + ;; + -t | --tag) + if [ -z "$2" ]; then + echo "Tag not specified." >&2 + show_help + exit 1 + fi + + usingTag=1 + tag=$2 + shift + ;; + -us | --update-sha) + if [ -z "$2" ]; then + echo "Commit SHA not specified." >&2 + show_help + exit 1 + fi + + usingSHA=1 + sha=$2 + shift + ;; + -uv | --update-version) + if [ -z "$2" ]; then + echo "Version not specified." >&2 + show_help + exit 1 + fi + + usingVersion=1 + version=$2 + shift + ;; + *) + echo "Invalid option: $1" >&2 + show_help + exit 1 + ;; + esac + shift + done +} + +handle_options "$@" + +if (( usingBuildPush + usingBuildPushExample + usingSHA + usingHelp + usingVersion > 1 )); then + echo "Only one of '-h', '-bp', '-bpe', '-us', or '-uv' is allowed at a time" >&2 + show_help + exit 1 +fi + +if (( (usingTag + usingSHA + usingHelp + usingVersion > 1) || (usingTag && usingBuildPush + usingBuildPushExample == 0) )); then + echo "Can only use -t with -bp or -bpe" >&2 + show_help + exit 1 +fi + +if [ -n "$sha" ]; then + echo "Using SHA: $sha" +fi + +if [ -n "$version" ]; then + echo "Using version: $version" +fi + +if [ -n "$directoryPath" ]; then + echo "Dockerfile path to use: $directoryPath" +fi + +if [ -n "$tag" ] && (( ! usingSHA )) && (( ! usingHelp )) && (( ! usingVersion )); then + echo "Using tag: $tag" +fi + +if (( usingBuildPush )); then + traverse_examples "make image-push TAG=$tag" +elif (( usingBuildPushExample )); then + cd "./$directoryPath" || exit + if ! make image-push TAG="$tag"; then + echo "Error: failed to run make image-push in $directoryPath" >&2 + exit 1 + fi +elif (( usingSHA )); then + traverse_examples "poetry add git+https://github.com/numaproj/numaflow-python.git@$sha" +elif (( usingVersion )); then + poetry version "$version" + traverse_examples "poetry add pynumaflow@~$version" +elif (( usingHelp )); then + show_help +fi diff --git a/examples/developer_guide/README.md b/examples/developer_guide/README.md index ee0d5596..6003d62f 100644 --- a/examples/developer_guide/README.md +++ b/examples/developer_guide/README.md @@ -1,11 +1,75 @@ # Developer Guide -This example is for numaflow-python contributors/developers. The example includes how to use your branch to build the UDF image to test your code change before submitting a PR. +This example is for numaflow-python contributors/developers. The example includes how to use your branch to build the UDF image to test your code change before submitting a PR. -1. Install [Poetry](https://python-poetry.org/docs/) before starting your test. Make sure you have the correct python version. +### Testing + +1. Install [Poetry](https://python-poetry.org/docs/) before starting your test. Make sure you have the correct Python version. 2. Push your code change to your branch. -3. Update the `pynumaflow` in `pyproject.tomal` file with your (forked) repo url and your branch name. For example, `pynumaflow = {git = "https://github.com/chromevoid/numaflow-python", rev = "test-branch"}` +3. Update the `pynumaflow` dependency in the `pyproject.toml` file with your (forked) repo url and your branch name. For example, `pynumaflow = {git = "https://github.com/chromevoid/numaflow-python", rev = "test-branch"}` 4. Run `poetry update -vv` from this `developer_guide` folder. You should get a `poetry.lock` file. -5. Update your `example.py` and the `image name` in `Makefile` as needed. +5. Update your `example.py` as/if needed. 6. Run `make image` to build your image. -7. Now you have the image with your customized example and your code change to test in the numaflow pipeline. Example pipeline `pipeline-numaflow.yaml` is also provided in this `developer_guide` folder. Please check [numaflow](https://numaflow.numaproj.io/) for more details. +7. Now you have the image with your customized example and your code change to test in a numaflow pipeline. Example pipeline `pipeline-numaflow.yaml` is also provided in this `developer_guide` folder. Please check [numaflow](https://numaflow.numaproj.io/) for more details. + +Each example directory has a Makefile which can be used to build, tag, and push images. +If you want to build the image and immediately push it to quay.io, use the `image-push` target. +If you want to build a local image without pushing, use the `image` target. + +After making changes to the SDK, if you want to build all the example images at once, in the root directory you can run: +```shell +./hack/update_examples.sh -bp -t +``` +The default tag is `stable`, but it is recommended you specify your own for testing purposes, as the Github Actions CI uses the `stable` tag. Note: do not forget to clean up testing tags +in the registry, i.e. delete them, once you are done testing. + +You can alternatively build a specific example image by running the following in the root directory and providing the path to the Dockerfile: +```shell +./hack/update_examples.sh -bpe -t +``` +This is essentially equivalent to running `make image-push TAG=` in the example directory itself. + +### Deploying + +Once you have confirmed that your changes pass local testing: +1. Revert the `pyproject.toml` file to its previous state, i.e. before you updated it with your forked repo and branch +2. Create a PR for your changes + +Once the PR has been merged it is important that the pynumaflow dependency of the example images use the merged commit SHA +as reference. Thus, before you delete/leave your branch, run: +```shell +./hack/update_examples.sh -u +./hack/update_examples.sh -bp +``` + +The above commands will update the pynumaflow dependency to the specified commit SHA, and build, tag, and push the image, respectively, +across all example directories. Since we do not want to flood the commit history with dependency updates, it is not necessary +to create a second PR with these changes. + +It is not necessary as due to the commands above, the images will be running with the latest commit SHA, i.e. their +pynumaflow dependency in the `pyproject.toml` will be +`pynumaflow = {git = "https://github.com/numaproj/numaflow-python.git", rev = "latest-sha"}` while the repo itself will show +`pynumaflow = "~"`. As a result, the server information will always print the correct SDK version in the logs, and +the example images will always be using the latest commit SHA. + + +### After Release + +Once a new release has been made, and its corresponding version tag exists on the remote repo, we want to update the dependency +management files to reflect this new version: + +```shell +./hack/update_examples.sh -r + ``` + +This will update the `pyproject.toml` files in all the example directories as well as the one in the root directory, +to depend on the specified version, i.e. the one just released. After running the above, create a PR for the changes +that the script made. + +Once your changes have been merged, similar to the deployment steps above, before deleting/leaving your branch, update +the example images to use the merged commit SHA: + +```shell +./hack/update_examples.sh -u +./hack/update_examples.sh -bp +``` diff --git a/examples/developer_guide/pipeline-numaflow.yaml b/examples/developer_guide/pipeline-numaflow.yaml index bc4dc062..27adf84a 100644 --- a/examples/developer_guide/pipeline-numaflow.yaml +++ b/examples/developer_guide/pipeline-numaflow.yaml @@ -14,17 +14,17 @@ spec: udf: container: # split the incoming message by comma - image: quay.io/numaio/numaflow-python/map-flatmap:latest + image: quay.io/numaio/numaflow-python/map-flatmap:stable - name: cat udf: container: # simply forward the message - image: quay.io/numaio/numaflow-python/map-forward-message:latest + image: quay.io/numaio/numaflow-python/map-forward-message:stable - name: counter udf: container: # count the number of incoming messages - image: quay.io/numaio/numaflow-python/reduce-counter:latest + image: quay.io/numaio/numaflow-python/reduce-counter:stable groupBy: window: fixed: @@ -36,7 +36,7 @@ spec: sink: udsink: container: - image: quay.io/numaio/numaflow-python/sink-log:latest + image: quay.io/numaio/numaflow-python/sink-log:stable edges: - from: in to: flatmap @@ -46,4 +46,4 @@ spec: to: counter parallelism: 1 - from: counter - to: sink \ No newline at end of file + to: sink diff --git a/examples/developer_guide/pyproject.toml b/examples/developer_guide/pyproject.toml index d8d73d32..a9c23be6 100644 --- a/examples/developer_guide/pyproject.toml +++ b/examples/developer_guide/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = ">=3.9, <3.12" -aiorun = "^2022.11.1" +aiorun = "^2023.7" pynumaflow = {git = "https://github.com//numaflow-python", rev = ""} [tool.poetry.dev-dependencies] diff --git a/examples/map/even_odd/Makefile b/examples/map/even_odd/Makefile index d6766612..9831b3eb 100644 --- a/examples/map/even_odd/Makefile +++ b/examples/map/even_odd/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/even-odd:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/even-odd:v0.5.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/even-odd:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/even-odd:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/even-odd:${TAG}"; fi diff --git a/examples/map/even_odd/pipeline-numaflow.yaml b/examples/map/even_odd/pipeline-numaflow.yaml index f03ca3d6..8d260450 100644 --- a/examples/map/even_odd/pipeline-numaflow.yaml +++ b/examples/map/even_odd/pipeline-numaflow.yaml @@ -17,7 +17,7 @@ spec: args: - python - example.py - image: "quay.io/numaio/numaflow-python/even-odd:v0.5.0" + image: "quay.io/numaio/numaflow-python/even-odd:stable" - name: even-sink scale: min: 1 diff --git a/examples/map/even_odd/pyproject.toml b/examples/map/even_odd/pyproject.toml index 63e388cf..d5df62f1 100644 --- a/examples/map/even_odd/pyproject.toml +++ b/examples/map/even_odd/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = "~0.7.0" +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/map/flatmap/Makefile b/examples/map/flatmap/Makefile index 302f5367..5146b70e 100644 --- a/examples/map/flatmap/Makefile +++ b/examples/map/flatmap/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/map-flatmap:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/map-flatmap:v0.6.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/map-flatmap:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/map-flatmap:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/map-flatmap:${TAG}"; fi diff --git a/examples/map/flatmap/pipeline.yaml b/examples/map/flatmap/pipeline.yaml index b34140fc..efe15754 100644 --- a/examples/map/flatmap/pipeline.yaml +++ b/examples/map/flatmap/pipeline.yaml @@ -15,7 +15,7 @@ spec: - name: flatmap udf: container: - image: "quay.io/numaio/numaflow-python/map-flatmap:v0.6.0" + image: "quay.io/numaio/numaflow-python/map-flatmap:stable" env: - name: PYTHONDEBUG value: "true" diff --git a/examples/map/flatmap/pyproject.toml b/examples/map/flatmap/pyproject.toml index eaf9aa64..d41d94cc 100644 --- a/examples/map/flatmap/pyproject.toml +++ b/examples/map/flatmap/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = { git = "https://github.com/numaproj/numaflow-python.git", rev = "897ebc49ca3db21bd9eeb91b09c9607e910d6776" } +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/map/forward_message/Makefile b/examples/map/forward_message/Makefile index 6bbab66a..b75bca61 100644 --- a/examples/map/forward_message/Makefile +++ b/examples/map/forward_message/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/map-forward-message:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/map-forward-message:v0.7.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/map-forward-message:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/map-forward-message:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/map-forward-message:${TAG}"; fi diff --git a/examples/map/forward_message/pipeline-numaflow.yaml b/examples/map/forward_message/pipeline-numaflow.yaml index 98e0b02f..8ba88980 100644 --- a/examples/map/forward_message/pipeline-numaflow.yaml +++ b/examples/map/forward_message/pipeline-numaflow.yaml @@ -17,7 +17,7 @@ spec: args: - python - example.py - image: docker.io/library/test-python-udf:v1 + image: "quay.io/numaio/numaflow-python/map-forward-message:stable" - name: log-output sink: log: {} diff --git a/examples/map/forward_message/pyproject.toml b/examples/map/forward_message/pyproject.toml index 441e8dd4..361ba9e5 100644 --- a/examples/map/forward_message/pyproject.toml +++ b/examples/map/forward_message/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = "~0.7.0" +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/map/multiproc_map/Makefile b/examples/map/multiproc_map/Makefile index e7679224..9c6647e3 100644 --- a/examples/map/multiproc_map/Makefile +++ b/examples/map/multiproc_map/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/multiproc:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/multiproc:v0.7.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/multiproc:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix \ No newline at end of file + docker build -t "quay.io/numaio/numaflow-python/multiproc:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/multiproc:${TAG}"; fi diff --git a/examples/map/multiproc_map/pipeline.yaml b/examples/map/multiproc_map/pipeline.yaml index 54fe2b6a..78fbadf0 100644 --- a/examples/map/multiproc_map/pipeline.yaml +++ b/examples/map/multiproc_map/pipeline.yaml @@ -15,7 +15,7 @@ spec: - name: mult udf: container: - image: "quay.io/numaio/numaflow-python/multiproc:v0.7.0" + image: "quay.io/numaio/numaflow-python/multiproc:stable" env: - name: PYTHONDEBUG value: "true" diff --git a/examples/map/multiproc_map/pyproject.toml b/examples/map/multiproc_map/pyproject.toml index 441e8dd4..361ba9e5 100644 --- a/examples/map/multiproc_map/pyproject.toml +++ b/examples/map/multiproc_map/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = "~0.7.0" +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/mapstream/flatmap_stream/Makefile b/examples/mapstream/flatmap_stream/Makefile index 9c756cd2..8eb08469 100644 --- a/examples/mapstream/flatmap_stream/Makefile +++ b/examples/mapstream/flatmap_stream/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/map-flatmap-stream:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/map-flatmap-stream:v0.6.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/map-flatmap-stream:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/map-flatmap-stream:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/map-flatmap-stream:${TAG}"; fi diff --git a/examples/mapstream/flatmap_stream/pipeline.yaml b/examples/mapstream/flatmap_stream/pipeline.yaml index 786e3f31..08f0b677 100644 --- a/examples/mapstream/flatmap_stream/pipeline.yaml +++ b/examples/mapstream/flatmap_stream/pipeline.yaml @@ -20,7 +20,7 @@ spec: readBatchSize: 1 udf: container: - image: "quay.io/numaio/numaflow-python/map-flatmap-stream:v0.6.0" + image: "quay.io/numaio/numaflow-python/map-flatmap-stream:stable" imagePullPolicy: Always env: - name: PYTHONDEBUG diff --git a/examples/mapstream/flatmap_stream/pyproject.toml b/examples/mapstream/flatmap_stream/pyproject.toml index bf4684df..7df9056e 100644 --- a/examples/mapstream/flatmap_stream/pyproject.toml +++ b/examples/mapstream/flatmap_stream/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = { git = "https://github.com/numaproj/numaflow-python.git", rev = "897ebc49ca3db21bd9eeb91b09c9607e910d6776" } +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/reduce/asyncio-reduce/Makefile b/examples/reduce/asyncio-reduce/Makefile index f43168ec..1fd0551f 100644 --- a/examples/reduce/asyncio-reduce/Makefile +++ b/examples/reduce/asyncio-reduce/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/asynctest:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/asynctest:v0.5.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/asynctest:v1" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix \ No newline at end of file + docker build -t "quay.io/numaio/numaflow-python/asynctest:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/asynctest:${TAG}"; fi diff --git a/examples/reduce/counter/Makefile b/examples/reduce/counter/Makefile index 363debad..639d6b78 100644 --- a/examples/reduce/counter/Makefile +++ b/examples/reduce/counter/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/reduce-counter:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/reduce-counter:v0.7.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/reduce-counter:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/reduce-counter:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/reduce-counter:${TAG}"; fi diff --git a/examples/reduce/counter/pipeline.yaml b/examples/reduce/counter/pipeline.yaml index 1b7d6b34..110e88ca 100644 --- a/examples/reduce/counter/pipeline.yaml +++ b/examples/reduce/counter/pipeline.yaml @@ -13,12 +13,12 @@ spec: udf: container: # Tell the input number is even or odd, see https://github.com/numaproj/numaflow-go/tree/main/pkg/mapper/examples/even_odd - image: quay.io/numaio/numaflow-go/map-even-odd:v0.5.0 + image: quay.io/numaio/numaflow-go/map-even-odd:stable - name: compute-sum udf: container: # compute the sum - image: quay.io/numaio/numaflow-python/reduce-counter:latest + image: quay.io/numaio/numaflow-python/reduce-counter:stable imagePullPolicy: Always env: - name: PYTHONDEBUG diff --git a/examples/reduce/counter/pyproject.toml b/examples/reduce/counter/pyproject.toml index dc3cc41d..5ce249e0 100644 --- a/examples/reduce/counter/pyproject.toml +++ b/examples/reduce/counter/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = "~0.7.0" +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/sideinput/simple-sideinput/Makefile b/examples/sideinput/simple-sideinput/Makefile index cd2f0add..795a5039 100644 --- a/examples/sideinput/simple-sideinput/Makefile +++ b/examples/sideinput/simple-sideinput/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/sideinput-example:v0.7.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/sideinput-example:v0.5.0" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/sideinput-example:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/sideinput-example:${TAG}"; fi diff --git a/examples/sideinput/simple-sideinput/pipeline-numaflow.yaml b/examples/sideinput/simple-sideinput/pipeline-numaflow.yaml index 4e07d5f1..bad095ef 100644 --- a/examples/sideinput/simple-sideinput/pipeline-numaflow.yaml +++ b/examples/sideinput/simple-sideinput/pipeline-numaflow.yaml @@ -6,7 +6,7 @@ spec: sideInputs: - name: myticker container: - image: "quay.io/numaio/numaflow-python/sideinput-example:v0.7.0" + image: "quay.io/numaio/numaflow-python/sideinput-example:stable" imagePullPolicy: Always trigger: schedule: "*/2 * * * *" @@ -21,7 +21,7 @@ spec: - name: si-log udf: container: - image: "quay.io/numaio/numaflow-python/udf-sideinput-example:v0.5.1" + image: "quay.io/numaio/numaflow-python/udf-sideinput-example:stable" imagePullPolicy: Always containerTemplate: env: @@ -37,4 +37,4 @@ spec: - from: in to: si-log - from: si-log - to: out \ No newline at end of file + to: out diff --git a/examples/sideinput/simple-sideinput/pyproject.toml b/examples/sideinput/simple-sideinput/pyproject.toml index 441e8dd4..361ba9e5 100644 --- a/examples/sideinput/simple-sideinput/pyproject.toml +++ b/examples/sideinput/simple-sideinput/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = "~0.7.0" +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/sideinput/simple-sideinput/udf/Makefile b/examples/sideinput/simple-sideinput/udf/Makefile index 73b82995..eaaf172d 100644 --- a/examples/sideinput/simple-sideinput/udf/Makefile +++ b/examples/sideinput/simple-sideinput/udf/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/udf-sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker buildx build -t "quay.io/numaio/numaflow-python/udf-sideinput-example:v0.5.1" --platform linux/amd64,linux/arm64 . --push -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/udf-sideinput-example:v0.5.1" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix \ No newline at end of file + docker build -t "quay.io/numaio/numaflow-python/udf-sideinput-example:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/udf-sideinput-example:${TAG}"; fi diff --git a/examples/sink/async_log/Makefile b/examples/sink/async_log/Makefile index 49ed516f..93fc7fcd 100644 --- a/examples/sink/async_log/Makefile +++ b/examples/sink/async_log/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/async-sink-log:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/async-sink-log:v0.7.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/async-sink-log:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/async-sink-log:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/async-sink-log:${TAG}"; fi diff --git a/examples/sink/async_log/pipeline-numaflow.yaml b/examples/sink/async_log/pipeline-numaflow.yaml index b2e23ac8..e6799bff 100644 --- a/examples/sink/async_log/pipeline-numaflow.yaml +++ b/examples/sink/async_log/pipeline-numaflow.yaml @@ -21,7 +21,7 @@ spec: args: - python - example.py - image: quay.io/numaio/numaflow-python/async-sink-log:v0.7.0 + image: quay.io/numaio/numaflow-python/async-sink-log:stable imagePullPolicy: Always env: - name: PYTHONDEBUG diff --git a/examples/sink/async_log/pyproject.toml b/examples/sink/async_log/pyproject.toml index 583a6388..629d9c26 100644 --- a/examples/sink/async_log/pyproject.toml +++ b/examples/sink/async_log/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = "~0.7.0" +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/sink/log/Makefile b/examples/sink/log/Makefile index baeeb1bb..0fbd100b 100644 --- a/examples/sink/log/Makefile +++ b/examples/sink/log/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/sink-log:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/sink-log:v0.6.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/sink-log:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/sink-log:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/sink-log:${TAG}"; fi diff --git a/examples/sink/log/pipeline-numaflow.yaml b/examples/sink/log/pipeline-numaflow.yaml index 984b5f3d..68bb8d1c 100644 --- a/examples/sink/log/pipeline-numaflow.yaml +++ b/examples/sink/log/pipeline-numaflow.yaml @@ -21,7 +21,7 @@ spec: args: - python - example.py - image: "quay.io/numaio/numaflow-python/sink-log:v0.6.0" + image: "quay.io/numaio/numaflow-python/sink-log:stable" imagePullPolicy: Always env: - name: PYTHONDEBUG diff --git a/examples/sink/log/pyproject.toml b/examples/sink/log/pyproject.toml index ab8b97f1..629d9c26 100644 --- a/examples/sink/log/pyproject.toml +++ b/examples/sink/log/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = { git = "https://github.com/numaproj/numaflow-python.git", rev = "897ebc49ca3db21bd9eeb91b09c9607e910d6776" } +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/source/async-source/Makefile b/examples/source/async-source/Makefile index ddcc242e..ed7ed485 100644 --- a/examples/source/async-source/Makefile +++ b/examples/source/async-source/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/async-source:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/async-source:v0.7.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/async-source:v0.5.5" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/async-source:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/async-source:${TAG}"; fi diff --git a/examples/source/async-source/pipeline-numaflow.yaml b/examples/source/async-source/pipeline-numaflow.yaml index da8d482d..8ddaf55d 100644 --- a/examples/source/async-source/pipeline-numaflow.yaml +++ b/examples/source/async-source/pipeline-numaflow.yaml @@ -9,7 +9,7 @@ spec: udsource: container: # A simple user-defined async source - image: "quay.io/numaio/numaflow-python/async-source:v0.7.0" + image: "quay.io/numaio/numaflow-python/async-source:stable" imagePullPolicy: Always limits: readBatchSize: 2 @@ -18,4 +18,4 @@ spec: log: {} edges: - from: in - to: out \ No newline at end of file + to: out diff --git a/examples/source/async-source/pyproject.toml b/examples/source/async-source/pyproject.toml index 60fba875..82428bc2 100644 --- a/examples/source/async-source/pyproject.toml +++ b/examples/source/async-source/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = "~0.7.0" +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/source/simple-source/Makefile b/examples/source/simple-source/Makefile index 8d351678..75639a3f 100644 --- a/examples/source/simple-source/Makefile +++ b/examples/source/simple-source/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/simple-source:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/simple-source:v0.6.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/simple-source:v0.5.5" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/simple-source:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/simple-source:${TAG}"; fi diff --git a/examples/source/simple-source/pipeline-numaflow.yaml b/examples/source/simple-source/pipeline-numaflow.yaml index 98c692b5..05b9c557 100644 --- a/examples/source/simple-source/pipeline-numaflow.yaml +++ b/examples/source/simple-source/pipeline-numaflow.yaml @@ -9,7 +9,7 @@ spec: udsource: container: # A simple user-defined source for e2e testing - image: quay.io/numaio/numaflow-python/simple-source:v0.6.0 + image: quay.io/numaio/numaflow-python/simple-source:stable imagePullPolicy: Always limits: readBatchSize: 2 diff --git a/examples/source/simple-source/pyproject.toml b/examples/source/simple-source/pyproject.toml index 34d4915d..262232e9 100644 --- a/examples/source/simple-source/pyproject.toml +++ b/examples/source/simple-source/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Numaflow developers"] [tool.poetry.dependencies] python = "~3.10" -pynumaflow = { git = "https://github.com/numaproj/numaflow-python.git", rev = "897ebc49ca3db21bd9eeb91b09c9607e910d6776" } +pynumaflow = "~0.6.0" [tool.poetry.dev-dependencies] diff --git a/examples/sourcetransform/event_time_filter/Makefile b/examples/sourcetransform/event_time_filter/Makefile index 6f763ecb..33e261d1 100644 --- a/examples/sourcetransform/event_time_filter/Makefile +++ b/examples/sourcetransform/event_time_filter/Makefile @@ -1,8 +1,11 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: image-push +image-push: + docker buildx build -t "quay.io/numaio/numaflow-python/mapt-event-time-filter:${TAG}" --platform linux/amd64,linux/arm64 . --push + .PHONY: image image: - docker build -t "quay.io/numaio/numaflow-python/mapt-event-time-filter:v0.6.0" . -# Github CI runner uses platform linux/amd64. If your local environment don't, the image built by command above might not work -# under the CI E2E test environment. -# To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -# docker buildx build -t "quay.io/numaio/numaflow-python/mapt-event-time-filter:latest" --platform linux/amd64,linux/arm64 . --push -# If command failed, refer to https://billglover.me/notes/build-multi-arch-docker-images/ to fix + docker build -t "quay.io/numaio/numaflow-python/mapt-event-time-filter:${TAG}" . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/mapt-event-time-filter:${TAG}"; fi diff --git a/examples/sourcetransform/event_time_filter/pyproject.toml b/examples/sourcetransform/event_time_filter/pyproject.toml index d223efd3..a6d19b57 100644 --- a/examples/sourcetransform/event_time_filter/pyproject.toml +++ b/examples/sourcetransform/event_time_filter/pyproject.toml @@ -8,7 +8,7 @@ packages = [{include = "mapt_event_time_filter"}] [tool.poetry.dependencies] python = ">=3.9, <3.12" -pynumaflow = { git = "https://github.com/numaproj/numaflow-python.git", rev = "897ebc49ca3db21bd9eeb91b09c9607e910d6776" } +pynumaflow = "~0.6.0" [build-system] requires = ["poetry-core"]