Merge branch 'main' into map-batch

.github/workflows/build-push.yaml

@@ -8,15 +8,14 @@ on:
 
 jobs:
   docker_publish:
-    # run it only on numaproj/numaflow-python repository
-    # forked repositories normally don't have the proper permission setup.
+    # run workflow only on numaproj/numaflow-python repository
     if: ${{ github.repository }} == "numaproj/numaflow-python"
     name: Build, Tag, and Push Image
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        # TODO: rewrite asyncio-reduce example using latest SDK version, as it is currently using old methods
+        # TODO: rewrite asyncio_reduce example using latest SDK version, as it is currently using old methods
         # then add to example_directories matrix
         example_directories: [
           "examples/map/even_odd", "examples/map/flatmap", "examples/map/forward_message",
@@ -49,5 +48,4 @@ jobs:
           password: ${{ secrets.NUMAIO_PASSWORD }}
       - name: Build, tag, and push images
         run: |
-          ./.hack/update_examples.sh --update ${{ matrix.example_directories }}
-          ./.hack/update_examples.sh --build-push-example ${{ matrix.example_directories }}
+          ./hack/update_examples.sh --build-push-example ${{ matrix.example_directories }}

.github/workflows/changelog.yml

@@ -16,7 +16,7 @@ jobs:
           fetch-depth: 0
       - run: git fetch --prune --prune-tags
       - run: git tag -l 'v*'
-      - run: ./.hack/changelog.sh > CHANGELOG.md
+      - run: ./hack/changelog.sh > CHANGELOG.md
       - uses: peter-evans/create-pull-request@v3
         with:
           title: 'docs: updated CHANGELOG.md'

Makefile

@@ -1,5 +1,5 @@
 clean:
-	@rm -rf build dist .eggs *.egg-info
+	@rm -rf build .eggs *.egg-info
 	@rm -rf .benchmarks .coverage coverage.xml htmlcov report.xml .tox
 	@find . -type d -name '.mypy_cache' -exec rm -rf {} +
 	@find . -type d -name '__pycache__' -exec rm -rf {} +

development.md

@@ -6,28 +6,18 @@ This document explains the development process for the Numaflow Python SDK.
 
 1. Install [Poetry](https://python-poetry.org/docs/) before starting your test. Make sure you have the correct Python version
 2. Make your SDK changes
-3. Run `./.hack/update_examples.sh -u <example-directory-path>`. This command will create a tarball of the SDK, that includes your changes, in the directory of the example specified
-4. Run `poetry update -vv` inside the example directory you used in the step above. This will create a `poetry.lock` file
-5. Update the `example.py` inside the same example directory if needed 
-6. Run `make image` inside the same example directory to build your image
-7. Now that you have the image with your customized example and code change, you can test it in a Numaflow pipeline. Example pipelines, `pipeline.yaml`, are also provided in most of the example directories.
+3. Update the `example.py` inside the same example directory if needed 
+4. Run `make image` inside the same example directory to build your image
+5. Now that you have the image with your customized example and code change, you can test it in a Numaflow pipeline. Example pipelines, `pipeline.yaml`, are also provided in most of the example directories.
 Please check [Numaflow](https://numaflow.numaproj.io/) for more details
 
 Each example directory has a Makefile which can be used to build, tag, and push images.
 If you want to build and tag the image and then immediately push it to quay.io, use the `image-push` target.
 If you want to build and tag a local image without pushing, use the `image` target.
 
-Everytime you make a new SDK change, make sure to create the `dist/` folder in the example directory you would like
-to test, by running:
+If you would like to build and push a specific example, you can do so by running:
 ```shell
-./.hack/update_examples.sh -u <example-directory-path>
-```
-When you are no longer using the `dist/` folders you can run `make clean` in the root directory, as well as in any example directory
-in order to remove them.
-
-If you would like to build and push a specific example, you can do so by running (make sure that the `dist/` folder is present in the directory):
-```shell
-./.hack/update_examples.sh -bpe <example-directory-path> -t <tag>
+./hack/update_examples.sh -bpe <example-directory-path> -t <tag>
 ```
 This is essentially equivalent to running `make image-push TAG=<tag>` in the example directory itself.
 The default tag is `stable`, but it is recommended you specify your own for testing purposes, as the Github Actions CI uses the `stable` tag.
@@ -38,7 +28,7 @@ Note: before running the script, ensure that through the CLI, you are logged int
 ### Deploying
 
 After confirming that your changes pass local testing:
-1. Clean up testing artifacts (remove any `dist/` folders created, remove any test images on quay.io, etc.)
+1. Clean up testing artifacts (remove any test images on quay.io, etc.)
 2. Create a PR for your changes. Once your PR has been merged, a Github Actions workflow (`Docker Publish`) will be triggered, to build, tag (with `stable`), and push
 all example images. This ensures that all example images are using the most up-to-date version of the SDK, i.e. the one including your
 changes

examples/map/even_odd/Dockerfile

@@ -39,12 +39,13 @@ RUN apt-get update \
 FROM builder AS udf
 
 WORKDIR $PYSETUP_PATH
-COPY dist/ ./dist/
-COPY pyproject.toml .
+COPY ./ ./
+
+WORKDIR $PYSETUP_PATH/examples/map/even_odd
 RUN poetry install --no-cache --no-root && \
     rm -rf ~/.cache/pypoetry/
 
-ADD . /app
+COPY examples/map/even_odd /app
 WORKDIR /app
 
 RUN chmod +x entry.sh

examples/map/even_odd/Makefile

@@ -1,15 +1,22 @@
 TAG ?= stable
 PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-python/even-odd:${TAG}
+DOCKER_FILE_PATH = examples/map/even_odd/Dockerfile
+
+.PHONY: update
+update:
+	poetry update -vv
 
 .PHONY: image-push
-image-push:
-	docker buildx build -t "quay.io/numaio/numaflow-python/even-odd:${TAG}" --platform linux/amd64,linux/arm64 . --push
+image-push: update
+	cd ../../../ && docker buildx build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} \
+	--platform linux/amd64,linux/arm64 . --push
 
 .PHONY: image
-image:
-	docker build -t "quay.io/numaio/numaflow-python/even-odd:${TAG}" .
-	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/even-odd:${TAG}"; fi
-
-.PHONY: clean
-clean:
-	rm -rf ./dist
+image: update
+	cd ../../../ && docker build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi

examples/map/even_odd/pipeline.yaml

@@ -17,7 +17,7 @@ spec:
         args:
         - python
         - example.py
-        image: "quay.io/numaio/numaflow-python/even-odd:stable"
+        image: quay.io/numaio/numaflow-python/even-odd:stable
   - name: even-sink
     scale:
       min: 1

examples/map/even_odd/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-pynumaflow = { path = "./dist/pynumaflow.tar.gz"}
+pynumaflow = { path = "../../../"}
 
 [tool.poetry.dev-dependencies]
 

examples/map/flatmap/Dockerfile

@@ -39,12 +39,13 @@ RUN apt-get update \
 FROM builder AS udf
 
 WORKDIR $PYSETUP_PATH
-COPY dist/ ./dist/
-COPY pyproject.toml .
+COPY ./ ./
+
+WORKDIR $PYSETUP_PATH/examples/map/flatmap
 RUN poetry install --no-cache --no-root && \
     rm -rf ~/.cache/pypoetry/
 
-ADD . /app
+COPY examples/map/even_odd /app
 WORKDIR /app
 
 RUN chmod +x entry.sh

examples/map/flatmap/Makefile

@@ -1,15 +1,22 @@
 TAG ?= stable
 PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-python/map-flatmap:${TAG}
+DOCKER_FILE_PATH = examples/map/flatmap/Dockerfile
+
+.PHONY: update
+update:
+	poetry update -vv
 
 .PHONY: image-push
-image-push:
-	docker buildx build -t "quay.io/numaio/numaflow-python/map-flatmap:${TAG}" --platform linux/amd64,linux/arm64 . --push
+image-push: update
+	cd ../../../ && docker buildx build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} \
+	--platform linux/amd64,linux/arm64 . --push
 
 .PHONY: image
-image:
-	docker build -t "quay.io/numaio/numaflow-python/map-flatmap:${TAG}" .
-	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/map-flatmap:${TAG}"; fi
-
-.PHONY: clean
-clean:
-	rm -rf ./dist
+image: update
+	cd ../../../ && docker build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi

examples/map/flatmap/pipeline.yaml

@@ -15,7 +15,7 @@ spec:
     - name: flatmap
       udf:
         container:
-          image: "quay.io/numaio/numaflow-python/map-flatmap:stable"
+          image: quay.io/numaio/numaflow-python/map-flatmap:stable
           env:
             - name: PYTHONDEBUG
               value: "true"

examples/map/flatmap/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-pynumaflow = { path = "./dist/pynumaflow.tar.gz"}
+pynumaflow = { path = "../../../"}
 
 [tool.poetry.dev-dependencies]
 

examples/map/forward_message/Dockerfile

@@ -39,12 +39,13 @@ RUN apt-get update \
 FROM builder AS udf
 
 WORKDIR $PYSETUP_PATH
-COPY dist/ ./dist/
-COPY pyproject.toml .
+COPY ./ ./
+
+WORKDIR $PYSETUP_PATH/examples/map/forward_message
 RUN poetry install --no-cache --no-root && \
     rm -rf ~/.cache/pypoetry/
 
-ADD . /app
+COPY examples/map/forward_message /app
 WORKDIR /app
 
 RUN chmod +x entry.sh

examples/map/forward_message/Makefile

@@ -1,15 +1,22 @@
 TAG ?= stable
 PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-python/map-forward-message:${TAG}
+DOCKER_FILE_PATH = examples/map/forward_message/Dockerfile
+
+.PHONY: update
+update:
+	poetry update -vv
 
 .PHONY: image-push
-image-push:
-	docker buildx build -t "quay.io/numaio/numaflow-python/map-forward-message:${TAG}" --platform linux/amd64,linux/arm64 . --push
+image-push: update
+	cd ../../../ && docker buildx build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} \
+	--platform linux/amd64,linux/arm64 . --push
 
 .PHONY: image
-image:
-	docker build -t "quay.io/numaio/numaflow-python/map-forward-message:${TAG}" .
-	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/map-forward-message:${TAG}"; fi
-
-.PHONY: clean
-clean:
-	rm -rf ./dist
+image: update
+	cd ../../../ && docker build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi

examples/map/forward_message/pipeline.yaml

@@ -17,7 +17,7 @@ spec:
         args:
         - python
         - example.py
-        image: "quay.io/numaio/numaflow-python/map-forward-message:stable"
+        image: quay.io/numaio/numaflow-python/map-forward-message:stable
   - name: log-output
     sink:
       log: {}

examples/map/forward_message/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-pynumaflow = { path = "./dist/pynumaflow.tar.gz"}
+pynumaflow = { path = "../../../"}
 
 [tool.poetry.dev-dependencies]
 

examples/map/multiproc_map/Dockerfile

@@ -39,12 +39,13 @@ RUN apt-get update \
 FROM builder AS udf
 
 WORKDIR $PYSETUP_PATH
-COPY dist/ ./dist/
-COPY pyproject.toml .
+COPY ./ ./
+
+WORKDIR $PYSETUP_PATH/examples/map/multiproc_map
 RUN poetry install --no-cache --no-root && \
     rm -rf ~/.cache/pypoetry/
 
-ADD . /app
+COPY examples/map/multiproc_map /app
 WORKDIR /app
 
 RUN chmod +x entry.sh

examples/map/multiproc_map/Makefile

@@ -1,15 +1,22 @@
 TAG ?= stable
 PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-python/multiproc:${TAG}
+DOCKER_FILE_PATH = examples/map/multiproc_map/Dockerfile
+
+.PHONY: update
+update:
+	poetry update -vv
 
 .PHONY: image-push
-image-push:
-	docker buildx build -t "quay.io/numaio/numaflow-python/multiproc:${TAG}" --platform linux/amd64,linux/arm64 . --push
+image-push: update
+	cd ../../../ && docker buildx build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} \
+	 --platform linux/amd64,linux/arm64 . --push
 
 .PHONY: image
-image:
-	docker build -t "quay.io/numaio/numaflow-python/multiproc:${TAG}" .
-	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-python/multiproc:${TAG}"; fi
-
-.PHONY: clean
-clean:
-	rm -rf ./dist
+image: update
+	cd ../../../ && docker build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi

examples/map/multiproc_map/pipeline.yaml

@@ -15,7 +15,7 @@ spec:
     - name: mult
       udf:
         container:
-          image: "quay.io/numaio/numaflow-python/multiproc:stable"
+          image: quay.io/numaio/numaflow-python/multiproc:stable
           env:
             - name: PYTHONDEBUG
               value: "true"

examples/map/multiproc_map/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Numaflow developers"]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-pynumaflow = { path = "./dist/pynumaflow.tar.gz"}
+pynumaflow = { path = "../../../"}
 
 [tool.poetry.dev-dependencies]
 

examples/mapstream/flatmap_stream/Dockerfile

@@ -39,12 +39,13 @@ RUN apt-get update \
 FROM builder AS udf
 
 WORKDIR $PYSETUP_PATH
-COPY dist/ ./dist/
-COPY pyproject.toml .
+COPY ./ ./
+
+WORKDIR $PYSETUP_PATH/examples/mapstream/flatmap_stream
 RUN poetry install --no-cache --no-root && \
     rm -rf ~/.cache/pypoetry/
 
-ADD . /app
+COPY examples/mapstream/flatmap_stream /app
 WORKDIR /app
 
 RUN chmod +x entry.sh