feat: Bidirectional Streaming for UDSink #6607
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "release-*" | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| ui: | |
| name: UI | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 6 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Setup Node-Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ui/node_modules | |
| key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} | |
| # Temporarily disable it. | |
| #- name: Yarn Build/Test | |
| # run: make ui-test | |
| - name: Ensure nothing changed | |
| run: git diff --exit-code | |
| codegen: | |
| name: Codegen | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Restore go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ hashFiles('**/go.mod') }} | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| - name: Add bins to PATH | |
| run: | | |
| echo /home/runner/go/bin >> $GITHUB_PATH | |
| echo /usr/local/bin >> $GITHUB_PATH | |
| - name: Get dependencies | |
| run: go mod download | |
| - name: Make codegen | |
| run: | | |
| echo 'GOPATH=/home/runner/go' >> $GITHUB_ENV | |
| make -B codegen | |
| - name: Ensure nothing changed | |
| run: git diff --exit-code | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| services: | |
| redis: | |
| image: redis:6.2 | |
| ports: | |
| - 6379:6379 | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| nats: | |
| image: 'bitnami/nats:latest' | |
| ports: | |
| - 4222:4222 | |
| env: | |
| NATS_EXTRA_ARGS: -js | |
| steps: | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| id: go | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Restore Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} | |
| - name: Get dependencies | |
| run: go mod download | |
| - name: Test Go | |
| run: make test-coverage-with-isb | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache-workspaces: rust -> target | |
| - name: Install llvm-tools-preview | |
| working-directory: ./rust | |
| run: rustup component add llvm-tools-preview | |
| - name: Install grcov | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: grcov | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| - name: Test Rust | |
| working-directory: ./rust | |
| run: | | |
| CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='./target/debug/coverage/cargo-test-%p-%m.profraw' cargo test --all-features --workspace --all | |
| grcov . -s ./target/debug/coverage/ --binary-path ./target/debug/ -t lcov --branch --ignore-not-existing -o ./target/debug/coverage/lcov.info | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: ./test/profile.cov,./rust/target/debug/coverage/lcov.info | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| env: | |
| GOPATH: /home/runner/go | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| - name: Restore Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} | |
| - run: make lint | |
| - run: git diff --exit-code | |
| build-rust-amd64: | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| working-directory: ./rust | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/[email protected] | |
| with: | |
| cache-workspaces: rust -> target | |
| rustflags: '' | |
| - name: Configure sccache | |
| run: | | |
| echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV | |
| echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Install dependencies | |
| run: sudo apt-get install -y protobuf-compiler | |
| - name: Print Protoc version | |
| run: protoc --version | |
| - name: Build binary | |
| run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu | |
| - name: Rename binary | |
| run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow ./numaflow-rs-linux-amd64 | |
| - name: List files | |
| run: pwd && ls -al && file ./numaflow-rs-linux-amd64 | |
| - name: Upload numaflow binary | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: numaflow-rs-linux-amd64 | |
| path: rust/numaflow-rs-linux-amd64 | |
| if-no-files-found: error | |
| e2e-tests: | |
| name: E2E Tests | |
| runs-on: ubuntu-latest | |
| needs: [ build-rust-amd64 ] | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| driver: [jetstream] | |
| case: [e2e, diamond-e2e, transformer-e2e, kafka-e2e, map-e2e, reduce-one-e2e, reduce-two-e2e, udsource-e2e, api-e2e, sideinputs-e2e, idle-source-e2e, monovertex-e2e, builtin-source-e2e] | |
| include: | |
| - driver: redis | |
| case: e2e | |
| - driver: redis | |
| case: kafka-e2e | |
| - driver: redis | |
| case: api-e2e | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Restore go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v2-${{ hashFiles('**/go.mod') }} | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| - name: Add bins to PATH | |
| run: | | |
| echo /home/runner/go/bin >> $GITHUB_PATH | |
| echo /usr/local/bin >> $GITHUB_PATH | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Setup Node-Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ui/node_modules | |
| key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} | |
| - name: Download Rust amd64 binaries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: numaflow-rs-linux-amd64 | |
| - name: Install k3d | |
| run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash & | |
| - name: Create a cluster | |
| run: | | |
| k3d cluster create e2e | |
| k3d kubeconfig get e2e > ~/.kube/numaflow-e2e-config | |
| - name: Install Numaflow | |
| env: | |
| GOPATH: /home/runner/go | |
| run: | | |
| KUBECONFIG=~/.kube/numaflow-e2e-config VERSION=${{ github.sha }} make start | |
| - name: Run tests | |
| env: | |
| GOPATH: /home/runner/go | |
| run: KUBECONFIG=~/.kube/numaflow-e2e-config VERSION=${{ github.sha }} ISBSVC=${{matrix.driver}} SKIP_IMAGE_BUILD=true make test-${{matrix.case}} |