Scenario load tests with simulator

.github/workflows/pr-tests-stack.yml

@@ -53,7 +53,7 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Run syft backend base image building test
@@ -96,7 +96,7 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Get uv cache dir
@@ -179,7 +179,7 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Get uv cache dir
@@ -328,7 +328,7 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Get uv cache dir
@@ -471,7 +471,7 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Get uv cache dir
@@ -597,7 +597,7 @@ jobs:
         if: steps.changes.outputs.syft == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Get uv cache dir
@@ -657,7 +657,7 @@ jobs:
         if: steps.changes.outputs.syft == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Get uv cache dir
@@ -734,7 +734,7 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
 
       - name: Get uv cache dir
@@ -875,7 +875,7 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install uv==0.2.17 tox==4.16.0 tox-uv==1.9.0
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
           uv --version
       - name: Get uv cache dir
         if: steps.changes.outputs.stack == 'true'
@@ -956,3 +956,130 @@ jobs:
           export PATH=`pwd`:$PATH
           k3d cluster delete bigquery-high || true
           k3d cluster delete bigquery-low || true
+
+  pr-tests-simulation-scenario-k8s:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      - name: Permission to home directory
+        run: |
+          sudo chown -R $USER:$USER $HOME
+      - uses: actions/checkout@v4
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Add K3d Registry
+        run: |
+          sudo python ./scripts/patch_hosts.py --add-k3d-registry
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          large-packages: false
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
+      - name: Install pip dependencies
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv==0.4.16 tox==4.20.0 tox-uv==1.13.0
+          uv --version
+
+      - name: Get uv cache dir
+        if: steps.changes.outputs.stack == 'true'
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Load github cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
+          restore-keys: |
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}
+
+      - name: Install kubectl
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # cleanup apt version
+          sudo apt remove kubectl || true
+          # install kubectl 1.27
+          curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          sudo install kubectl /usr/local/bin;
+
+      - name: Install helm
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # install helm
+          curl  -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+          chmod 700 get_helm.sh
+          ./get_helm.sh
+
+      - name: Install just
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
+
+      - name: Run scenario tests
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        env:
+          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
+          BUMP_VERSION: "${{ matrix.bump-version }}"
+          TOX_PYTHON: python${{ matrix.python-version }}
+        shell: bash
+        run: |
+          K3D_VERSION=v5.6.3
+          DEVSPACE_VERSION=v6.3.12
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+          tox -e stack.test.scenario.k8s
+
+      # todo: collect logs
+
+      - name: Cleanup k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          export PATH=`pwd`:$PATH
+          k3d cluster delete syft-high || true
+          k3d cluster delete syft-low || true
+          k3d registry delete k3d-registry.localhost || true

.github/workflows/pr-tests-syft.yml

@@ -188,6 +188,91 @@ jobs:
           max_attempts: 3
           command: tox -e syft.test.notebook
 
+  pr-tests-syft-scenario:
+    if: ${{ false }}
+    strategy:
+      max-parallel: 99
+      matrix:
+        # TODO try enabling on other OS
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+        deployment-type: ["python"]
+        bump-version: ["False"]
+        include:
+          - python-version: "3.11"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+          - python-version: "3.10"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+          - python-version: "3.12"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+            bump-version: "True"
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      # - name: Permission to home directory
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: |
+      #     sudo chown -R $USER:$USER $HOME
+      - name: "clean .git/config"
+        if: matrix.os == 'windows-latest'
+        continue-on-error: true
+        shell: bash
+        run: |
+          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
+          rm ${GITHUB_WORKSPACE}/.git/config
+
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pip packages
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          uv --version
+
+      - name: Get uv cache dir
+        id: pip-cache
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Load github cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}-
+
+      - name: Run scenario tests
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        env:
+          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
+          BUMP_VERSION: "${{ matrix.bump-version }}"
+          TOX_PYTHON: python${{ matrix.python-version }}
+        shell: bash
+        run: |
+          tox -e syft.test.scenario
+
   pr-tests-syft-notebook-scenario:
     strategy:
       max-parallel: 99

.gitignore

@@ -92,5 +92,8 @@ notebooks/scenarios/bigquery/sync/*.json
 notebooks/scenarios/bigquery/sync/*.json.lock
 notebooks/tutorials/version-upgrades/*.yaml
 notebooks/tutorials/version-upgrades/*.blob
-
 notebooks/scenarios/bigquery/sync/emails.json
+
+# logs dir generated by sim tests
+.logs
+

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
         always_run: true
       - id: name-tests-test
         always_run: true
-        exclude: ^(.*/tests/utils/)|^(.*fixtures.py)|^(tests/scenarios/bigquery/helpers)
+        exclude: ^(.*/tests/utils/)|^(.*fixtures.py)|^(tests/scenariosv2/(sim|flows))
       - id: requirements-txt-fixer
         always_run: true
       - id: mixed-line-ending

packages/syft/setup.cfg

@@ -139,10 +139,9 @@ test_plugins =
     faker
     distro
     dynaconf
-    ; pytest-asyncio
-    ; pytest-timeout
-    ; anyio
-    ; unsync
+    pytest-asyncio
+    pytest-timeout
+    anyio
 
 [options.entry_points]
 console_scripts =

packages/syft/src/syft/util/test_helpers/apis/mock/schema.py

@@ -9,15 +9,19 @@
 from .data import schema_dict
 
 
-def make_schema(settings, worker_pool_name) -> Callable:
+def make_schema(
+    settings,
+    worker_pool_name,
+    path="bigquery.schema",
+) -> Callable:
     updated_settings = {
         "calls_per_min": 5,
         "rate_limiter_enabled": True,
         "schema_dict": schema_dict,
     } | settings
 
     @sy.api_endpoint(
-        path="bigquery.schema",
+        path=path,
         description="This endpoint allows for visualising the metadata of tables available in BigQuery.",
         settings=updated_settings,
         helper_functions=[is_within_rate_limit],

packages/syft/src/syft/util/test_helpers/worker_helpers.py

@@ -50,7 +50,10 @@ def build_and_launch_worker_pool_from_docker_str(
     print(result)
     # assert 'success' in str(result.message)
 
-    if environment == "remote":
+    # scale_to > 1 is valid for scale up
+    # scale_to = 0 is valid for removing all pods
+    # scale_to < 0 should return error from server
+    if environment == "remote" and scale_to != 1:
         result = client.worker_pools.scale(number=scale_to, pool_name=worker_pool_name)
         print(result)