Merge pull request #7998 from OpenMined/add_container_stack_tests

Add notebook container stack tests and Fix Bug

.github/workflows/pr-tests-stack.yml

@@ -230,6 +230,215 @@ jobs:
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}
 
+  pr-tests-notebook-stack:
+    strategy:
+      max-parallel: 99
+      matrix:
+        # os: [ubuntu-latest, macos-latest, windows-latest, windows]
+        os: [om-ci-16vcpu-ubuntu2204]
+        python-version: ["3.11"]
+        notebook-paths: ["api/0.8"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      - name: Permission to home directory
+        run: |
+          sudo chown -R $USER:$USER $HOME
+      - name: "clean .git/config"
+        if: matrix.os == 'windows'
+        continue-on-error: true
+        shell: bash
+        run: |
+          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
+          rm ${GITHUB_WORKSPACE}/.git/config
+
+      - uses: actions/checkout@v3
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Get pip cache dir
+        if: steps.changes.outputs.stack == 'true'
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: buildjet/cache@v3
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Install tox
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          pip install -U tox
+
+      - name: Show choco installed packages
+        if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows'
+        uses: crazy-max/ghaction-chocolatey@v1
+        with:
+          args: list --localonly
+
+      - name: Install git
+        if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows'
+        uses: crazy-max/ghaction-chocolatey@v1
+        with:
+          args: install git.install --params "/GitAndUnixToolsOnPath /WindowsTerminal /NoAutoCrlf" -y
+
+      - name: Install cmake
+        if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows'
+        uses: crazy-max/ghaction-chocolatey@v1
+        with:
+          args: install cmake.portable --installargs 'ADD_CMAKE_TO_PATH=System' -y
+
+      - name: Check cmake version
+        if: steps.changes.outputs.stack == 'true' &&  matrix.os == 'windows'
+        run: |
+          cmake --version
+        shell: cmd
+
+      - name: Install visualcpp-build-tools
+        if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows'
+        uses: crazy-max/ghaction-chocolatey@v1
+        with:
+          args: install visualstudio2019-workload-vctools -y
+
+      - name: Install jaxlib
+        if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows'
+        run: |
+          python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''"
+
+      - name: Install Docker Compose
+        if: steps.changes.outputs.stack == 'true' && runner.os == 'Linux'
+        shell: bash
+        run: |
+          mkdir -p ~/.docker/cli-plugins
+          DOCKER_COMPOSE_VERSION=v2.16.0
+          curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
+
+      - name: Docker on MacOS
+        if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
+        uses: crazy-max/[email protected]
+
+      - name: Docker Compose on MacOS
+        if: steps.changes.outputs.stack == 'true' && matrix.os == 'macos-latest'
+        shell: bash
+        run: |
+          brew install docker-compose
+          mkdir -p ~/.docker/cli-plugins
+          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose || true
+          docker compose version
+
+      - name: Remove existing containers
+        if: steps.changes.outputs.stack == 'true'
+        continue-on-error: true
+        shell: bash
+        run: |
+          docker rm $(docker ps -aq) --force || true
+          docker volume prune -f || true
+          docker buildx use default || true
+
+      - name: Run Notebook integration tests
+        if: steps.changes.outputs.stack == 'true'
+        timeout-minutes: 60
+        env:
+          ORCHESTRA_DEPLOYMENT_TYPE: "container_stack"
+          TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
+          PYTEST_MODULES: "${{ matrix.pytest-modules }}"
+        run: |
+          tox -e stack.test.notebook
+
+      #Run log collector python script
+      - name: Run log collector
+        timeout-minutes: 5
+        if: failure()
+        shell: bash
+        run: |
+          python ./scripts/container_log_collector.py
+
+      # Get Job name and url
+      - name: Get job name and url
+        id: job_name
+        if: failure()
+        shell: bash
+        run: |
+          echo "job_name=$(echo ${{ github.job }})" >> $GITHUB_OUTPUT
+          echo "url=$(echo ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_OUTPUT
+
+      - name: Get current date
+        id: date
+        if: failure()
+        shell: bash
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@master
+        if: failure()
+        with:
+          name: ${{ matrix.os }}-${{ steps.job_name.outputs.job_name }}-${{ matrix.pytest-modules }}-logs-${{ steps.date.outputs.date }}
+          path: ./logs/${{ steps.job_name.outputs.job_name}}/
+
+      - name: Mandatory Container cleanup
+        if: steps.changes.outputs.stack == 'true'
+        continue-on-error: true
+        shell: bash
+        run: |
+          docker rm `docker ps -aq` --force || true
+          docker volume prune -f || true
+
+      # Get Job name and url
+      - name: Reboot node
+        if: matrix.os == 'windows' && failure()
+        run: |
+          shutdown /r /t 1
+
+      #Get Pull request url
+      - name: Get pull request url
+        id: pull_request
+        if: failure()
+        shell: bash
+        run: |
+          echo "url=$(echo ${{ github.event.pull_request.html_url }})" >> $GITHUB_OUTPUT
+
+      - name: Job Report Status
+        # cant access secrets on forks
+        if: github.repository == 'OpenMined/PySyft' && failure()
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: " {workflow} has {status_message}"
+          message_format: "${{matrix.os}} {emoji} *{job}* {status_message} in {run_url}"
+          footer: "Find the PR here ${{ steps.pull_request.outputs.url }}"
+          mention_users: "U01LNCACY03,U8KUAD396,UNMQ2SJSW,U01SAESBJA0"
+          mention_users_when: "failure,warnings"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}
+
   pr-tests-stack-k8s:
     strategy:
       max-parallel: 99

notebooks/api/0.8/05-custom-policy.ipynb

@@ -229,7 +229,8 @@
    },
    "outputs": [],
    "source": [
-    "domain_client.code.request_code_execution(func)"
+    "request = domain_client.code.request_code_execution(func)\n",
+    "request"
    ]
   },
   {
@@ -253,20 +254,7 @@
    },
    "outputs": [],
    "source": [
-    "func = domain_client.code[-1]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7edf4fc6-302e-4bfd-94e5-33cc911f6db0",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "request = domain_client.requests[-1]\n",
-    "request"
+    "func = request.code"
    ]
   },
   {
@@ -420,7 +408,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.11.4"
   },
   "toc": {
    "base_numbering": 1,

packages/hagrid/hagrid/deps.py

@@ -333,7 +333,7 @@ def extract_version(self, lines: List[str]) -> None:
                     if "-gitpod" in self.version:
                         parts = self.version.split("-gitpod")
                         self.version = parts[0]
-                    if "-desktop":
+                    if "-desktop" in self.version:
                         parts = self.version.split("-desktop")
                         self.version = parts[0]
                     self.version = version.parse(self.version)

packages/hagrid/hagrid/orchestra.py

@@ -69,18 +69,19 @@ def container_exists(name: str) -> bool:
     return len(output) > 0
 
 
-def container_id(name: str) -> Optional[str]:
-    output = shell(f"docker ps -q -f name='{name}'")
-    if len(output) > 0:
-        return output[0].strip()
-    return None
-
+def port_from_container(name: str, deployment_type: DeploymentType) -> Optional[int]:
+    container_suffix = ""
+    if deployment_type == DeploymentType.SINGLE_CONTAINER:
+        container_suffix = "-worker-1"
+    elif deployment_type == DeploymentType.CONTAINER_STACK:
+        container_suffix = "-proxy-1"
+    else:
+        raise NotImplementedError(
+            f"port_from_container not implemented for the deployment type:{deployment_type}"
+        )
 
-def port_from_container(name: str) -> Optional[int]:
-    cid = container_id(name)
-    if cid is None:
-        return None
-    output = shell(f"docker port {cid}")
+    container_name = name + container_suffix
+    output = shell(f"docker port {container_name}")
     if len(output) > 0:
         try:
             # 80/tcp -> 0.0.0.0:8080
@@ -94,7 +95,9 @@ def port_from_container(name: str) -> Optional[int]:
 
 
 def container_exists_with(name: str, port: int) -> bool:
-    output = shell(f"docker ps -q -f name='{name}' -f expose='{port}'")
+    output = shell(
+        f"docker ps -q -f name={name} | xargs -n 1 docker port | grep 0.0.0.0:{port}"
+    )
     return len(output) > 0
 
 
@@ -357,7 +360,7 @@ def deploy_to_container(
 ) -> Optional[NodeHandle]:
     if port == "auto" or port is None:
         if container_exists(name=name):
-            port = port_from_container(name=name)  # type: ignore
+            port = port_from_container(name=name, deployment_type=deployment_type_enum)  # type: ignore
         else:
             port = find_available_port(host="localhost", port=DEFAULT_PORT, search=True)
 
@@ -566,16 +569,19 @@ def reset(name: str, deployment_type_enum: DeploymentType) -> None:
 
             snake_name = to_snake_case(name)
 
-            volume_output = shell(
-                f"docker volume rm {snake_name}_credentials-data --force || true"
-            )
+            volumes = ["mongo-data", "credentials-data"]
 
-            if "Error" not in volume_output:
-                print(f" ✅ {snake_name} Volume Removed")
-            else:
-                print(
-                    f"❌ Unable to remove container volume: {snake_name} :{volume_output}"
+            for volume in volumes:
+                volume_output = shell(
+                    f"docker volume rm {snake_name}_{volume} --force || true"
                 )
+
+                if "Error" not in volume_output:
+                    print(f" ✅ {snake_name}_{volume} Volume Removed")
+                else:
+                    print(
+                        f"❌ Unable to remove container volume: {snake_name} :{volume_output}"
+                    )
         else:
             raise NotImplementedError(
                 f"Reset not implemented for the deployment type:{deployment_type_enum}"

packages/syft/src/syft/store/mongo_document_store.py

@@ -348,6 +348,10 @@ def has_permission(self, permission: ActionObjectPermission) -> bool:
         # TODO: implement
         return True
 
+    def add_permissions(self, permissions: List[ActionObjectPermission]) -> None:
+        # TODO: implemenet
+        pass
+
     def _all(
         self,
         credentials: SyftVerifyKey,

tox.ini

@@ -391,6 +391,11 @@ setenv =
     TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:api/0.8,tutorials}
     ENABLE_SIGNUP=True
 commands =
+
+    # Volume cleanup
+    bash -c "docker volume rm test-domain-1_mongo-data --force || true"
+    bash -c "docker volume rm test-domain-1_credentials-data --force || true"
+
     bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE DEV_MODE=$DEV_MODE TEST_NOTEBOOK_PATHS=$TEST_NOTEBOOK_PATHS; date"
     bash -c "for subfolder in $(echo ${TEST_NOTEBOOK_PATHS} | tr ',' ' ');\
     do \