Merge pull request #9297 from khoaguin/remove-document-stores

Remove kv and sqlite document stores
OpenMined · Sep 20, 2024 · ea17414 · ea17414
2 parents 5c09f0a + ccffcc4
commit ea17414
Show file tree

Hide file tree

Showing 29 changed files with 336 additions and 1,744 deletions.
diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -621,6 +621,150 @@ jobs:
         run: |
           tox -e migration.test
 
+  pr-tests-migrations-k8s:
+    if: false # skipping this job for now
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      - name: Permission to home directory
+        run: |
+          sudo chown -R $USER:$USER $HOME
+      - uses: actions/checkout@v4
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Add K3d Registry
+        run: |
+          sudo python ./scripts/patch_hosts.py --add-k3d-registry
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          large-packages: false
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
+      - name: Install pip dependencies
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2
+          uv --version
+
+      - name: Get uv cache dir
+        if: steps.changes.outputs.stack == 'true'
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Load github cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
+          restore-keys: |
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}
+
+      - name: Install kubectl
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # cleanup apt version
+          sudo apt remove kubectl || true
+          # install kubectl 1.27
+          curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          sudo install kubectl /usr/local/bin;
+
+      - name: Install helm
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # install helm
+          curl  -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+          chmod 700 get_helm.sh
+          ./get_helm.sh
+
+      - name: Run Migrations Tests
+        if: steps.changes.outputs.stack == 'true'
+        timeout-minutes: 60
+        env:
+          GITHUB_CI: true
+        shell: bash
+        run: |
+          K3D_VERSION=v5.6.3
+          DEVSPACE_VERSION=v6.3.12
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+          tox -e migration.k8s.test
+
+      - name: Get current timestamp
+        id: date
+        if: failure()
+        shell: bash
+        run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT
+
+      - name: Collect logs from k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          mkdir -p ./k8s-logs
+          # kubectl describe all -A --context k3d-syft-migration-source --namespace syft > ./k8s-logs/syft-migration-source-desc-${{ steps.date.outputs.date }}.txt
+          # kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-syft-migration-source --namespace syft > ./k8s-logs/syft-migration-source-logs-${{ steps.date.outputs.date }}.txt
+          kubectl describe all -A --context k3d-test-datasite-1 --namespace syft > ./k8s-logs/test-datasite-1-desc-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-datasite-1 --namespace syft > ./k8s-logs/test-datasite-1-logs-${{ steps.date.outputs.date }}.txt
+          ls -la ./k8s-logs
+
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@master
+        if: steps.changes.outputs.stack == 'true' && failure()
+        with:
+          name: k8s-logs-notebook-${{ matrix.os }}-${{ steps.date.outputs.date }}
+          path: ./k8s-logs/
+
+      - name: Cleanup k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          export PATH=`pwd`:$PATH
+          k3d cluster delete syft-migration-source || true
+          k3d cluster delete test-datasite-1 || true
+          k3d registry delete k3d-registry.localhost || true
+
   pr-tests-notebook-scenario-k8s-sync:
     strategy:
       max-parallel: 99

diff --git a/docs/source/api_reference/syft.store.dict_document_store.rst b/docs/source/api_reference/syft.store.dict_document_store.rst
diff --git a/docs/source/api_reference/syft.store.kv_document_store.rst b/docs/source/api_reference/syft.store.kv_document_store.rst
diff --git a/docs/source/api_reference/syft.store.sqlite_document_store.rst b/docs/source/api_reference/syft.store.sqlite_document_store.rst
diff --git a/packages/grid/backend/grid/core/server.py b/packages/grid/backend/grid/core/server.py
@@ -101,7 +101,6 @@ def seaweedfs_config() -> SeaweedFSConfig:
     server_side_type=server_side_type,
     enable_warnings=enable_warnings,
     blob_storage_config=blob_storage_config,
-    local_db=single_container_mode,
     queue_config=queue_config,
     migrate=False,
     in_memory_workers=settings.INMEMORY_WORKERS,

diff --git a/packages/grid/helm/examples/dev/base.yaml b/packages/grid/helm/examples/dev/base.yaml
@@ -28,6 +28,14 @@ postgres:
   secret:
     rootPassword: example
 
+# Deprecated, support will be removed in 0.9.3
+mongo:
+  resourcesPreset: null
+  resources: null
+
+  secret:
+    rootPassword: example
+
 seaweedfs:
   resourcesPreset: null
   resources: null

diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py
@@ -77,6 +77,8 @@
 from .service.user.roles import Roles as roles
 from .service.user.user_service import UserService
 from .stable_version import LATEST_STABLE_SYFT
+from .store.mongo_document_store import MongoStoreConfig
+from .store.sqlite_document_store import SQLiteStoreConfig
 from .types.errors import SyftException
 from .types.errors import raises
 from .types.result import as_result

diff --git a/packages/syft/src/syft/orchestra.py b/packages/syft/src/syft/orchestra.py
@@ -171,7 +171,6 @@ def deploy_to_python(
     tail: bool,
     dev_mode: bool,
     processes: int,
-    local_db: bool,
     server_side_type: ServerSideType,
     enable_warnings: bool,
     n_consumers: int,
@@ -248,7 +247,6 @@ def deploy_to_python(
             server_side_type=server_side_type,
         )
     else:
-        kwargs["local_db"] = local_db
         kwargs["thread_workers"] = thread_workers
         if server_type_enum in worker_classes:
             worker_class = worker_classes[server_type_enum]
@@ -314,7 +312,6 @@ def launch(
         # worker related inputs
         port: int | str | None = None,
         processes: int = 1,  # temporary work around for jax in subprocess
-        local_db: bool = False,
         dev_mode: bool = False,
         reset: bool = False,
         log_level: str | int | None = None,
@@ -368,7 +365,6 @@ def launch(
                 tail=tail,
                 dev_mode=dev_mode,
                 processes=processes,
-                local_db=local_db,
                 server_side_type=server_side_type_enum,
                 enable_warnings=enable_warnings,
                 log_level=log_level,

diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py
@@ -36,12 +36,8 @@
 
 IGNORE_TYPES = [
     "mock_type",
-    "MockStore",
-    "MockSyftObject",
-    "MockStoreConfig",
     "MockWrapper",
     "base_stash_mock_object_type",
-    "MockKeyValueBackingStore",
     "MockObjectFromSyftBaseObj",
     "MockObjectToSyftBaseObj",
 ]
@@ -237,9 +233,7 @@ def diff_state(self, state: dict) -> tuple[dict, dict]:
             cls, version = serde_properties[7], serde_properties[9]
             if issubclass(cls, SyftBaseObject):
                 canonical_name = cls.__canonical_name__
-                if canonical_name in IGNORE_TYPES or canonical_name.startswith(
-                    "MockSyftObject_"
-                ):
+                if canonical_name in IGNORE_TYPES:
                     continue
 
                 hash_str = DataProtocol._calculate_object_hash(cls)