Merge remote-tracking branch 'upstream/master' into issues/5200

modin-project · Nov 28, 2022 · 11cde06 · 11cde06
2 parents 1c9ff0a + 9534478
commit 11cde06
Show file tree

Hide file tree

Showing 35 changed files with 396 additions and 99 deletions.
diff --git a/.github/workflows/ci-notebooks.yml b/.github/workflows/ci-notebooks.yml
@@ -44,7 +44,9 @@ jobs:
           environment-file: requirements/env_hdk.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
         if: matrix.execution == 'hdk_on_native'
       - name: Cache datasets
         uses: actions/cache@v2

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -192,7 +192,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -230,7 +232,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -308,7 +312,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -360,7 +366,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -417,7 +425,9 @@ jobs:
           activate-environment: modin_on_hdk
           environment-file: requirements/env_hdk.yml
           python-version: 3.8
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -549,7 +559,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -628,7 +640,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -675,7 +689,9 @@ jobs:
           environment-file: requirements/environment-py36.yml
           python-version: 3.6
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -736,7 +752,9 @@ jobs:
           environment-file: requirements/environment-py36.yml
           python-version: 3.6
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -805,7 +823,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -873,8 +893,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
-          auto-update-conda: true # this enable `use-only-tar-bz2` feature on Windows
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -920,7 +941,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -963,7 +986,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -0,0 +1,40 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ python ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+          config-file: ./.github/workflows/codeql/codeql-config.yml
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"
diff --git a/.github/workflows/codeql/codeql-config.yml b/.github/workflows/codeql/codeql-config.yml
@@ -0,0 +1,7 @@
+name: "Modin CodeQL config"
+
+paths:
+  - modin/** 
+paths-ignore: 
+  - modin/experimental/cloud/** # TODO: fix module-level cyclic error, see #5228
+  - modin/experimental/core/execution/native/implementations/hdk_on_native/test/** # TODO: fix unhashable list error, see #5227
diff --git a/.github/workflows/fuzzydata-test.yml b/.github/workflows/fuzzydata-test.yml
@@ -45,7 +45,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info

diff --git a/.github/workflows/push-to-master.yml b/.github/workflows/push-to-master.yml
@@ -20,7 +20,9 @@ jobs:
           environment-file: requirements/requirements-no-engine.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: install Ray nightly build
         run: pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
       - name: Conda environment
@@ -76,7 +78,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -125,7 +129,9 @@ jobs:
           activate-environment: modin
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - run: pip install -r requirements-dev.txt
       # Use a ray master commit that includes the fix here: https://github.com/ray-project/ray/pull/16278
       # Can be changed after a Ray version > 1.4 is released.

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -28,7 +28,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -76,7 +78,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: 3.8
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -122,7 +126,9 @@ jobs:
           activate-environment: modin_on_hdk
           environment-file: requirements/env_hdk.yml
           python-version: 3.8
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -177,7 +183,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -276,8 +284,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
-          auto-update-conda: true # this enable `use-only-tar-bz2` feature on Windows
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -322,7 +331,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info
@@ -364,7 +375,9 @@ jobs:
           environment-file: environment-dev.yml
           python-version: ${{matrix.python-version}}
           channel-priority: strict
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
+          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
+          use-only-tar-bz2: false
       - name: Conda environment
         run: |
           conda info

diff --git a/environment-dev.yml b/environment-dev.yml
@@ -2,10 +2,10 @@ name: modin
 channels:
   - conda-forge
 dependencies:
-  - pandas==1.5.1
+  - pandas==1.5.2
   - numpy>=1.18.5
   - pyarrow>=4.0.1
-  - dask[complete]>=2.22.0
+  - dask>=2.22.0
   - distributed>=2.22.0
   - fsspec
   - xarray

diff --git a/examples/tutorial/jupyter/execution/hdk_on_native/test/test_notebooks.py b/examples/tutorial/jupyter/execution/hdk_on_native/test/test_notebooks.py
@@ -13,6 +13,7 @@
 
 import os
 import sys
+import pytest
 
 import nbformat
 
@@ -44,6 +45,9 @@ def test_exercise_1():
 
 
 # this notebook works "as is"
+# GH #5252: Access to the modin-test bucket has changed, so we cannot currently run this test.
+# We will need to come back and unskip this test once the access control issue is resolved.
+@pytest.mark.skip(reason="Bucket cannot currently be accessed.")
 def test_exercise_2():
     modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_2_test.ipynb")
     nb = nbformat.read(

diff --git a/examples/tutorial/jupyter/execution/pandas_on_dask/test/test_notebooks.py b/examples/tutorial/jupyter/execution/pandas_on_dask/test/test_notebooks.py
@@ -13,6 +13,7 @@
 
 import os
 import sys
+import pytest
 
 import nbformat
 
@@ -46,6 +47,9 @@ def test_exercise_1():
 
 
 # this notebook works "as is" but for testing purposes we can use smaller dataset
+# GH #5252: Access to the modin-test bucket has changed, so we cannot currently run this test.
+# We will need to come back and unskip this test once the access control issue is resolved.
+@pytest.mark.skip(reason="Bucket cannot currently be accessed.")
 def test_exercise_2():
     modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_2_test.ipynb")
     nb = nbformat.read(
@@ -99,6 +103,9 @@ def sq_mad_func(self, axis=None, skipna=True, level=None, **kwargs):
 
 
 # this notebook works "as is" but for testing purposes we can use smaller dataset
+# GH #5252: Access to the modin-test bucket has changed, so we cannot currently run this test.
+# We will need to come back and unskip this test once the access control issue is resolved.
+@pytest.mark.skip(reason="Bucket cannot currently be accessed.")
 def test_exercise_4():
     modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_4_test.ipynb")
     nb = nbformat.read(