From 697006394f32a14051e017a5110c14d0cb3c534f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 2 Jul 2024 17:27:25 -0600 Subject: [PATCH 1/3] add streaming extractor tests to main script --- .github/determine_testing_environment.py | 11 ++++- .github/workflows/all-tests.yml | 7 +++ .../workflows/streaming-extractor-test.yml | 49 ------------------- conftest.py | 12 ----- .../extractors/iblextractors.py | 2 +- 5 files changed, 18 insertions(+), 63 deletions(-) delete mode 100644 .github/workflows/streaming-extractor-test.yml diff --git a/.github/determine_testing_environment.py b/.github/determine_testing_environment.py index 0c0c5ef95b..e392895aa4 100644 --- a/.github/determine_testing_environment.py +++ b/.github/determine_testing_environment.py @@ -36,13 +36,18 @@ file_is_in_src = changed_file.parts[0] == "src" - if changed_file.name == "pyproject.toml": pyproject_toml_changed = True elif changed_file.name == "neobaseextractor.py": neobaseextractor_changed = True + extractors_changed = True elif changed_file.name == "plexon2.py": plexon2_changed = True + elif changed_file.filename == "nwbextractors.py": + stream_extractors_changed = True + extractors_changed = True + elif changed_file.filename == "iblstreamingrecording.py": + stream_extractors_changed = True elif "core" in changed_file.parts: core_changed = True elif "extractors" in changed_file.parts: @@ -90,8 +95,11 @@ run_sorters_test = run_everything or sorters_changed run_internal_sorters_test = run_everything or run_sortingcomponents_tests or sorters_internal_changed +run_streaming_extractors_test = stream_extractors_changed + install_plexon_dependencies = plexon2_changed + environment_varaiables_to_add = { "RUN_EXTRACTORS_TESTS": run_extractor_tests, "RUN_PREPROCESSING_TESTS": run_preprocessing_tests, @@ -106,6 +114,7 @@ "RUN_SORTERS_TESTS": run_sorters_test, "RUN_INTERNAL_SORTERS_TESTS": run_internal_sorters_test, "INSTALL_PLEXON_DEPENDENCIES": install_plexon_dependencies, + "RUN_STREAMING_EXTRACTORS_TESTS": run_streaming_extractors_test, } # Write the conditions to the GITHUB_ENV file diff --git a/.github/workflows/all-tests.yml b/.github/workflows/all-tests.yml index 8dfea50faf..b44c56fbae 100644 --- a/.github/workflows/all-tests.yml +++ b/.github/workflows/all-tests.yml @@ -136,6 +136,13 @@ jobs: pip install -e .[extractors,streaming_extractors,test_extractors] ./.github/run_tests.sh "extractors and not streaming_extractors" --no-virtual-env + - name: Test streaming extracotors + shell: bash + if: env.RUN_STREAMING_EXTRACTORS_TESTS + run: | + pip install -e .[streaming_extractors,test_extractors] + ./.github/run_tests.sh "streaming_extractors" --no-virtual-env + - name: Test preprocessing shell: bash if: env.RUN_PREPROCESSING_TESTS == 'true' diff --git a/.github/workflows/streaming-extractor-test.yml b/.github/workflows/streaming-extractor-test.yml deleted file mode 100644 index 0cb88c3077..0000000000 --- a/.github/workflows/streaming-extractor-test.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Test streaming extractors - -on: - pull_request: - types: [synchronize, opened, reopened] - branches: - - main - -concurrency: # Cancel previous workflows on the same pull request - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test-streaming-extractors: - name: Testing using ${{ matrix.os }} with ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - python-version: ["3.10"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - uses: s-weigand/setup-conda@v1 - with: - update-conda: true - python-version: ${{ matrix.python-version }} - conda-channels: conda-forge - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v41 - - name: Module changes - id: modules-changed - run: | - for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - if [[ $file == *"/nwbextractors.py" || $file == *"/iblstreamingrecording.py"* ]]; then - echo "Streaming files changed changed" - echo "STREAMING_CHANGED=true" >> $GITHUB_OUTPUT - fi - done - - name: Install package and streaming extractor dependencies - if: ${{ steps.modules-changed.outputs.STREAMING_CHANGED == 'true' }} - run: pip install -e .[test_core,streaming_extractors] - - name: run tests - if: steps.modules-changed.outputs.STREAMING_CHANGED == 'true' - run: pytest -m "streaming_extractors" -vv -ra diff --git a/conftest.py b/conftest.py index c4bac6628a..ce5e07b47b 100644 --- a/conftest.py +++ b/conftest.py @@ -1,19 +1,7 @@ import pytest -import shutil -import os from pathlib import Path -ON_GITHUB = bool(os.getenv('GITHUB_ACTIONS')) - - -# define marks -mark_names = ["core", "extractors", "preprocessing", "postprocessing", - "sorters_external", "sorters_internal", "sorters", - "qualitymetrics", "comparison", "curation", - "widgets", "exporters", "sortingcomponents", "generation"] - - @pytest.fixture(scope="module") def create_cache_folder(tmp_path_factory): cache_folder = tmp_path_factory.mktemp("cache_folder") diff --git a/src/spikeinterface/extractors/iblextractors.py b/src/spikeinterface/extractors/iblextractors.py index 27bb95854f..24dc96b10c 100644 --- a/src/spikeinterface/extractors/iblextractors.py +++ b/src/spikeinterface/extractors/iblextractors.py @@ -309,7 +309,7 @@ class IblSortingExtractor(BaseSorting): name = "ibl" installation_mesg = "IBL extractors require ibllib as a dependency." " To install, run: \n\n pip install ibllib\n\n" - def __init__(self, pid, good_clusters_only=False, load_unit_properties=True, one=None): + def __init__(self, pid: str, good_clusters_only: bool = False, load_unit_properties: bool = True, one=None): try: from one.api import ONE from brainbox.io.one import SpikeSortingLoader From 0dc4faf022d5e8740583fe8895da56f476aa92bc Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 2 Jul 2024 17:31:20 -0600 Subject: [PATCH 2/3] wrong use of pathlib --- .github/determine_testing_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/determine_testing_environment.py b/.github/determine_testing_environment.py index e392895aa4..0adb2d4b52 100644 --- a/.github/determine_testing_environment.py +++ b/.github/determine_testing_environment.py @@ -43,10 +43,10 @@ extractors_changed = True elif changed_file.name == "plexon2.py": plexon2_changed = True - elif changed_file.filename == "nwbextractors.py": + elif changed_file.name == "nwbextractors.py": stream_extractors_changed = True extractors_changed = True - elif changed_file.filename == "iblstreamingrecording.py": + elif changed_file.name == "iblstreamingrecording.py": stream_extractors_changed = True elif "core" in changed_file.parts: core_changed = True From 578a7de1077bf5a80ddb8cfe1aa2b75c18101a29 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 2 Jul 2024 17:39:00 -0600 Subject: [PATCH 3/3] another correction --- .github/determine_testing_environment.py | 5 +++-- .github/workflows/all-tests.yml | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/determine_testing_environment.py b/.github/determine_testing_environment.py index 0adb2d4b52..95ad0afc49 100644 --- a/.github/determine_testing_environment.py +++ b/.github/determine_testing_environment.py @@ -30,6 +30,7 @@ exporters_changed = False sortingcomponents_changed = False generation_changed = False +stream_extractors_changed = False for changed_file in changed_files_in_the_pull_request_paths: @@ -44,9 +45,9 @@ elif changed_file.name == "plexon2.py": plexon2_changed = True elif changed_file.name == "nwbextractors.py": + extractors_changed = True # There are NWB tests that are not streaming stream_extractors_changed = True - extractors_changed = True - elif changed_file.name == "iblstreamingrecording.py": + elif changed_file.name == "iblextractors.py": stream_extractors_changed = True elif "core" in changed_file.parts: core_changed = True diff --git a/.github/workflows/all-tests.yml b/.github/workflows/all-tests.yml index b44c56fbae..855fd072de 100644 --- a/.github/workflows/all-tests.yml +++ b/.github/workflows/all-tests.yml @@ -68,6 +68,7 @@ jobs: echo "RUN_SORTERS_TESTS=${RUN_SORTERS_TESTS}" echo "RUN_INTERNAL_SORTERS_TESTS=${RUN_INTERNAL_SORTERS_TESTS}" echo "INSTALL_PLEXON_DEPENDENCIES=${INSTALL_PLEXON_DEPENDENCIES}" + echo "RUN_STREAMING_EXTRACTORS_TESTS=${RUN_STREAMING_EXTRACTORS_TESTS}" - name: Install packages run: |