diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 4c48b54b7..6caa31e97 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -78,8 +78,7 @@ jobs: - name: Run generic unit tests to download data and construct CSVs if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change run: | - pytest --cov=. --cov-report=xml -k "generic_download" - pytest --cov=. --cov-report=xml -k "generic_constructTrainingCSV" + pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" # openfl tests start here - name: Run OpenFL tests if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change diff --git a/docs/extending.md b/docs/extending.md index 51a44fb03..2677f8c04 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -90,6 +90,15 @@ Once you have made changes to functionality, it is imperative that the unit test ## Run Tests +### Prerequisites + +There are two types of tests: unit tests for GaNDLF code, which tests the functionality, and integration tests for deploying and running mlcubes. Some additional steps are required for running tests: + +1. Ensure that the install optional dependencies [[ref](https://mlcommons.github.io/GaNDLF/setup/#optional-dependencies)] have been installed. +2. Tests are using [sample data](https://drive.google.com/uc?id=1c4Yrv-jnK6Tk7Ne1HmMTChv-4nYk43NT), which gets downloaded and prepared automatically when you run unit tests. Prepared data is stored at `${GaNDLF_root_dir}/testing/data/` folder. However, you may want to download & explore data by yourself. + +### Unit tests + Once you have the virtual environment set up, tests can be run using the following command: ```bash @@ -99,10 +108,19 @@ Once you have the virtual environment set up, tests can be run using the followi Any failures will be reported in the file [`${GaNDLF_HOME}/testing/failures.log`](https://github.com/mlcommons/GaNDLF/blob/5030ff83a38947c1583b58a08598308886ee9a0a/testing/conftest.py#L25). +### Integration tests + +All integration tests are combined to one shell script: + +```shell +# it's assumed you are in `GaNDLF/` repo root directory +cd testing/ +./test_deploy.sh +``` ### Code coverage -The code coverage for the tests can be obtained by the following command: +The code coverage for the unit tests can be obtained by the following command: ```powershell bash diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 000000000..a77626ec0 --- /dev/null +++ b/testing/README.md @@ -0,0 +1,3 @@ +# Running tests locally + +Read more about running tests & sample data: https://mlcommons.github.io/GaNDLF/extending/#run-tests diff --git a/testing/__init__.py b/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/testing/conftest.py b/testing/conftest.py index 742c14a19..6437979c2 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,6 +1,8 @@ import os, pathlib, pytest from pytest import fixture +from .test_full import prerequisites_hook_download_data, prerequisites_constructTrainingCSV + def pytest_addoption(parser): parser.addoption( @@ -27,3 +29,11 @@ def pytest_runtest_makereport(item, call): mode = "a" if os.path.exists(log_filename) else "w" with open(log_filename, mode) as f: f.write(rep.longreprtext + "\n") + + +def pytest_sessionstart(session): + """ + This hook is executed before the pytest session starts. + """ + prerequisites_hook_download_data() + prerequisites_constructTrainingCSV() diff --git a/testing/test_deploy.sh b/testing/test_deploy.sh index bb5d1dfaa..e6e8f04cf 100644 --- a/testing/test_deploy.sh +++ b/testing/test_deploy.sh @@ -15,8 +15,8 @@ mkdir -p test_deploy cd test_deploy # Download the data -FILENAME=y8162xkq1zz5555ye3pwadry2m2e39bs.zip -wget https://upenn.box.com/shared/static/$FILENAME +FILENAME=data.zip +wget https://drive.google.com/uc?id=1c4Yrv-jnK6Tk7Ne1HmMTChv-4nYk43NT --output-document=$FILENAME unzip $FILENAME mv data/3d_rad_segmentation . rm $FILENAME diff --git a/testing/test_full.py b/testing/test_full.py index e6e003467..9fc04a8cb 100644 --- a/testing/test_full.py +++ b/testing/test_full.py @@ -108,7 +108,7 @@ """ -def test_generic_download_data(): +def prerequisites_hook_download_data(): print("00: Downloading the sample data") urlToDownload = "https://drive.google.com/uc?id=1c4Yrv-jnK6Tk7Ne1HmMTChv-4nYk43NT" @@ -132,7 +132,7 @@ def test_generic_download_data(): print("passed") -def test_generic_constructTrainingCSV(): +def prerequisites_constructTrainingCSV(): print("01: Constructing training CSVs") # delete previous csv files files = os.listdir(inputDir) @@ -212,6 +212,13 @@ def test_generic_constructTrainingCSV(): i += 1 +def test_prepare_data_for_ci(): + # is used to run pytest session (i.e. to prepare environment, download data etc) + # without any real test execution + # to see what happens, refer to `conftest.py:pytest_sessionstart` + pass + + # # these are helper functions to be used in other tests def sanitize_outputDir(): print("02_1: Sanitizing outputDir")