Merge pull request #827 from VukW/testing_readme

Added readme for running tests locally

.github/workflows/openfl-test.yml

@@ -78,8 +78,7 @@ jobs:
     - name: Run generic unit tests to download data and construct CSVs
       if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
       run: |
-        pytest --cov=. --cov-report=xml -k "generic_download"
-        pytest --cov=. --cov-report=xml -k "generic_constructTrainingCSV"
+        pytest --cov=. --cov-report=xml -k "prepare_data_for_ci"
     # openfl tests start here
     - name: Run OpenFL tests
       if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change

docs/extending.md

@@ -90,6 +90,15 @@ Once you have made changes to functionality, it is imperative that the unit test
 
 ## Run Tests
 
+### Prerequisites
+
+There are two types of tests: unit tests for GaNDLF code, which tests the functionality, and integration tests for deploying and running mlcubes. Some additional steps are required for running tests:
+
+1. Ensure that the install optional dependencies [[ref](https://mlcommons.github.io/GaNDLF/setup/#optional-dependencies)] have been installed.
+2. Tests are using [sample data](https://drive.google.com/uc?id=1c4Yrv-jnK6Tk7Ne1HmMTChv-4nYk43NT), which gets downloaded and prepared automatically when you run unit tests. Prepared data is stored at `${GaNDLF_root_dir}/testing/data/` folder. However, you may want to download & explore data by yourself.
+
+### Unit tests
+
 Once you have the virtual environment set up, tests can be run using the following command:
 
 ```bash
@@ -99,10 +108,19 @@ Once you have the virtual environment set up, tests can be run using the followi
 
 Any failures will be reported in the file [`${GaNDLF_HOME}/testing/failures.log`](https://github.com/mlcommons/GaNDLF/blob/5030ff83a38947c1583b58a08598308886ee9a0a/testing/conftest.py#L25).
 
+### Integration tests
+
+All integration tests are combined to one shell script:
+
+```shell
+# it's assumed you are in `GaNDLF/` repo root directory
+cd testing/
+./test_deploy.sh
+```
 
 ### Code coverage
 
-The code coverage for the tests can be obtained by the following command:
+The code coverage for the unit tests can be obtained by the following command:
 
 ```powershell
 bash

testing/README.md

@@ -0,0 +1,3 @@
+# Running tests locally
+
+Read more about running tests & sample data: https://mlcommons.github.io/GaNDLF/extending/#run-tests

testing/conftest.py

@@ -1,6 +1,8 @@
 import os, pathlib, pytest
 from pytest import fixture
 
+from .test_full import prerequisites_hook_download_data, prerequisites_constructTrainingCSV
+
 
 def pytest_addoption(parser):
     parser.addoption(
@@ -27,3 +29,11 @@ def pytest_runtest_makereport(item, call):
         mode = "a" if os.path.exists(log_filename) else "w"
         with open(log_filename, mode) as f:
             f.write(rep.longreprtext + "\n")
+
+
+def pytest_sessionstart(session):
+    """
+    This hook is executed before the pytest session starts.
+    """
+    prerequisites_hook_download_data()
+    prerequisites_constructTrainingCSV()

testing/test_deploy.sh

@@ -15,8 +15,8 @@ mkdir -p test_deploy
 cd test_deploy
 
 # Download the data
-FILENAME=y8162xkq1zz5555ye3pwadry2m2e39bs.zip
-wget https://upenn.box.com/shared/static/$FILENAME
+FILENAME=data.zip
+wget https://drive.google.com/uc?id=1c4Yrv-jnK6Tk7Ne1HmMTChv-4nYk43NT --output-document=$FILENAME
 unzip $FILENAME
 mv data/3d_rad_segmentation .
 rm $FILENAME

testing/test_full.py

@@ -108,7 +108,7 @@
 """
 
 
-def test_generic_download_data():
+def prerequisites_hook_download_data():
     print("00: Downloading the sample data")
     urlToDownload = "https://drive.google.com/uc?id=1c4Yrv-jnK6Tk7Ne1HmMTChv-4nYk43NT"
 
@@ -132,7 +132,7 @@ def test_generic_download_data():
     print("passed")
 
 
-def test_generic_constructTrainingCSV():
+def prerequisites_constructTrainingCSV():
     print("01: Constructing training CSVs")
     # delete previous csv files
     files = os.listdir(inputDir)
@@ -212,6 +212,13 @@ def test_generic_constructTrainingCSV():
                 i += 1
 
 
+def test_prepare_data_for_ci():
+    # is used to run pytest session (i.e. to prepare environment, download data etc)
+    # without any real test execution
+    # to see what happens, refer to `conftest.py:pytest_sessionstart`
+    pass
+
+
 # # these are helper functions to be used in other tests
 def sanitize_outputDir():
     print("02_1: Sanitizing outputDir")