Merge branch 'main' into pypi-package

cli/cli_chestxray_tutorial_test.sh

@@ -98,6 +98,20 @@ checkFailed "testdata profile activation failed"
 medperf run -b 1 -d $DSET_UID -m 4 -y
 checkFailed "Benchmark execution step failed"
 
+# Test offline compatibility test
+wget -P $MODEL_LOCAL/workspace/additional_files "https://storage.googleapis.com/medperf-storage/chestxray_tutorial/cnn_weights.tar.gz"
+tar -xzvf $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz -C $MODEL_LOCAL/workspace/additional_files
+medperf test run --offline --no-cache \
+  --demo_dataset_url https://storage.googleapis.com/medperf-storage/chestxray_tutorial/demo_data.tar.gz \
+  --demo_dataset_hash "71faabd59139bee698010a0ae3a69e16d97bc4f2dde799d9e187b94ff9157c00" \
+  -p $PREP_LOCAL \
+  -m $MODEL_LOCAL \
+  -e $METRIC_LOCAL
+
+checkFailed "offline compatibility test execution step failed"
+rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz
+rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.pth
+
 echo "====================================="
 echo "Logout users"
 echo "====================================="
@@ -113,7 +127,6 @@ checkFailed "testdata profile activation failed"
 medperf auth logout
 checkFailed "logout failed"
 
-
 echo "====================================="
 echo "Delete test profiles"
 echo "====================================="
@@ -128,4 +141,4 @@ checkFailed "Profile deletion failed"
 
 if ${CLEANUP}; then
   clean
-fi
+fi

cli/medperf/commands/compatibility_test/utils.py

@@ -1,6 +1,6 @@
 from medperf.commands.dataset.prepare import DataPreparation
 from medperf.commands.dataset.submit import DataCreation
-from medperf.utils import get_folders_hash, remove_path
+from medperf.utils import get_file_hash, get_folders_hash, remove_path
 from medperf.exceptions import InvalidArgumentError, InvalidEntityError
 
 from medperf.comms.entity_resources import resources
@@ -48,11 +48,13 @@ def prepare_local_cube(path):
     config.tmp_paths.append(dst)
     cube_metadata_file = os.path.join(path, config.cube_metadata_filename)
     if not os.path.exists(cube_metadata_file):
+        mlcube_yaml_path = os.path.join(path, config.cube_filename)
+        mlcube_yaml_hash = get_file_hash(mlcube_yaml_path)
         temp_metadata = {
             "id": None,
             "name": temp_uid,
             "git_mlcube_url": "mock_url",
-            "mlcube_hash": "",
+            "mlcube_hash": mlcube_yaml_hash,
             "parameters_hash": "",
             "image_tarball_hash": "",
             "additional_files_tarball_hash": "",
@@ -88,9 +90,11 @@ def prepare_cube(cube_uid: str):
     path = path.resolve()
 
     if os.path.exists(path):
-        logging.info("local path provided. Creating symbolic link")
-        temp_uid = prepare_local_cube(path)
-        return temp_uid
+        mlcube_yaml_path = os.path.join(path, config.cube_filename)
+        if os.path.exists(mlcube_yaml_path):
+            logging.info("local path provided. Creating symbolic link")
+            temp_uid = prepare_local_cube(path)
+            return temp_uid
 
     logging.error(f"mlcube {cube_uid} was not found as an existing mlcube")
     raise InvalidArgumentError(
@@ -127,12 +131,12 @@ def create_test_dataset(
         location="local",
         approved=False,
         submit_as_prepared=skip_data_preparation_step,
+        for_test=True,
     )
     data_creation.validate()
     data_creation.create_dataset_object()
     # TODO: existing dataset could make problems
     # make some changes since this is a test dataset
-    data_creation.dataset.for_test = True
     config.tmp_paths.remove(data_creation.dataset.path)
     data_creation.dataset.write()
     if skip_data_preparation_step:

cli/medperf/commands/dataset/submit.py

@@ -28,6 +28,7 @@ def run(
         location: str = None,
         approved: bool = False,
         submit_as_prepared: bool = False,
+        for_test: bool = False,
     ):
         preparation = cls(
             benchmark_uid,
@@ -40,6 +41,7 @@ def run(
             location,
             approved,
             submit_as_prepared,
+            for_test,
         )
         preparation.validate()
         preparation.validate_prep_cube()
@@ -64,6 +66,7 @@ def __init__(
         location: str,
         approved: bool,
         submit_as_prepared: bool,
+        for_test: bool,
     ):
         self.ui = config.ui
         self.data_path = str(Path(data_path).resolve())
@@ -76,6 +79,7 @@ def __init__(
         self.prep_cube_uid = prep_cube_uid
         self.approved = approved
         self.submit_as_prepared = submit_as_prepared
+        self.for_test = for_test
 
     def validate(self):
         if not os.path.exists(self.data_path):
@@ -122,6 +126,7 @@ def create_dataset_object(self):
             generated_metadata={},
             state="DEVELOPMENT",
             submitted_as_prepared=self.submit_as_prepared,
+            for_test=self.for_test,
         )
         dataset.write()
         config.tmp_paths.append(dataset.path)

cli/medperf/tests/commands/dataset/test_submit.py

@@ -37,6 +37,7 @@ def creation(mocker, comms, ui):
         LOCATION,
         False,
         IS_PREPARED,
+        for_test=False,
     )
     mocker.patch(
         PATCH_DATAPREP.format("Cube.get"), return_value=TestCube(is_valid=True)
@@ -76,7 +77,7 @@ def test_validate_prep_cube_gets_prep_cube_if_provided(
         )
 
         # Act
-        creation = DataCreation(None, cube_uid, *[""] * 7, False)
+        creation = DataCreation(None, cube_uid, *[""] * 7, False, False)
         creation.validate_prep_cube()
 
         # Assert
@@ -94,7 +95,7 @@ def test_validate_prep_cube_gets_benchmark_cube_if_provided(
         )
 
         # Act
-        creation = DataCreation(cube_uid, None, *[""] * 7, False)
+        creation = DataCreation(cube_uid, None, *[""] * 7, False, False)
         creation.validate_prep_cube()
 
         # Assert
@@ -107,7 +108,7 @@ def test_fails_if_invalid_params(self, mocker, benchmark_uid, cube_uid, comms, u
         num_arguments = int(benchmark_uid is None) + int(cube_uid is None)
 
         # Act
-        creation = DataCreation(benchmark_uid, cube_uid, *[""] * 7, False)
+        creation = DataCreation(benchmark_uid, cube_uid, *[""] * 7, False, False)
         # Assert
 
         if num_arguments != 1:

cli/tests_setup.sh

@@ -15,6 +15,7 @@ DIRECTORY="${DIRECTORY:-/tmp/medperf_test_files}"
 CLEANUP="${CLEANUP:-false}"
 FRESH="${FRESH:-false}"
 MEDPERF_STORAGE=~/.medperf
+MEDPERF_LOG_STORAGE=~/.medperf_logs
 SERVER_STORAGE_ID="$(echo $SERVER_URL | cut -d '/' -f 3 | sed -e 's/[.:]/_/g')"
 TIMEOUT="${TIMEOUT:-30}"
 VERSION_PREFIX="/api/v0"
@@ -46,7 +47,7 @@ checkFailed(){
     fi
     echo $1
     echo "medperf log:"
-    tail "$MEDPERF_STORAGE/logs/medperf.log"
+    tail "$MEDPERF_LOG_STORAGE/medperf.log"
     if ${CLEANUP}; then
       clean
     fi
@@ -104,3 +105,8 @@ MODELOWNER="[email protected]"
 DATAOWNER="[email protected]"
 BENCHMARKOWNER="[email protected]"
 ADMIN="[email protected]"
+
+# local MLCubes for local compatibility tests
+PREP_LOCAL="$(dirname $(dirname $(realpath "$0")))/examples/chestxray_tutorial/data_preparator/mlcube"
+MODEL_LOCAL="$(dirname $(dirname $(realpath "$0")))/examples/chestxray_tutorial/model_custom_cnn/mlcube"
+METRIC_LOCAL="$(dirname $(dirname $(realpath "$0")))/examples/chestxray_tutorial/metrics/mlcube"