From 71582b7955defc18c1c2ccfd90a6508cbaddb970 Mon Sep 17 00:00:00 2001 From: hasan7n <78664424+hasan7n@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:54:51 +0100 Subject: [PATCH] Singularity support, refactor cube.download (#526) * refactor mlcube.download * support singularity * outdated code * update docs * use accelerator count 0 * unittests fix/add * add singularity integration tests * update --gpus help msg * update tests mockcube commit hash * fix bug * remove outdated test * improve integration tests logging * fix integration tests * tmp code for debugging * empty * debug * try singularity 3.10.1 * debugging singularity version * Revert "debugging singularity version" This reverts commit aade207da1ced419deadbcb1b1150791a2dfa0c3. * Revert "try singularity 3.10.1" This reverts commit c2a8b3b393bf4771281bbc177a6140bc8953c88e. * Revert "debug" This reverts commit 9793a8cbf920853cc5805b0e4afbfc048afbb3ea. * Revert "tmp code for debugging" This reverts commit 22853b8b11b20937fadcf0f11a3ce24ace9f9de6. * change working directory of installing singularity * do same for docker-ci --- .github/workflows/docker-ci.yml | 1 + .github/workflows/local-ci.yml | 3 + cli/cli_tests.sh | 16 ++- .../commands/compatibility_test/utils.py | 1 + cli/medperf/commands/dataset/create.py | 1 + cli/medperf/commands/mlcube/submit.py | 3 +- cli/medperf/commands/result/create.py | 1 + cli/medperf/decorators.py | 10 +- cli/medperf/entities/cube.py | 126 ++++++++++++------ .../tests/commands/dataset/test_create.py | 17 +-- .../tests/commands/mlcube/test_submit.py | 10 +- .../tests/commands/result/test_create.py | 1 + cli/medperf/tests/entities/test_cube.py | 56 +++++--- cli/medperf/tests/mocks/__init__.py | 4 +- cli/medperf/tests/mocks/cube.py | 20 --- cli/requirements.txt | 6 +- cli/tests_setup.sh | 5 +- docs/getting_started/installation.md | 5 +- docs/getting_started/setup.md | 3 - 19 files changed, 174 insertions(+), 115 deletions(-) diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index 51fecafbc..b7d778782 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -18,6 +18,7 @@ jobs: python-version: '3.9' - name: Install Singularity + working-directory: .. run: | sudo apt-get update sudo apt-get install -y build-essential libssl-dev uuid-dev libgpgme11-dev \ diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml index ed42a7134..e48a4b0de 100644 --- a/.github/workflows/local-ci.yml +++ b/.github/workflows/local-ci.yml @@ -16,6 +16,9 @@ jobs: python-version: '3.9' - name: Install Singularity + # The way we install singularity shouldn't happen inside another git repo, otherwise + # the singularity installer will set that repo's tag/commit as the singularity version. + working-directory: .. run: | sudo apt-get update sudo apt-get install -y build-essential libssl-dev uuid-dev libgpgme11-dev \ diff --git a/cli/cli_tests.sh b/cli/cli_tests.sh index 45466b339..df803391f 100755 --- a/cli/cli_tests.sh +++ b/cli/cli_tests.sh @@ -107,6 +107,7 @@ medperf mlcube submit --name model2 -m $MODEL_MLCUBE -p $MODEL2_PARAMS -a $MODEL checkFailed "Model2 submission failed" MODEL2_UID=$(medperf mlcube ls | tail -n 1 | tr -s ' ' | cut -d ' ' -f 2) +# MLCube with singularity section medperf mlcube submit --name model3 -m $MODEL_WITH_SINGULARITY -p $MODEL3_PARAMS -a $MODEL_ADD -i $MODEL_SING_IMAGE checkFailed "Model3 submission failed" MODEL3_UID=$(medperf mlcube ls | tail -n 1 | tr -s ' ' | cut -d ' ' -f 2) @@ -236,11 +237,12 @@ echo "\n" ########################################################## echo "=====================================" -echo "Running model3 association" +echo "Running model3 association (with singularity)" echo "=====================================" -# medperf --platform singularity mlcube associate -m $MODEL3_UID -b $BMK_UID -y -# TMP: revert to singularity when MLCube issue is fixed -medperf mlcube associate -m $MODEL3_UID -b $BMK_UID -y +# this will run two types of singularity mlcubes: +# 1) an already built singularity image (model 3) +# 2) a docker image to be converted (metrics) +medperf --platform singularity mlcube associate -m $MODEL3_UID -b $BMK_UID -y checkFailed "Model3 association failed" ########################################################## @@ -312,10 +314,10 @@ echo "\n" ########################################################## echo "=====================================" -echo "Running model2" +echo "Running model3 (with singularity)" echo "=====================================" -medperf run -b $BMK_UID -d $DSET_A_UID -m $MODEL2_UID -y -checkFailed "Model2 run failed" +medperf --platform=singularity run -b $BMK_UID -d $DSET_A_UID -m $MODEL3_UID -y +checkFailed "Model3 run failed" ########################################################## echo "\n" diff --git a/cli/medperf/commands/compatibility_test/utils.py b/cli/medperf/commands/compatibility_test/utils.py index 66363edeb..48eb784c1 100644 --- a/cli/medperf/commands/compatibility_test/utils.py +++ b/cli/medperf/commands/compatibility_test/utils.py @@ -95,5 +95,6 @@ def prepare_cube(cube_uid: str): def get_cube(uid: int, name: str, local_only: bool = False) -> Cube: config.ui.text = f"Retrieving {name} cube" cube = Cube.get(uid, local_only=local_only) + cube.download_run_files() config.ui.print(f"> {name} cube download complete") return cube diff --git a/cli/medperf/commands/dataset/create.py b/cli/medperf/commands/dataset/create.py index 7541fdf6b..8d321c74b 100644 --- a/cli/medperf/commands/dataset/create.py +++ b/cli/medperf/commands/dataset/create.py @@ -96,6 +96,7 @@ def get_prep_cube(self): self.ui.print(f"Benchmark Data Preparation: {benchmark.name}") self.ui.text = "Retrieving data preparation cube" self.cube = Cube.get(cube_uid) + self.cube.download_run_files() self.ui.print("> Preparation cube download complete") def run_cube_tasks(self): diff --git a/cli/medperf/commands/mlcube/submit.py b/cli/medperf/commands/mlcube/submit.py index 57e7ecd8a..346aaf97a 100644 --- a/cli/medperf/commands/mlcube/submit.py +++ b/cli/medperf/commands/mlcube/submit.py @@ -31,7 +31,8 @@ def __init__(self, submit_info: dict): config.tmp_paths.append(self.cube.path) def download(self): - self.cube.download() + self.cube.download_config_files() + self.cube.download_run_files() def upload(self): updated_body = self.cube.upload() diff --git a/cli/medperf/commands/result/create.py b/cli/medperf/commands/result/create.py index 576cd3555..a1b7e4aea 100644 --- a/cli/medperf/commands/result/create.py +++ b/cli/medperf/commands/result/create.py @@ -153,6 +153,7 @@ def load_cached_results(self): def __get_cube(self, uid: int, name: str) -> Cube: self.ui.text = f"Retrieving {name} cube" cube = Cube.get(uid) + cube.download_run_files() self.ui.print(f"> {name} cube download complete") return cube diff --git a/cli/medperf/decorators.py b/cli/medperf/decorators.py index e94ec1b90..7f5f6493c 100644 --- a/cli/medperf/decorators.py +++ b/cli/medperf/decorators.py @@ -198,9 +198,13 @@ def wrapper( "--gpus", help=""" What GPUs to expose to MLCube. - Accepted Values are comma separated GPU IDs (e.g "1,2"), or \"all\". - MLCubes that aren't configured to use GPUs won't be affected by this. - Defaults to all available GPUs""", + Accepted Values are:\n + - "" or 0: to expose no GPUs (e.g.: --gpus="")\n + - "all": to expose all GPUs. (e.g.: --gpus=all)\n + - an integer: to expose a certain number of GPUs. ONLY AVAILABLE FOR DOCKER + (e.g., --gpus=2 to expose 2 GPUs)\n + - Form "device=,": to expose specific GPUs. + (e.g., --gpus="device=0,2")\n""", ), cleanup: bool = typer.Option( config.cleanup, diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index 2b5f941ef..534106380 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -167,7 +167,7 @@ def get(cls, cube_uid: Union[str, int], local_only: bool = False) -> "Cube": if not cube.is_valid: raise InvalidEntityError("The requested MLCube is marked as INVALID.") - cube.download() + cube.download_config_files() return cube @classmethod @@ -211,45 +211,66 @@ def download_additional(self): def download_image(self): url = self.image_tarball_url tarball_hash = self.image_tarball_hash - img_hash = self.image_hash if url: _, local_hash = resources.get_cube_image(url, self.path, tarball_hash) self.image_tarball_hash = local_hash else: - # Retrieve image from image registry - logging.debug(f"Retrieving {self.id} image") - cmd = f"mlcube configure --mlcube={self.cube_path}" - with pexpect.spawn(cmd, timeout=config.mlcube_configure_timeout) as proc: - proc_out = proc.read() - if proc.exitstatus != 0: - raise ExecutionError( - "There was an error while retrieving the MLCube image" - ) - logging.debug(proc_out) - - # Retrieve image hash from MLCube - logging.debug(f"Retrieving {self.id} image hash") - tmp_out_yaml = generate_tmp_path() - cmd = f"mlcube inspect --mlcube={self.cube_path} --format=yaml" - cmd += f" --output-file {tmp_out_yaml}" - with pexpect.spawn(cmd, timeout=config.mlcube_inspect_timeout) as proc: - proc_stdout = proc.read() - logging.debug(proc_stdout) - if proc.exitstatus != 0: - raise ExecutionError( - "There was an error while inspecting the image hash" - ) - with open(tmp_out_yaml) as f: - mlcube_details = yaml.safe_load(f) - remove_path(tmp_out_yaml) - local_hash = mlcube_details["hash"] - verify_hash(local_hash, img_hash) - self.image_hash = local_hash - - def download(self): - """Downloads the required elements for an mlcube to run locally.""" + if config.platform == "docker": + # For docker, image should be pulled before calculating its hash + self._get_image_from_registry() + self._set_image_hash_from_registry() + elif config.platform == "singularity": + # For singularity, we need the hash first before trying to convert + self._set_image_hash_from_registry() + + image_folder = os.path.join(config.cubes_folder, config.image_path) + if os.path.exists(image_folder): + for file in os.listdir(image_folder): + if file == self._converted_singularity_image_name: + return + remove_path(os.path.join(image_folder, file)) + + self._get_image_from_registry() + else: + # TODO: such a check should happen on commands entrypoints, not here + raise InvalidArgumentError("Unsupported platform") + + @property + def _converted_singularity_image_name(self): + return f"{self.image_hash}.sif" + + def _set_image_hash_from_registry(self): + # Retrieve image hash from MLCube + logging.debug(f"Retrieving {self.id} image hash") + tmp_out_yaml = generate_tmp_path() + cmd = f"mlcube inspect --mlcube={self.cube_path} --format=yaml" + cmd += f" --platform={config.platform} --output-file {tmp_out_yaml}" + with pexpect.spawn(cmd, timeout=config.mlcube_inspect_timeout) as proc: + proc_stdout = proc.read() + logging.debug(proc_stdout) + if proc.exitstatus != 0: + raise ExecutionError("There was an error while inspecting the image hash") + with open(tmp_out_yaml) as f: + mlcube_details = yaml.safe_load(f) + remove_path(tmp_out_yaml) + local_hash = mlcube_details["hash"] + verify_hash(local_hash, self.image_hash) + self.image_hash = local_hash + + def _get_image_from_registry(self): + # Retrieve image from image registry + logging.debug(f"Retrieving {self.id} image") + cmd = f"mlcube configure --mlcube={self.cube_path} --platform={config.platform}" + if config.platform == "singularity": + cmd += f" -Psingularity.image={self._converted_singularity_image_name}" + with pexpect.spawn(cmd, timeout=config.mlcube_configure_timeout) as proc: + proc_out = proc.read() + if proc.exitstatus != 0: + raise ExecutionError("There was an error while retrieving the MLCube image") + logging.debug(proc_out) + def download_config_files(self): try: self.download_mlcube() except InvalidEntityError as e: @@ -260,6 +281,7 @@ def download(self): except InvalidEntityError as e: raise InvalidEntityError(f"MLCube {self.name} parameters file: {e}") + def download_run_files(self): try: self.download_additional() except InvalidEntityError as e: @@ -301,12 +323,36 @@ def run( cmd_arg = f'{k}="{v}"' cmd = " ".join([cmd, cmd_arg]) - cpu_args = self.get_config("docker.cpu_args") or "" - gpu_args = self.get_config("docker.gpu_args") or "" - cpu_args = " ".join([cpu_args, "-u $(id -u):$(id -g)"]).strip() - gpu_args = " ".join([gpu_args, "-u $(id -u):$(id -g)"]).strip() - cmd += f' -Pdocker.cpu_args="{cpu_args}"' - cmd += f' -Pdocker.gpu_args="{gpu_args}"' + # TODO: we should override run args instead of what we are doing below + # we shouldn't allow arbitrary run args unless our client allows it + if config.platform == "docker": + # use current user + cpu_args = self.get_config("docker.cpu_args") or "" + gpu_args = self.get_config("docker.gpu_args") or "" + cpu_args = " ".join([cpu_args, "-u $(id -u):$(id -g)"]).strip() + gpu_args = " ".join([gpu_args, "-u $(id -u):$(id -g)"]).strip() + cmd += f' -Pdocker.cpu_args="{cpu_args}"' + cmd += f' -Pdocker.gpu_args="{gpu_args}"' + elif config.platform == "singularity": + # use -e to discard host env vars, -C to isolate the container (see singularity run --help) + run_args = self.get_config("singularity.run_args") or "" + run_args = " ".join([run_args, "-eC"]).strip() + cmd += f' -Psingularity.run_args="{run_args}"' + + # set image name in case of running docker image with singularity + # Assuming we only accept mlcube.yamls with either singularity or docker sections + # TODO: make checks on submitted mlcubes + singularity_config = self.get_config("singularity") + if singularity_config is None: + cmd += ( + f' -Psingularity.image="{self._converted_singularity_image_name}"' + ) + else: + raise InvalidArgumentError("Unsupported platform") + + # set accelerator count to zero to avoid unexpected behaviours and + # force mlcube to only use --gpus to figure out GPU config + cmd += " -Pplatform.accelerator_count=0" logging.info(f"Running MLCube command: {cmd}") proc = pexpect.spawn(cmd, timeout=timeout) diff --git a/cli/medperf/tests/commands/dataset/test_create.py b/cli/medperf/tests/commands/dataset/test_create.py index 60ef877ee..081a63bb2 100644 --- a/cli/medperf/tests/commands/dataset/test_create.py +++ b/cli/medperf/tests/commands/dataset/test_create.py @@ -4,7 +4,7 @@ import pytest from unittest.mock import call -from medperf.tests.mocks import MockCube +from medperf.tests.mocks import TestCube from medperf.tests.mocks.benchmark import TestBenchmark from medperf.tests.mocks.dataset import TestDataset from medperf.commands.dataset.create import DataPreparation @@ -39,7 +39,8 @@ def preparation(mocker, comms, ui): DESCRIPTION, LOCATION, ) - mocker.patch(PATCH_DATAPREP.format("Cube.get"), return_value=MockCube(True)) + mocker.patch(PATCH_DATAPREP.format("Cube.get"), return_value=TestCube()) + mocker.patch(PATCH_DATAPREP.format("Cube.download_run_files")) preparation.get_prep_cube() preparation.data_path = DATA_PATH preparation.labels_path = LABELS_PATH @@ -75,9 +76,8 @@ def test_get_prep_cube_gets_prep_cube_if_provided( self, mocker, cube_uid, comms, ui ): # Arrange - spy = mocker.patch( - PATCH_DATAPREP.format("Cube.get"), return_value=MockCube(True) - ) + spy = mocker.patch(PATCH_DATAPREP.format("Cube.get"), return_value=TestCube()) + down_spy = mocker.patch(PATCH_DATAPREP.format("Cube.download_run_files")) # Act preparation = DataPreparation(None, cube_uid, *[""] * 5) @@ -85,6 +85,7 @@ def test_get_prep_cube_gets_prep_cube_if_provided( # Assert spy.assert_called_once_with(cube_uid) + down_spy.assert_called_once() @pytest.mark.parametrize("cube_uid", [998, 68, 109]) def test_get_prep_cube_gets_benchmark_cube_if_provided( @@ -93,9 +94,8 @@ def test_get_prep_cube_gets_benchmark_cube_if_provided( # Arrange benchmark = TestBenchmark(data_preparation_mlcube=cube_uid) mocker.patch(PATCH_DATAPREP.format("Benchmark.get"), return_value=benchmark) - spy = mocker.patch( - PATCH_DATAPREP.format("Cube.get"), return_value=MockCube(True) - ) + spy = mocker.patch(PATCH_DATAPREP.format("Cube.get"), return_value=TestCube()) + down_spy = mocker.patch(PATCH_DATAPREP.format("Cube.download_run_files")) # Act preparation = DataPreparation(cube_uid, None, *[""] * 5) @@ -103,6 +103,7 @@ def test_get_prep_cube_gets_benchmark_cube_if_provided( # Assert spy.assert_called_once_with(cube_uid) + down_spy.assert_called_once() def test_run_cube_tasks_runs_required_tasks(self, mocker, preparation): # Arrange diff --git a/cli/medperf/tests/commands/mlcube/test_submit.py b/cli/medperf/tests/commands/mlcube/test_submit.py index 2d0d35498..630390205 100644 --- a/cli/medperf/tests/commands/mlcube/test_submit.py +++ b/cli/medperf/tests/commands/mlcube/test_submit.py @@ -10,7 +10,8 @@ @pytest.fixture def cube(mocker): - mocker.patch(PATCH_MLCUBE.format("Cube.download")) + mocker.patch(PATCH_MLCUBE.format("Cube.download_config_files")) + mocker.patch(PATCH_MLCUBE.format("Cube.download_run_files")) mocker.patch(PATCH_MLCUBE.format("Cube.upload")) mocker.patch(PATCH_MLCUBE.format("Cube.write")) return TestCube() @@ -90,10 +91,11 @@ def test_upload_uploads_using_entity(mocker, comms, ui, cube): def test_download_executes_expected_commands(mocker, comms, ui, cube): submission = SubmitCube(cube.todict()) - down_spy = mocker.patch(PATCH_MLCUBE.format("Cube.download")) - + config_down_spy = mocker.patch(PATCH_MLCUBE.format("Cube.download_config_files")) + run_down_spy = mocker.patch(PATCH_MLCUBE.format("Cube.download_run_files")) # Act submission.download() # Assert - down_spy.assert_called_once_with() + config_down_spy.assert_called_once_with() + run_down_spy.assert_called_once_with() diff --git a/cli/medperf/tests/commands/result/test_create.py b/cli/medperf/tests/commands/result/test_create.py index 0311b6318..aa5adfa79 100644 --- a/cli/medperf/tests/commands/result/test_create.py +++ b/cli/medperf/tests/commands/result/test_create.py @@ -74,6 +74,7 @@ def __get_side_effect(id): return cube mocker.patch(PATCH_EXECUTION.format("Cube.get"), side_effect=__get_side_effect) + mocker.patch(PATCH_EXECUTION.format("Cube.download_run_files")) def mock_execution(mocker, state_variables): diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index d71b4982c..cf341ec02 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -60,12 +60,8 @@ def set_common_attributes(self, setup): self.cube_path, config.additional_path, config.tarball_filename ) self.img_path = os.path.join(self.cube_path, config.image_path, "img.tar.gz") - self.file_paths = [ - self.manifest_path, - self.params_path, - self.add_path, - self.img_path, - ] + self.config_files_paths = [self.manifest_path, self.params_path] + self.run_files_paths = [self.add_path, self.img_path] @pytest.mark.parametrize("setup", [{"remote": [DEFAULT_CUBE]}], indirect=True) def test_get_cube_retrieves_files(self, setup): @@ -73,11 +69,25 @@ def test_get_cube_retrieves_files(self, setup): Cube.get(self.id) # Assert - for file in self.file_paths: + for file in self.config_files_paths: + assert os.path.exists(file) and os.path.isfile(file) + for file in self.run_files_paths: + assert not os.path.exists(file) + + @pytest.mark.parametrize("setup", [{"remote": [DEFAULT_CUBE]}], indirect=True) + def test_download_run_files_retrieves_files(self, setup): + # Act + cube = Cube.get(self.id) + cube.download_run_files() + + # Assert + for file in self.config_files_paths + self.run_files_paths: assert os.path.exists(file) and os.path.isfile(file) @pytest.mark.parametrize("setup", [{"remote": [NO_IMG_CUBE]}], indirect=True) - def test_get_cube_without_image_configures_mlcube(self, mocker, setup, fs): + def test_download_run_files_without_image_configures_mlcube( + self, mocker, setup, fs + ): # Arrange tmp_path = "tmp_path" mocker.patch(PATCH_CUBE.format("generate_tmp_path"), return_value=tmp_path) @@ -87,20 +97,23 @@ def test_get_cube_without_image_configures_mlcube(self, mocker, setup, fs): ) spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") expected_cmds = [ - f"mlcube configure --mlcube={self.manifest_path}", + f"mlcube configure --mlcube={self.manifest_path} --platform={config.platform}", f"mlcube inspect --mlcube={self.manifest_path}" - f" --format=yaml --output-file {tmp_path}", + f" --format=yaml --platform={config.platform} --output-file {tmp_path}", ] expected_cmds = [call(cmd, timeout=None) for cmd in expected_cmds] # Act - Cube.get(self.id) + cube = Cube.get(self.id) + cube.download_run_files() # Assert spy.assert_has_calls(expected_cmds) @pytest.mark.parametrize("setup", [{"remote": [NO_IMG_CUBE]}], indirect=True) - def test_get_cube_stops_execution_if_configure_fails(self, mocker, setup, fs): + def test_download_run_files_stops_execution_if_configure_fails( + self, mocker, setup, fs + ): # Arrange tmp_path = "tmp_path" mocker.patch(PATCH_CUBE.format("generate_tmp_path"), return_value=tmp_path) @@ -112,11 +125,14 @@ def test_get_cube_stops_execution_if_configure_fails(self, mocker, setup, fs): mocker.patch("pexpect.spawn", side_effect=mpexpect.spawn) # Act & Assert + cube = Cube.get(self.id) with pytest.raises(ExecutionError): - Cube.get(self.id) + cube.download_run_files() @pytest.mark.parametrize("setup", [{"remote": [NO_IMG_CUBE]}], indirect=True) - def test_get_cube_without_image_fails_with_wrong_hash(self, mocker, setup, fs): + def test_download_run_files_without_image_fails_with_wrong_hash( + self, mocker, setup, fs + ): # Arrange tmp_path = "tmp_path" mocker.patch(PATCH_CUBE.format("generate_tmp_path"), return_value=tmp_path) @@ -124,16 +140,18 @@ def test_get_cube_without_image_fails_with_wrong_hash(self, mocker, setup, fs): fs.create_file("tmp_path", contents=yaml.dump({"hash": "invalid hash"})) # Act & Assert + cube = Cube.get(self.id) with pytest.raises(InvalidEntityError): - Cube.get(self.id) + cube.download_run_files() @pytest.mark.parametrize("setup", [{"remote": [DEFAULT_CUBE]}], indirect=True) - def test_get_cube_with_image_isnt_configured(self, mocker, setup): + def test_download_run_files_with_image_isnt_configured(self, mocker, setup): # Arrange spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") # Act - Cube.get(self.id) + cube = Cube.get(self.id) + cube.download_run_files() # Assert spy.assert_not_called() @@ -165,6 +183,7 @@ def test_cube_runs_command(self, mocker, timeout, setup, task): + f"--platform={self.platform} --network=none --mount=ro" + ' -Pdocker.cpu_args="-u $(id -u):$(id -g)"' + ' -Pdocker.gpu_args="-u $(id -u):$(id -g)"' + + " -Pplatform.accelerator_count=0" ) # Act @@ -187,6 +206,7 @@ def test_cube_runs_command_with_rw_access(self, mocker, setup, task): + f"--platform={self.platform} --network=none" + ' -Pdocker.cpu_args="-u $(id -u):$(id -g)"' + ' -Pdocker.gpu_args="-u $(id -u):$(id -g)"' + + " -Pplatform.accelerator_count=0" ) # Act @@ -206,6 +226,7 @@ def test_cube_runs_command_with_extra_args(self, mocker, setup, task): + f'--platform={self.platform} --network=none --mount=ro test="test"' + ' -Pdocker.cpu_args="-u $(id -u):$(id -g)"' + ' -Pdocker.gpu_args="-u $(id -u):$(id -g)"' + + " -Pplatform.accelerator_count=0" ) # Act @@ -228,6 +249,7 @@ def test_cube_runs_command_and_preserves_runtime_args(self, mocker, setup, task) + f"--platform={self.platform} --network=none --mount=ro" + ' -Pdocker.cpu_args="cpuarg cpuval -u $(id -u):$(id -g)"' + ' -Pdocker.gpu_args="gpuarg gpuval -u $(id -u):$(id -g)"' + + " -Pplatform.accelerator_count=0" ) # Act diff --git a/cli/medperf/tests/mocks/__init__.py b/cli/medperf/tests/mocks/__init__.py index bbbab6638..faaa893e7 100644 --- a/cli/medperf/tests/mocks/__init__.py +++ b/cli/medperf/tests/mocks/__init__.py @@ -1,6 +1,6 @@ from .requests import MockResponse from .benchmark import Benchmark -from .cube import MockCube +from .cube import TestCube from .tarfile import MockTar -all = [MockResponse, Benchmark, MockCube, MockTar] +all = [MockResponse, Benchmark, TestCube, MockTar] diff --git a/cli/medperf/tests/mocks/cube.py b/cli/medperf/tests/mocks/cube.py index 0ad5ba326..46551ef66 100644 --- a/cli/medperf/tests/mocks/cube.py +++ b/cli/medperf/tests/mocks/cube.py @@ -5,26 +5,6 @@ EMPTY_FILE_HASH = "da39a3ee5e6b4b0d3255bfef95601890afd80709" -class MockCube: - def __init__(self, is_valid): - self.name = "Test" - self.is_valid = is_valid - self.id = 1 - - def valid(self): - return self.is_valid - - def run(self): - pass - - def get_default_output(self, *args, **kwargs): - return "out_path" - - @property - def identifier(self): - return self.id or self.name - - class TestCube(Cube): id: Optional[int] = 1 name: str = "name" diff --git a/cli/requirements.txt b/cli/requirements.txt index eec3d960b..077ca0640 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -10,9 +10,9 @@ colorama==0.4.4 time-machine==2.4.0 pytest-mock==1.13.0 pyfakefs==5.0.0 -mlcube @ git+https://github.com/mlcommons/mlcube@755d388edc9fb32dfa2b09c244252485b2b554b3#subdirectory=mlcube -mlcube-docker @ git+https://github.com/mlcommons/mlcube@755d388edc9fb32dfa2b09c244252485b2b554b3#subdirectory=runners/mlcube_docker -mlcube-singularity @ git+https://github.com/mlcommons/mlcube@755d388edc9fb32dfa2b09c244252485b2b554b3#subdirectory=runners/mlcube_singularity +mlcube @ git+https://github.com/hasan7n/mlcube@404140c3b6d3e903da6828b741c3168b407737dc#subdirectory=mlcube +mlcube-docker @ git+https://github.com/hasan7n/mlcube@404140c3b6d3e903da6828b741c3168b407737dc#subdirectory=runners/mlcube_docker +mlcube-singularity @ git+https://github.com/hasan7n/mlcube@404140c3b6d3e903da6828b741c3168b407737dc#subdirectory=runners/mlcube_singularity validators==0.18.2 merge-args==0.1.4 synapseclient==2.7.0 diff --git a/cli/tests_setup.sh b/cli/tests_setup.sh index d3865cd2e..5f9c376a1 100644 --- a/cli/tests_setup.sh +++ b/cli/tests_setup.sh @@ -16,7 +16,6 @@ CLEANUP="${CLEANUP:-false}" FRESH="${FRESH:-false}" MEDPERF_STORAGE=~/.medperf SERVER_STORAGE_ID="$(echo $SERVER_URL | cut -d '/' -f 3 | sed -e 's/[.:]/_/g')" -MEDPERF_LOG_STORAGE="$MEDPERF_STORAGE/logs/medperf.log" TIMEOUT="${TIMEOUT:-30}" VERSION_PREFIX="/api/v0" LOGIN_SCRIPT="$(dirname $(realpath "$0"))/auto_login.sh" @@ -47,7 +46,7 @@ checkFailed(){ fi echo $1 echo "medperf log:" - tail "$MEDPERF_LOG_STORAGE" + tail "$MEDPERF_STORAGE/logs/medperf.log" if ${CLEANUP}; then clean fi @@ -62,7 +61,7 @@ fi ########################################################## ########################## Setup ######################### ########################################################## -ASSETS_URL="https://raw.githubusercontent.com/hasan7n/mockcube/036aaa0c156e1ce9b7952f3b4d5275caec30a0a9" +ASSETS_URL="https://raw.githubusercontent.com/hasan7n/mockcube/32294ec0babbcb7773e40075e605ed6f18f6b125" # datasets DSET_A_URL="$ASSETS_URL/assets/datasets/dataset_a.tar.gz" diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index 861179ead..ee54f8e98 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -22,10 +22,7 @@ We will assume the commands' names are `pip` and `python`. Use `pip3` and `pytho #### Docker or Singularity -!!! warning - Singularity is temporarily not supported. - -Make sure you have the latest version of [Docker](https://docs.docker.com/get-docker/){target="\_blank"} or [Singularity 3.10](https://docs.sylabs.io/guides/3.0/user-guide/installation.html){target="\_blank"} installed. +Make sure you have the latest version of [Docker](https://docs.docker.com/get-docker/){target="\_blank"} or [Singularity 3.10](https://docs.sylabs.io/guides/3.10/admin-guide/installation.html){target="\_blank"} installed. To verify docker is installed, run: diff --git a/docs/getting_started/setup.md b/docs/getting_started/setup.md index 01fda250d..a3697b0eb 100644 --- a/docs/getting_started/setup.md +++ b/docs/getting_started/setup.md @@ -56,9 +56,6 @@ medperf profile view #### Choose the Container Runner -!!! warning - Singularity is temporarily not supported. - You can configure the MedPerf client to use either Docker or Singularity. The `local` profile is configured to use Docker. If you want to use MedPerf with Singularity, modify the `local` profile configured parameters by running the following: ```bash