Make storage configurable, refactor storage and config (#516)

* various storage and config modifications

* use configurable again in profile.create

* fix failing tests

* change serverpath convention, add test, add ls

* edit integration tests

* fix problem of broken links after moving storage

* fix cli integraton tests

* fix test mistake

cli/cli_tests.sh

@@ -181,6 +181,18 @@ DSET_A_UID=$(medperf dataset ls | grep dataset_a | tr -s ' ' | cut -d ' ' -f 1)
 
 echo "\n"
 
+
+##########################################################
+echo "====================================="
+echo "Moving storage to some other location"
+echo "====================================="
+medperf storage move -t /tmp/some_folder
+checkFailed "moving storage failed"
+MEDPERF_STORAGE="/tmp/some_folder/.medperf"
+##########################################################
+
+echo "\n"
+
 ##########################################################
 echo "====================================="
 echo "Running data association step"
@@ -335,7 +347,7 @@ echo "\n"
 echo "====================================================================="
 echo "Run failing cube with ignore errors after deleting predictions folder"
 echo "====================================================================="
-rm -rf $MEDPERF_SUBSTORAGE/predictions/model-fail/$DSET_A_GENUID
+rm -rf $MEDPERF_STORAGE/predictions/$SERVER_STORAGE_ID/model-fail/$DSET_A_GENUID
 medperf run -b $BMK_UID -d $DSET_A_UID -m $FAILING_MODEL_UID -y --ignore-model-errors
 checkFailed "Failing mlcube run with ignore errors failed"
 ##########################################################

cli/medperf/__main__.py

@@ -1,120 +1,4 @@
-import sys
-import typer
-import logging
-import logging.handlers
+from medperf.init import initialize
 
-from medperf import __version__
-import medperf.config as config
-from medperf.ui.factory import UIFactory
-from medperf.decorators import clean_except, configurable
-from medperf.comms.factory import CommsFactory
-import medperf.comms.auth as auth_module
-import medperf.commands.result.result as result
-from medperf.commands.result.create import BenchmarkExecution
-from medperf.commands.result.submit import ResultSubmission
-import medperf.commands.mlcube.mlcube as mlcube
-import medperf.commands.dataset.dataset as dataset
-import medperf.commands.auth.auth as auth
-import medperf.commands.benchmark.benchmark as benchmark
-import medperf.commands.profile as profile
-from medperf.utils import (
-    set_custom_config,
-    set_unique_tmp_config,
-    init_storage,
-    setup_logging,
-)
-import medperf.commands.association.association as association
-import medperf.commands.compatibility_test.compatibility_test as compatibility_test
-
-
-app = typer.Typer()
-app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes")
-app.add_typer(result.app, name="result", help="Manage results")
-app.add_typer(dataset.app, name="dataset", help="Manage datasets")
-app.add_typer(benchmark.app, name="benchmark", help="Manage benchmarks")
-app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes")
-app.add_typer(result.app, name="result", help="Manage results")
-app.add_typer(association.app, name="association", help="Manage associations")
-app.add_typer(profile.app, name="profile", help="Manage profiles")
-app.add_typer(compatibility_test.app, name="test", help="Manage compatibility tests")
-app.add_typer(auth.app, name="auth", help="Authentication")
-
-
-@app.command("run")
-@clean_except
-def execute(
-    benchmark_uid: int = typer.Option(
-        ..., "--benchmark", "-b", help="UID of the desired benchmark"
-    ),
-    data_uid: int = typer.Option(
-        ..., "--data_uid", "-d", help="Registered Dataset UID"
-    ),
-    model_uid: int = typer.Option(
-        ..., "--model_uid", "-m", help="UID of model to execute"
-    ),
-    approval: bool = typer.Option(False, "-y", help="Skip approval step"),
-    ignore_model_errors: bool = typer.Option(
-        False,
-        "--ignore-model-errors",
-        help="Ignore failing model cubes, allowing for possibly submitting partial results",
-    ),
-    no_cache: bool = typer.Option(
-        False,
-        "--no-cache",
-        help="Ignore existing results. The experiment then will be rerun",
-    ),
-):
-    """Runs the benchmark execution step for a given benchmark, prepared dataset and model"""
-    result = BenchmarkExecution.run(
-        benchmark_uid,
-        data_uid,
-        [model_uid],
-        ignore_model_errors=ignore_model_errors,
-        no_cache=no_cache,
-    )[0]
-    if result.id:  # TODO: use result.is_registered once PR #338 is merged
-        config.ui.print(  # TODO: msg should be colored yellow
-            """An existing registered result for the requested execution has been\n
-            found. If you wish to submit a new result for the same execution,\n
-            please run the command again with the --no-cache option.\n"""
-        )
-    else:
-        ResultSubmission.run(result.generated_uid, approved=approval)
-    config.ui.print("✅ Done!")
-
-
-def version_callback(value: bool):
-    if value:
-        print(f"MedPerf version {__version__}")
-        raise typer.Exit()
-
-
-@app.callback()
-@configurable
-def main(
-    ctx: typer.Context,
-    version: bool = typer.Option(
-        None, "--version", callback=version_callback, is_eager=True
-    ),
-):
-    # Set inline parameters
-    inline_args = ctx.params
-    set_custom_config(inline_args)
-    set_unique_tmp_config()
-
-    init_storage()
-    log = config.loglevel.upper()
-    log_lvl = getattr(logging, log)
-    setup_logging(log_lvl)
-    logging.info(f"Running MedPerf v{__version__} on {log_lvl} logging level")
-    logging.info(f"Executed command: {' '.join(sys.argv[1:])}")
-
-    config.ui = UIFactory.create_ui(config.ui)
-    config.comms = CommsFactory.create_comms(config.comms, config.server)
-    auth_class = getattr(auth_module, config.auth_class)
-    config.auth = auth_class()
-    config.ui.print(f"MedPerf {__version__}")
-
-
-if __name__ == "__main__":
-    app()
+initialize()
+from medperf.cli import app  # noqa

cli/medperf/account_management/account_management.py

@@ -1,5 +1,5 @@
 from .token_storage import TokenStore
-from medperf.utils import read_config, write_config
+from medperf.config_management import read_config, write_config
 from medperf import config
 from medperf.exceptions import MedperfException
 

cli/medperf/account_management/token_storage/filesystem.py

@@ -1,13 +1,13 @@
 import os
 import base64
-from medperf.utils import base_storage_path, remove_path
+from medperf.utils import remove_path
 from medperf import config
 
 
 class FilesystemTokenStore:
     def __init__(self):
-        self.creds_folder = base_storage_path(config.creds_folder)
-        os.makedirs(self.creds_folder, exist_ok=True)
+        self.creds_folder = config.creds_folder
+        os.makedirs(self.creds_folder, mode=0o700, exist_ok=True)
 
     def __get_paths(self, account_id):
         # Base64 encoding is used just to avoid facing a filesystem that doesn't support
@@ -16,7 +16,7 @@ def __get_paths(self, account_id):
             "utf-8"
         )
         account_folder = os.path.join(self.creds_folder, account_id_encoded)
-        os.makedirs(account_folder, exist_ok=True)
+        os.makedirs(account_folder, mode=0o700, exist_ok=True)
 
         access_token_file = os.path.join(account_folder, config.access_token_storage_id)
         refresh_token_file = os.path.join(

cli/medperf/cli.py

@@ -0,0 +1,105 @@
+import sys
+import typer
+import logging
+import logging.handlers
+
+from medperf import __version__
+import medperf.config as config
+from medperf.decorators import clean_except, add_inline_parameters
+import medperf.commands.result.result as result
+from medperf.commands.result.create import BenchmarkExecution
+from medperf.commands.result.submit import ResultSubmission
+import medperf.commands.mlcube.mlcube as mlcube
+import medperf.commands.dataset.dataset as dataset
+import medperf.commands.auth.auth as auth
+import medperf.commands.benchmark.benchmark as benchmark
+import medperf.commands.profile as profile
+import medperf.commands.association.association as association
+import medperf.commands.compatibility_test.compatibility_test as compatibility_test
+import medperf.commands.storage as storage
+
+app = typer.Typer()
+app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes")
+app.add_typer(result.app, name="result", help="Manage results")
+app.add_typer(dataset.app, name="dataset", help="Manage datasets")
+app.add_typer(benchmark.app, name="benchmark", help="Manage benchmarks")
+app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes")
+app.add_typer(result.app, name="result", help="Manage results")
+app.add_typer(association.app, name="association", help="Manage associations")
+app.add_typer(profile.app, name="profile", help="Manage profiles")
+app.add_typer(compatibility_test.app, name="test", help="Manage compatibility tests")
+app.add_typer(auth.app, name="auth", help="Authentication")
+app.add_typer(storage.app, name="storage", help="Storage management")
+
+
+@app.command("run")
+@clean_except
+def execute(
+    benchmark_uid: int = typer.Option(
+        ..., "--benchmark", "-b", help="UID of the desired benchmark"
+    ),
+    data_uid: int = typer.Option(
+        ..., "--data_uid", "-d", help="Registered Dataset UID"
+    ),
+    model_uid: int = typer.Option(
+        ..., "--model_uid", "-m", help="UID of model to execute"
+    ),
+    approval: bool = typer.Option(False, "-y", help="Skip approval step"),
+    ignore_model_errors: bool = typer.Option(
+        False,
+        "--ignore-model-errors",
+        help="Ignore failing model cubes, allowing for possibly submitting partial results",
+    ),
+    no_cache: bool = typer.Option(
+        False,
+        "--no-cache",
+        help="Ignore existing results. The experiment then will be rerun",
+    ),
+):
+    """Runs the benchmark execution step for a given benchmark, prepared dataset and model"""
+    result = BenchmarkExecution.run(
+        benchmark_uid,
+        data_uid,
+        [model_uid],
+        ignore_model_errors=ignore_model_errors,
+        no_cache=no_cache,
+    )[0]
+    if result.id:  # TODO: use result.is_registered once PR #338 is merged
+        config.ui.print(  # TODO: msg should be colored yellow
+            """An existing registered result for the requested execution has been\n
+            found. If you wish to submit a new result for the same execution,\n
+            please run the command again with the --no-cache option.\n"""
+        )
+    else:
+        ResultSubmission.run(result.generated_uid, approved=approval)
+    config.ui.print("✅ Done!")
+
+
+def version_callback(value: bool):
+    if value:
+        print(f"MedPerf version {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+@add_inline_parameters
+def main(
+    ctx: typer.Context,
+    version: bool = typer.Option(
+        None, "--version", callback=version_callback, is_eager=True
+    ),
+):
+    # Set inline parameters
+    inline_args = ctx.params
+    for param in inline_args:
+        setattr(config, param, inline_args[param])
+
+    # Update logging level according to the passed inline params
+    loglevel = config.loglevel.upper()
+    logging.getLogger().setLevel(loglevel)
+    logging.getLogger("requests").setLevel(loglevel)
+
+    logging.info(f"Running MedPerf v{__version__} on {loglevel} logging level")
+    logging.info(f"Executed command: {' '.join(sys.argv[1:])}")
+
+    config.ui.print(f"MedPerf {__version__}")

cli/medperf/commands/compatibility_test/utils.py

@@ -1,4 +1,4 @@
-from medperf.utils import storage_path, get_folders_hash
+from medperf.utils import get_folders_hash
 from medperf.exceptions import InvalidArgumentError, InvalidEntityError
 
 from medperf.comms.entity_resources import resources
@@ -35,8 +35,8 @@ def download_demo_data(dset_url, dset_hash):
 
 def prepare_local_cube(path):
     temp_uid = get_folders_hash([path])
-    cubes_storage = storage_path(config.cubes_storage)
-    dst = os.path.join(cubes_storage, temp_uid)
+    cubes_folder = config.cubes_folder
+    dst = os.path.join(cubes_folder, temp_uid)
     os.symlink(path, dst)
     logging.info(f"local cube will be linked to path: {dst}")
     config.tmp_paths.append(dst)

cli/medperf/commands/dataset/create.py

@@ -6,12 +6,7 @@
 import medperf.config as config
 from medperf.entities.cube import Cube
 from medperf.entities.benchmark import Benchmark
-from medperf.utils import (
-    remove_path,
-    generate_tmp_path,
-    get_folders_hash,
-    storage_path,
-)
+from medperf.utils import remove_path, generate_tmp_path, get_folders_hash
 from medperf.exceptions import InvalidArgumentError
 import yaml
 
@@ -170,7 +165,7 @@ def to_permanent_path(self) -> str:
         """Renames the temporary data folder to permanent one using the hash of
         the registration file
         """
-        new_path = os.path.join(storage_path(config.data_storage), self.generated_uid)
+        new_path = os.path.join(config.datasets_folder, self.generated_uid)
         remove_path(new_path)
         os.rename(self.out_path, new_path)
         self.out_path = new_path

cli/medperf/commands/execution.py

@@ -3,7 +3,7 @@
 
 from medperf.entities.cube import Cube
 from medperf.entities.dataset import Dataset
-from medperf.utils import generate_tmp_path, storage_path
+from medperf.utils import generate_tmp_path
 import medperf.config as config
 from medperf.exceptions import ExecutionError
 import yaml
@@ -52,9 +52,8 @@ def __setup_logs_path(self):
         data_hash = self.dataset.generated_uid
 
         logs_path = os.path.join(
-            config.experiments_logs_storage, str(model_uid), str(data_hash)
+            config.experiments_logs_folder, str(model_uid), str(data_hash)
         )
-        logs_path = storage_path(logs_path)
         os.makedirs(logs_path, exist_ok=True)
         model_logs_path = os.path.join(logs_path, "model.log")
         metrics_logs_path = os.path.join(logs_path, f"metrics_{eval_uid}.log")
@@ -64,9 +63,8 @@ def __setup_predictions_path(self):
         model_uid = self.model.generated_uid
         data_hash = self.dataset.generated_uid
         preds_path = os.path.join(
-            config.predictions_storage, str(model_uid), str(data_hash)
+            config.predictions_folder, str(model_uid), str(data_hash)
         )
-        preds_path = storage_path(preds_path)
         if os.path.exists(preds_path):
             msg = f"Found existing predictions for model {self.model.id} on dataset "
             msg += f"{self.dataset.id} at {preds_path}. Consider deleting this "

cli/medperf/commands/profile.py

@@ -2,7 +2,8 @@
 
 from medperf import config
 from medperf.decorators import configurable, clean_except
-from medperf.utils import dict_pretty_print, read_config, write_config
+from medperf.utils import dict_pretty_print
+from medperf.config_management import read_config, write_config
 from medperf.exceptions import InvalidArgumentError
 
 app = typer.Typer()

cli/medperf/commands/result/submit.py

@@ -1,7 +1,7 @@
 import os
 
 from medperf.exceptions import CleanExit
-from medperf.utils import remove_path, dict_pretty_print, approval_prompt, storage_path
+from medperf.utils import remove_path, dict_pretty_print, approval_prompt
 from medperf.entities.result import Result
 from medperf.enums import Status
 from medperf import config
@@ -51,7 +51,7 @@ def to_permanent_path(self, result_dict: dict):
             result_dict (dict): updated results dictionary
         """
         result = Result(**result_dict)
-        result_storage = storage_path(config.results_storage)
+        result_storage = config.results_folder
         old_res_loc = os.path.join(result_storage, result.generated_uid)
         new_res_loc = result.path
         remove_path(new_res_loc)