Merge branch 'master' into master

.devcontainer/onCreateCommand.sh

@@ -2,7 +2,7 @@
 
 pip install --upgrade pip
 pip install wheel
-pip install openvino-dev==2022.1.0 # [OPTIONAL] to generate optimized models for inference
+pip install openvino-dev==2023.0.1 # [OPTIONAL] to generate optimized models for inference
 pip install mlcube_docker          # [OPTIONAL] to deploy GaNDLF models as MLCube-compliant Docker containers
 pip install medmnist==2.1.0
 pip install torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cpu

.github/workflows/mlcube-test.yml

@@ -0,0 +1,82 @@
+# This workflow will test gandlf_deploy for model and metrics MLCubes
+
+name: MLCube-Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+
+jobs:
+  test-deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Free space
+      run: |
+        df -h
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        sudo rm -rf "$ANDROID_SDK_ROOT"
+        df -h
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    # Use changed-files-specific action to collect file changes.
+    # The following commented condition applied to a step will run that step only if non-docs files have changed.
+    # It should be applied to all functionality-related steps.
+    # if: steps.changed-files-specific.outputs.only_modified == 'false'
+    - name: Detect and screen file changes
+      id: changed-files-specific
+      uses: tj-actions/changed-files@v34
+      with:
+          files: |
+            .github/*.md
+            .github/ISSUE_TEMPLATE/*.md
+            .github/workflows/devcontainer.yml
+            .github/workflows/docker-image.yml
+            .devcontainer/**
+            docs/**
+            mlcube/**
+            *.md
+            LICENSE
+            Dockerfile-*
+
+    - name: Summarize docs and non-docs modifications
+      run: |
+        echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}"
+        echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}"
+    
+    # This second step is unnecessary but highly recommended because
+    # It will cache database and saves time re-downloading it if database isn't stale.
+    - name: Cache pip
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Set up Python 3.8
+      if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+    - name: Install dependencies and package
+      if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
+      run: |
+        sudo apt-get update
+        sudo apt-get install libvips libvips-tools -y
+        python -m pip install --upgrade pip
+        python -m pip install wheel
+        python -m pip install openvino-dev==2023.0.1 mlcube_docker
+        pip install torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cpu
+        pip install -e .
+    - name: Run mlcube deploy tests
+      working-directory: ./testing
+      if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
+      run: |
+        sh test_deploy.sh

.github/workflows/openfl-test.yml

@@ -11,7 +11,7 @@ on:
 
 
 jobs:
-  build:
+  openfl-test:
 
     runs-on: ubuntu-latest
 
@@ -25,7 +25,7 @@ jobs:
         sudo rm -rf "$ANDROID_SDK_ROOT"
         df -h
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Use changed-files-specific action to collect file changes.
     # The following commented condition applied to a step will run that step only if non-docs files have changed.
@@ -69,6 +69,7 @@ jobs:
     - name: Install dependencies and package
       if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
       run: |
+        sudo apt-get update
         sudo apt-get install libvips libvips-tools -y
         python -m pip install --upgrade pip
         python -m pip install wheel
@@ -90,6 +91,7 @@ jobs:
         cd openfl
         git fetch --tags
         git checkout $(git describe --tags --abbrev=0)
+        sed -i -e 's/protobuf==3.19.6/protobuf/g' setup.py ## this should NOT be there
         pip install -e .
         cd ..
         echo "Copying files to appropriate directories"

.github/workflows/python-test.yml

@@ -11,7 +11,7 @@ on:
 
 
 jobs:
-  build:
+  full-test:
 
     runs-on: ubuntu-latest
 
@@ -25,7 +25,7 @@ jobs:
         sudo rm -rf "$ANDROID_SDK_ROOT"
         df -h
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Use changed-files-specific action to collect file changes.
     # The following commented condition applied to a step will run that step only if non-docs files have changed.
@@ -69,10 +69,11 @@ jobs:
     - name: Install dependencies and package
       if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
       run: |
+        sudo apt-get update
         sudo apt-get install libvips libvips-tools -y
         python -m pip install --upgrade pip
         python -m pip install wheel
-        python -m pip install openvino-dev==2022.1.0 mlcube_docker
+        python -m pip install openvino-dev==2023.0.1 mlcube_docker
         pip install torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cpu
         pip install -e .
     - name: Run generic unit tests

Dockerfile-CPU

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y python3.8 python3-pip libjpeg8-dev zlib
 RUN python3.8 -m pip install --upgrade pip
 # EXPLICITLY install cpu versions of torch/torchvision (not all versions have +cpu modes on PyPI...)
 RUN python3.8 -m pip install torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cpu
-RUN python3.8 -m pip install openvino-dev==2022.1.0 opencv-python-headless mlcube_docker
+RUN python3.8 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker
 
 # Do some dependency installation separately here to make layer caching more efficient
 COPY ./setup.py ./setup.py

Dockerfile-CUDA11.6

@@ -10,7 +10,7 @@ LABEL version=1.0
 RUN apt-get update && apt-get install -y python3.8 python3-pip libjpeg8-dev zlib1g-dev python3-dev libpython3.8-dev libffi-dev libgl1
 RUN python3.8 -m pip install --upgrade pip
 RUN python3.8 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
-RUN python3.8 -m pip install openvino-dev==2022.1.0 opencv-python-headless mlcube_docker
+RUN python3.8 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker
 
 # Do some dependency installation separately here to make layer caching more efficient
 COPY ./setup.py ./setup.py

Dockerfile-ROCm

@@ -8,7 +8,7 @@ LABEL version=1.0
 # The base image contains ROCm, python 3.8 and pytorch already, no need to install those
 RUN python3 -m pip install --upgrade pip
 RUN python3.8 -m pip install torch==1.13.1+rocm5.2 torchvision==0.14.1+rocm5.2 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/rocm5.2
-RUN python3 -m pip install --upgrade pip && python3 -m pip install openvino-dev==2022.1.0 opencv-python-headless mlcube_docker
+RUN python3 -m pip install --upgrade pip && python3 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker
 RUN apt-get update && apt-get install -y libgl1
 
 # Do some dependency installation separately here to make layer caching more efficient

GANDLF/cli/deploy.py

@@ -35,6 +35,7 @@ def run_deployment(
         outputdir (str): The path to the output directory.
         target (str): The target to deploy to.
         mlcube_type (str): Either 'model' or 'metrics'
+        entrypoint_script (str): The path of entrypoint script. Only used for metrics and inference
         configfile (str, Optional): The path to the configuration file. Required for models
         modeldir (str, Optional): The path to the model directory. Required for models
         requires_gpu (str, Optional): Whether the model requires GPU. Required for models
@@ -98,6 +99,7 @@ def deploy_docker_mlcube(
     Args:
         mlcubedir (str): The path to the mlcube directory.
         outputdir (str): The path to the output directory.
+        entrypoint_script (str): The path of entrypoint script. Only used for metrics and inference
         config (str, Optional): The path to the configuration file. Required for models
         modeldir (str, Optional): The path to the model directory. Required for models
         requires_gpu (str, Optional): Whether the model requires GPU. Required for models
@@ -124,7 +126,9 @@ def deploy_docker_mlcube(
     # First grab the existing the mlcube config
     if modeldir is not None:
         # we can use that as an indicator if we are doing model or metrics deployment
-        mlcube_config = get_model_mlcube_config(mlcube_config_file, requires_gpu)
+        mlcube_config = get_model_mlcube_config(
+            mlcube_config_file, requires_gpu, entrypoint_script
+        )
     else:
         mlcube_config = get_metrics_mlcube_config(mlcube_config_file, entrypoint_script)
 
@@ -252,13 +256,14 @@ def get_metrics_mlcube_config(mlcube_config_file, entrypoint_script):
     return mlcube_config
 
 
-def get_model_mlcube_config(mlcube_config_file, requires_gpu):
+def get_model_mlcube_config(mlcube_config_file, requires_gpu, entrypoint_script):
     """
     This function returns the mlcube config for the model.
 
     Args:
         mlcube_config_file (str): Path to mlcube config file.
         requires_gpu (bool): Whether the model requires GPU.
+        entrypoint_script (str): The path of entrypoint script. Only used for infer task
     """
     mlcube_config = None
     with open(mlcube_config_file, "r") as f:
@@ -305,6 +310,13 @@ def get_model_mlcube_config(mlcube_config_file, requires_gpu):
             "--device cpu", "--device cuda"
         )
 
+    if entrypoint_script:
+        # modify the infer entrypoint to run a custom script.
+        device = "cuda" if requires_gpu else "cpu"
+        mlcube_config["tasks"]["infer"][
+            "entrypoint"
+        ] = f"python3.8 /entrypoint.py --device {device}"
+
     return mlcube_config
     # Duplicate training task into one from reset (must be explicit) and one that resumes with new data
     # In either case, the embedded model will not change persistently.

GANDLF/cli/generate_metrics.py

@@ -1,3 +1,4 @@
+import sys
 import yaml
 from pprint import pprint
 import pandas as pd
@@ -193,6 +194,30 @@ def __fix_2d_tensor(input_tensor):
             else:
                 return input_tensor
 
+        def __percentile_clip(input_tensor, reference_tensor=None, p_min=0.5, p_max=99.5, strictlyPositive=True):
+            """Normalizes a tensor based on percentiles. Clips values below and above the percentile.
+            Percentiles for normalization can come from another tensor.
+
+            Args:
+                input_tensor (torch.Tensor): Tensor to be normalized based on the data from the reference_tensor.
+                    If reference_tensor is None, the percentiles from this tensor will be used.
+                reference_tensor (torch.Tensor, optional): The tensor used for obtaining the percentiles.
+                p_min (float, optional): Lower end percentile. Defaults to 0.5.
+                p_max (float, optional): Upper end percentile. Defaults to 99.5.
+                strictlyPositive (bool, optional): Ensures that really all values are above 0 before normalization. Defaults to True.
+
+            Returns:
+                torch.Tensor: The input_tensor normalized based on the percentiles of the reference tensor.
+            """
+            reference_tensor = input_tensor if reference_tensor is None else reference_tensor
+            v_min, v_max = np.percentile(reference_tensor, [p_min,p_max]) #get p_min percentile and p_max percentile
+
+            # set lower bound to be 0 if strictlyPositive is enabled
+            v_min = max(v_min, 0.0) if strictlyPositive else v_min
+            output_tensor = np.clip(input_tensor,v_min,v_max) #clip values to percentiles from reference_tensor
+            output_tensor = (output_tensor - v_min)/(v_max-v_min) #normalizes values to [0;1]
+            return output_tensor
+
         for _, row in tqdm(input_df.iterrows(), total=input_df.shape[0]):
             current_subject_id = row[headers["subjectid"]]
             overall_stats_dict[current_subject_id] = {}
@@ -219,27 +244,29 @@ def __fix_2d_tensor(input_tensor):
             # Normalize to [0;1] based on GT (otherwise MSE will depend on the image intensity range)
             normalize = parameters.get("normalize", True)
             if normalize:
-                v_max = gt_image_infill.max()
-                output_infill /= v_max
-                gt_image_infill /= v_max
+                reference_tensor = target_image * ~mask #use all the tissue that is not masked for normalization
+                gt_image_infill = __percentile_clip(gt_image_infill, reference_tensor=reference_tensor, p_min=0.5, p_max=99.5, strictlyPositive=True)
+                output_infill = __percentile_clip(output_infill, reference_tensor=reference_tensor, p_min=0.5, p_max=99.5, strictlyPositive=True)
 
             overall_stats_dict[current_subject_id][
                 "ssim"
             ] = structural_similarity_index(gt_image_infill, output_infill, mask).item()
 
             # ncc metrics
-            overall_stats_dict[current_subject_id]["ncc_mean"] = ncc_mean(
-                gt_image_infill, output_infill
-            )
-            overall_stats_dict[current_subject_id]["ncc_std"] = ncc_std(
-                gt_image_infill, output_infill
-            )
-            overall_stats_dict[current_subject_id]["ncc_max"] = ncc_max(
-                gt_image_infill, output_infill
-            )
-            overall_stats_dict[current_subject_id]["ncc_min"] = ncc_min(
-                gt_image_infill, output_infill
-            )
+            compute_ncc = parameters.get("compute_ncc", True)
+            if compute_ncc:
+                overall_stats_dict[current_subject_id]["ncc_mean"] = ncc_mean(
+                    gt_image_infill, output_infill
+                )
+                overall_stats_dict[current_subject_id]["ncc_std"] = ncc_std(
+                    gt_image_infill, output_infill
+                )
+                overall_stats_dict[current_subject_id]["ncc_max"] = ncc_max(
+                    gt_image_infill, output_infill
+                )
+                overall_stats_dict[current_subject_id]["ncc_min"] = ncc_min(
+                    gt_image_infill, output_infill
+                )
 
             # only voxels that are to be inferred (-> flat array)
             # these are required for mse, psnr, etc.
@@ -258,11 +285,34 @@ def __fix_2d_tensor(input_tensor):
                 gt_image_infill, output_infill
             ).item()
 
-            # PSNR - similar to pytorch PeakSignalNoiseRatio until 4 digits after decimal point
+            # torchmetrics PSNR using "max"
             overall_stats_dict[current_subject_id]["psnr"] = peak_signal_noise_ratio(
                 gt_image_infill, output_infill
             ).item()
 
+            # same as above but with epsilon for robustness
+            overall_stats_dict[current_subject_id][
+                "psnr_eps"
+            ] = peak_signal_noise_ratio(
+                gt_image_infill, output_infill, epsilon=sys.float_info.epsilon
+            ).item()
+
+            # only use fix data range to [0;1] if the data was normalized before
+            if normalize:
+                # torchmetrics PSNR but with fixed data range of 0 to 1
+                overall_stats_dict[current_subject_id][
+                    "psnr_01"
+                ] = peak_signal_noise_ratio(
+                    gt_image_infill, output_infill, data_range=(0,1)
+                ).item()
+
+                # same as above but with epsilon for robustness
+                overall_stats_dict[current_subject_id][
+                    "psnr_01_eps"
+                ] = peak_signal_noise_ratio(
+                    gt_image_infill, output_infill, data_range=(0,1), epsilon=sys.float_info.epsilon
+                ).item()
+
     pprint(overall_stats_dict)
     if outputfile is not None:
         with open(outputfile, "w") as outfile: