From 37b59b11b98cc9c7cbb3c1456e0722815e03e3b4 Mon Sep 17 00:00:00 2001 From: Kevin Ta <116312994+kta-intel@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:23:38 -0400 Subject: [PATCH] [Fix] Update requirement.txt to install latest torch version in torch workspaces (#1000) * remove unused pt_cnn.py from torch_cnn_histology Signed-off-by: kta-intel * update validate to validate_task in unet_kvasir plan.yaml Signed-off-by: kta-intel * update package versions Signed-off-by: kta-intel --------- Signed-off-by: kta-intel --- .../torch_cnn_histology/requirements.txt | 4 +- .../torch_cnn_histology/src/pt_cnn.py | 165 ------------------ .../requirements.txt | 6 +- .../torch_cnn_mnist/requirements.txt | 4 +- .../requirements.txt | 4 +- .../torch_cnn_mnist_fed_eval/requirements.txt | 4 +- .../requirements.txt | 4 +- .../torch_unet_kvasir/plan/plan.yaml | 4 +- .../torch_unet_kvasir/requirements.txt | 6 +- .../plan/plan.yaml | 4 +- .../requirements.txt | 10 +- 11 files changed, 26 insertions(+), 189 deletions(-) delete mode 100644 openfl-workspace/torch_cnn_histology/src/pt_cnn.py diff --git a/openfl-workspace/torch_cnn_histology/requirements.txt b/openfl-workspace/torch_cnn_histology/requirements.txt index c44f6b0daa..8e908c595d 100644 --- a/openfl-workspace/torch_cnn_histology/requirements.txt +++ b/openfl-workspace/torch_cnn_histology/requirements.txt @@ -1,2 +1,2 @@ -torchvision==0.18.0 -f https://download.pytorch.org/whl/torch_stable.html -torch==2.3.0 -f https://download.pytorch.org/whl/torch_stable.html \ No newline at end of file +torchvision==0.18.1 +torch==2.3.1 \ No newline at end of file diff --git a/openfl-workspace/torch_cnn_histology/src/pt_cnn.py b/openfl-workspace/torch_cnn_histology/src/pt_cnn.py deleted file mode 100644 index cfa7122243..0000000000 --- a/openfl-workspace/torch_cnn_histology/src/pt_cnn.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""You may copy this file as the starting point of your own model.""" -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -import tqdm - -from openfl.federated import PyTorchTaskRunner -from openfl.utilities import TensorKey - - -def cross_entropy(output, target): - """Calculate Cross-entropy loss.""" - return F.cross_entropy(input=output, target=target) - - -class PyTorchCNN(PyTorchTaskRunner): - """Simple CNN for classification.""" - - def __init__(self, **kwargs): - """Initialize. - - Args: - **kwargs: Additional arguments to pass to the function - """ - super().__init__(loss_fn=cross_entropy, **kwargs) - - torch.manual_seed(0) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - - self.num_classes = self.data_loader.num_classes - self.init_network(device=self.device, **kwargs) - self._init_optimizer(lr=kwargs.get("lr")) - self.initialize_tensorkeys_for_functions() - - def _init_optimizer(self, lr): - """Initialize the optimizer.""" - self.optimizer = optim.Adam(self.parameters(), lr=float(lr or 1e-3)) - - def init_network(self, device, print_model=True, **kwargs): - """Create the network (model). - - Args: - device: The hardware device to use for training - print_model (bool): Print the model topology (Default=True) - **kwargs: Additional arguments to pass to the function - - """ - channel = self.data_loader.get_feature_shape()[0] # (channel, dim1, dim2) - conv_kwargs = {"kernel_size": 3, "stride": 1, "padding": 1} - self.conv1 = nn.Conv2d(channel, 16, **conv_kwargs) - self.conv2 = nn.Conv2d(16, 32, **conv_kwargs) - self.conv3 = nn.Conv2d(32, 64, **conv_kwargs) - self.conv4 = nn.Conv2d(64, 128, **conv_kwargs) - self.conv5 = nn.Conv2d(128 + 32, 256, **conv_kwargs) - self.conv6 = nn.Conv2d(256, 512, **conv_kwargs) - self.conv7 = nn.Conv2d(512 + 128 + 32, 256, **conv_kwargs) - self.conv8 = nn.Conv2d(256, 512, **conv_kwargs) - self.fc1 = nn.Linear(1184 * 9 * 9, 128) - self.fc2 = nn.Linear(128, 8) - if print_model: - print(self) - self.to(device) - - def forward(self, x): - """Forward pass of the model. - - Args: - x: Data input to the model for the forward pass - """ - x = F.relu(self.conv1(x)) - x = F.relu(self.conv2(x)) - maxpool = F.max_pool2d(x, 2, 2) - - x = F.relu(self.conv3(maxpool)) - x = F.relu(self.conv4(x)) - concat = torch.cat([maxpool, x], dim=1) - maxpool = F.max_pool2d(concat, 2, 2) - - x = F.relu(self.conv5(maxpool)) - x = F.relu(self.conv6(x)) - concat = torch.cat([maxpool, x], dim=1) - maxpool = F.max_pool2d(concat, 2, 2) - - x = F.relu(self.conv7(maxpool)) - x = F.relu(self.conv8(x)) - concat = torch.cat([maxpool, x], dim=1) - maxpool = F.max_pool2d(concat, 2, 2) - - x = maxpool.flatten(start_dim=1) - x = F.dropout(self.fc1(x), p=0.5) - x = self.fc2(x) - return x - - def validate_task( - self, col_name, round_num, input_tensor_dict, use_tqdm=False, **kwargs - ): - """Validate. - - Run validation of the model on the local data. - - Args: - col_name: Name of the collaborator - round_num: What round is it - input_tensor_dict: Required input tensors (for model) - use_tqdm (bool): Use tqdm to print a progress - bar (Default=True) - - Returns: - global_output_dict: Tensors to send back to the aggregator - local_output_dict: Tensors to maintain in the local TensorDB - - """ - self.rebuild_model(round_num, input_tensor_dict, validation=True) - self.eval() - val_score = 0 - total_samples = 0 - - loader = self.data_loader.get_valid_loader() - if use_tqdm: - loader = tqdm.tqdm(loader, desc="validate") - - with torch.no_grad(): - for data, target in loader: - samples = target.shape[0] - total_samples += samples - data, target = ( - torch.tensor(data).to(self.device), - torch.tensor(target).to(self.device), - ) - output = self(data) - # get the index of the max log-probability - pred = output.argmax(dim=1) - val_score += pred.eq(target).sum().cpu().numpy() - - origin = col_name - suffix = "validate" - if kwargs["apply"] == "local": - suffix += "_local" - else: - suffix += "_agg" - tags = ("metric", suffix) - # TODO figure out a better way to pass in metric for - # this pytorch validate function - output_tensor_dict = { - TensorKey("acc", origin, round_num, True, tags): np.array( - val_score / total_samples - ) - } - - # empty list represents metrics that should only be stored locally - return output_tensor_dict, {} - - def reset_opt_vars(self): - """Reset optimizer variables. - - Resets the optimizer state variables. - - """ - self._init_optimizer(lr=self.optimizer.defaults.get("lr")) diff --git a/openfl-workspace/torch_cnn_histology_gramine_ready/requirements.txt b/openfl-workspace/torch_cnn_histology_gramine_ready/requirements.txt index 4c955d10b1..75593eb9c8 100644 --- a/openfl-workspace/torch_cnn_histology_gramine_ready/requirements.txt +++ b/openfl-workspace/torch_cnn_histology_gramine_ready/requirements.txt @@ -1,3 +1,3 @@ --f https://download.pytorch.org/whl/cpu/torch_stable.html -torch==1.13.1+cpu -torchvision==0.14.1+cpu \ No newline at end of file +--index-url https://download.pytorch.org/whl/cpu +torch==2.3.1 +torchvision==0.18.1 \ No newline at end of file diff --git a/openfl-workspace/torch_cnn_mnist/requirements.txt b/openfl-workspace/torch_cnn_mnist/requirements.txt index 16b349007c..f93aef46e7 100644 --- a/openfl-workspace/torch_cnn_mnist/requirements.txt +++ b/openfl-workspace/torch_cnn_mnist/requirements.txt @@ -1,4 +1,4 @@ -torch==1.13.1 -torchvision==0.14.1 +torch==2.3.1 +torchvision==0.18.1 tensorboard wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt b/openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt index 16b349007c..f93aef46e7 100644 --- a/openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt +++ b/openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt @@ -1,4 +1,4 @@ -torch==1.13.1 -torchvision==0.14.1 +torch==2.3.1 +torchvision==0.18.1 tensorboard wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt b/openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt index 16b349007c..f93aef46e7 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt @@ -1,4 +1,4 @@ -torch==1.13.1 -torchvision==0.14.1 +torch==2.3.1 +torchvision==0.18.1 tensorboard wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt b/openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt index b2ea9e3330..5db2ffa017 100644 --- a/openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt +++ b/openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt @@ -1,5 +1,5 @@ -torch==1.13.1 -torchvision==0.14.1 +torch==2.3.1 +torchvision==0.18.1 tensorboard numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/openfl-workspace/torch_unet_kvasir/plan/plan.yaml b/openfl-workspace/torch_unet_kvasir/plan/plan.yaml index 18c24498d3..b37a8a7d91 100644 --- a/openfl-workspace/torch_unet_kvasir/plan/plan.yaml +++ b/openfl-workspace/torch_unet_kvasir/plan/plan.yaml @@ -43,14 +43,14 @@ assigner : tasks : defaults : plan/defaults/tasks_torch.yaml aggregated_model_validation: - function : validate + function : validate_task kwargs : apply : global metrics : - dice_coef locally_tuned_model_validation: - function : validate + function : validate_task kwargs : apply : local metrics : diff --git a/openfl-workspace/torch_unet_kvasir/requirements.txt b/openfl-workspace/torch_unet_kvasir/requirements.txt index 6ef48deefe..e9bd2a7b71 100644 --- a/openfl-workspace/torch_unet_kvasir/requirements.txt +++ b/openfl-workspace/torch_unet_kvasir/requirements.txt @@ -1,3 +1,3 @@ -torch==1.13.1 -torchvision==0.14.1 -scikit-image==0.19.3 +torch==2.3.1 +torchvision==0.18.1 +scikit-image==0.24.0 diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/plan/plan.yaml b/openfl-workspace/torch_unet_kvasir_gramine_ready/plan/plan.yaml index 4f35c84bed..c41bfbcffd 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/plan/plan.yaml +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/plan/plan.yaml @@ -44,14 +44,14 @@ assigner : tasks : defaults : plan/defaults/tasks_torch.yaml aggregated_model_validation: - function : validate + function : validate_task kwargs : apply : global metrics : - dice_coef locally_tuned_model_validation: - function : validate + function : validate_task kwargs : apply : local metrics : diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/requirements.txt b/openfl-workspace/torch_unet_kvasir_gramine_ready/requirements.txt index 2523ff5e98..bca8e7792b 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/requirements.txt +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/requirements.txt @@ -1,4 +1,6 @@ --f https://download.pytorch.org/whl/cpu/torch_stable.html -torch==1.13.1+cpu -torchvision==0.14.1+cpu -scikit-image==0.19.3 +scikit-image==0.24.0 +pillow==10.4.0 + +--extra-index-url https://download.pytorch.org/whl/cpu +torch==2.3.1 +torchvision==0.18.1 \ No newline at end of file