From 82dfe070e15f5688ccedff6ecbfc9be77ff82eb3 Mon Sep 17 00:00:00 2001
From: Jing Lin <82669431+linjing-lab@users.noreply.github.com>
Date: Sat, 28 Oct 2023 18:01:58 +0800
Subject: [PATCH] support set_freeze module
---
released_box/README.md | 8 ++++++-
released_box/perming/__init__.py | 2 +-
released_box/perming/_utils.py | 14 +++++++++++
.../tests/Multi-classification Task.ipynb | 23 +++++++++++--------
4 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/released_box/README.md b/released_box/README.md
index fa5ba99..a6db00c 100644
--- a/released_box/README.md
+++ b/released_box/README.md
@@ -35,6 +35,9 @@ data_loader:
- worker_set: *Dict[str, int]={'train': 8, 'test': 2, 'val': 1}*, manually set by users need.
- random_seed: *Optional[int]=None*, manually set any int value by users to fixed sequence.
+set_freeze:
+- require_grad: *Dict[int, bool]*, manually set freezed layers by given serial numbers according to `self.model`. (if users set require_grad with `{0: False}`, it means freeze the first layer of `self.model`.
+
train_val:
- num_epochs: *int=2*, define numbers of epochs in main training cycle. any int value > 0.
- interval: *int=100*, define console print length of whole epochs by interval. any int value > 0.
@@ -52,7 +55,6 @@ save or load:
- con: *bool=True*, configure whether to print model.state_dict(). False or True.
- dir: *dir='./model'*, configure model path that *save to* or *load from*. correct path defined by users.
-
## general model
|GENERAL_BOX(Box)|Parameters|Meaning|
@@ -74,6 +76,7 @@ save or load:
|`__init__`|input_: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='MSELoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Regressier Based on Basic Information of the Regression Dataset Obtained through Data Preprocessing and Feature Engineering with `num_classes=1`.|
|print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.|
|data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Regression Dataset with Numpy format into `torch.utils.data.DataLoader`.|
+|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. |
|train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.|
|test|/|Test Module Only Show with Loss at 3 Stages: Train, Test, Val|
|save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.|
@@ -86,6 +89,7 @@ save or load:
|`__init__`|input_: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='BCELoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Classifier Based on Basic Information of the Classification Dataset Obtained through Data Preprocessing and Feature Engineering with `num_classes=2`.|
|print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.|
|data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Binary-classification Dataset with Numpy format into `torch.utils.data.DataLoader`.|
+|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. |
|train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.|
|test|sort_by: str='accuracy'
sort_state: bool=True|Test Module con with Correct Class and Loss at 3 Stages: Train, Test, Val|
|save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.|
@@ -98,6 +102,7 @@ save or load:
|`__init__`|input_: int
num_classes: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='CrossEntropyLoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Classifier Based on Basic Information of the Classification Dataset Obtained through Data Preprocessing and Feature Engineering with `num_classes>2`.|
|print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.|
|data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Multi-classification Dataset with Numpy format into `torch.utils.data.DataLoader`.|
+|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. |
|train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.|
|test|sort_by: str='accuracy'
sort_state: bool=True|Sort Returned Test Result about Correct Classes with `sort_by` and `sort_state` Which Only Appears in Classification.|
|save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.|
@@ -110,6 +115,7 @@ save or load:
|`__init__`|input_: int
num_outputs: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='MultiLabelSoftMarginLoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Ranker Based on Basic Information of the Classification Dataset Obtained through Data Preprocessing and Feature Engineering with (n_samples, n_outputs).|
|print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.|
|data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Multi-outputs Dataset with Numpy format into `torch.utils.data.DataLoader`.|
+|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. |
|train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.|
|test|sort_by: str='accuracy'
sort_state: bool=True|Sort Returned Test Result about Correct Classes with `sort_by` and `sort_state` Which Only Appears in Classification.|
|save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.|
diff --git a/released_box/perming/__init__.py b/released_box/perming/__init__.py
index 0bf12ec..5cfd496 100644
--- a/released_box/perming/__init__.py
+++ b/released_box/perming/__init__.py
@@ -27,4 +27,4 @@
'Multi-outputs': Ranker
}
-__version__ = '1.7.1'
\ No newline at end of file
+__version__ = '1.8.0'
\ No newline at end of file
diff --git a/released_box/perming/_utils.py b/released_box/perming/_utils.py
index d1f6110..ab88d1c 100644
--- a/released_box/perming/_utils.py
+++ b/released_box/perming/_utils.py
@@ -200,6 +200,20 @@ def data_loader(self,
self.test_loader = torch.utils.data.DataLoader(TabularDataset(test_['features'], test_['target'], roc), batch_size=self.batch_size, shuffle=True, num_workers=worker_set['test'])
self.val_loader = torch.utils.data.DataLoader(TabularDataset(val_['features'], val_['target'], roc), batch_size=self.batch_size, shuffle=False, num_workers=worker_set['val'])
+ def set_freeze(self, require_grad: Dict[int, bool]):
+ '''
+ Freeze Network Layers of Trained Model by given `require_grad=False`.
+ :param require_grad: Dict[int, bool], whether to freeze grad on serial number of model layers.
+ '''
+ operat: bool = False
+ for name, param in self.model.named_parameters():
+ serial = int(name.strip('mlp.Linear.weight.bias'))
+ if serial in require_grad:
+ param.requires_grad = require_grad[serial]
+ operat: bool = True
+ if not operat:
+ raise ValueError('Please set require_grad states according to `self.model`.')
+
def train_val(self,
num_epochs: int=2,
interval: int=100,
diff --git a/released_box/tests/Multi-classification Task.ipynb b/released_box/tests/Multi-classification Task.ipynb
index 3d5b3cd..70d99d2 100644
--- a/released_box/tests/Multi-classification Task.ipynb
+++ b/released_box/tests/Multi-classification Task.ipynb
@@ -520,7 +520,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 8,
"id": "a771705d",
"metadata": {},
"outputs": [],
@@ -539,7 +539,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 9,
"id": "eee8abfb",
"metadata": {},
"outputs": [
@@ -547,7 +547,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "loss of Box on the 104960 test dataset: 0.46509239077568054.\n"
+ "Epoch [1/1], Step [100/3277], Training Loss: 0.4768, Validation Loss: 0.4207\n",
+ "Epoch [1/1], Step [200/3277], Training Loss: 0.3780, Validation Loss: 0.3931\n",
+ "Process stop at epoch [1/1] with patience 10 within tolerance 0.001\n",
+ "loss of Box on the 104960 test dataset: 0.38367217779159546.\n"
]
},
{
@@ -588,9 +591,9 @@
" 'princetonLocky': [100761, 104857],\n",
" 'white': [100761, 104857]}),\n",
" ('loss',\n",
- " {'train': 0.5337933897972107,\n",
- " 'val': 0.4650214910507202,\n",
- " 'test': 0.46509239077568054}),\n",
+ " {'train': 0.34570279717445374,\n",
+ " 'val': 0.3820456862449646,\n",
+ " 'test': 0.38367217779159546}),\n",
" ('sorted',\n",
" [('montrealAPT', [100761, 104857]),\n",
" ('montrealComradeCircle', [100761, 104857]),\n",
@@ -623,17 +626,19 @@
" ('white', [100761, 104857])])])"
]
},
- "execution_count": 13,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "main.test()\n",
"# main = perming.Box(8, 29, (40,), batch_size=256, activation='relu', inplace_on=True, solver='sgd', learning_rate_init=0.01)\n",
"# main = perming.Multiple(8, 9, (40,), batch_size=256, activation='relu', solver='sgd', learning_rate_init=0.01)\n",
"# main = perming.COMMON_MODELS['Mutiple Classifier'](8, 9, (40,), batch_size=256, activation='relu', solver='sgd', learning_rate_init=0.01)\n",
- "# main.print_config()"
+ "# main.print_config()\n",
+ "main.set_freeze({0:False}) # freeze the first layer of `self.model`\n",
+ "main.train_val(num_epochs=1, interval=100, early_stop=True)\n",
+ "main.test()"
]
},
{