From 82dfe070e15f5688ccedff6ecbfc9be77ff82eb3 Mon Sep 17 00:00:00 2001 From: Jing Lin <82669431+linjing-lab@users.noreply.github.com> Date: Sat, 28 Oct 2023 18:01:58 +0800 Subject: [PATCH] support set_freeze module --- released_box/README.md | 8 ++++++- released_box/perming/__init__.py | 2 +- released_box/perming/_utils.py | 14 +++++++++++ .../tests/Multi-classification Task.ipynb | 23 +++++++++++-------- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/released_box/README.md b/released_box/README.md index fa5ba99..a6db00c 100644 --- a/released_box/README.md +++ b/released_box/README.md @@ -35,6 +35,9 @@ data_loader: - worker_set: *Dict[str, int]={'train': 8, 'test': 2, 'val': 1}*, manually set by users need. - random_seed: *Optional[int]=None*, manually set any int value by users to fixed sequence. +set_freeze: +- require_grad: *Dict[int, bool]*, manually set freezed layers by given serial numbers according to `self.model`. (if users set require_grad with `{0: False}`, it means freeze the first layer of `self.model`. + train_val: - num_epochs: *int=2*, define numbers of epochs in main training cycle. any int value > 0. - interval: *int=100*, define console print length of whole epochs by interval. any int value > 0. @@ -52,7 +55,6 @@ save or load: - con: *bool=True*, configure whether to print model.state_dict(). False or True. - dir: *dir='./model'*, configure model path that *save to* or *load from*. correct path defined by users. - ## general model |GENERAL_BOX(Box)|Parameters|Meaning| @@ -74,6 +76,7 @@ save or load: |`__init__`|input_: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='MSELoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Regressier Based on Basic Information of the Regression Dataset Obtained through Data Preprocessing and Feature Engineering with `num_classes=1`.| |print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.| |data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Regression Dataset with Numpy format into `torch.utils.data.DataLoader`.| +|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. | |train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.| |test|/|Test Module Only Show with Loss at 3 Stages: Train, Test, Val| |save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.| @@ -86,6 +89,7 @@ save or load: |`__init__`|input_: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='BCELoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Classifier Based on Basic Information of the Classification Dataset Obtained through Data Preprocessing and Feature Engineering with `num_classes=2`.| |print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.| |data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Binary-classification Dataset with Numpy format into `torch.utils.data.DataLoader`.| +|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. | |train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.| |test|sort_by: str='accuracy'
sort_state: bool=True|Test Module con with Correct Class and Loss at 3 Stages: Train, Test, Val| |save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.| @@ -98,6 +102,7 @@ save or load: |`__init__`|input_: int
num_classes: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='CrossEntropyLoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Classifier Based on Basic Information of the Classification Dataset Obtained through Data Preprocessing and Feature Engineering with `num_classes>2`.| |print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.| |data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Multi-classification Dataset with Numpy format into `torch.utils.data.DataLoader`.| +|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. | |train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.| |test|sort_by: str='accuracy'
sort_state: bool=True|Sort Returned Test Result about Correct Classes with `sort_by` and `sort_state` Which Only Appears in Classification.| |save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.| @@ -110,6 +115,7 @@ save or load: |`__init__`|input_: int
num_outputs: int
hidden_layer_sizes: Tuple[int]=(100,)
*
activation: str='relu'
criterion: str='MultiLabelSoftMarginLoss'
solver: str='adam'
batch_size: int=32
learning_rate_init: float=1e-2
lr_scheduler: Optional[str]=None|Initialize Ranker Based on Basic Information of the Classification Dataset Obtained through Data Preprocessing and Feature Engineering with (n_samples, n_outputs).| |print_config|/|Return Initialized Parameters of Multi-layer Perceptron and Graph.| |data_loader|features: TabularData
labels: TabularData
ratio_set: Dict[str, int]={'train': 8, 'test': 1, 'val': 1}
worker_set: Dict[str, int]={'train': 8, 'test': 2, 'val': 1}
random_seed: Optional[int]=None|Using `ratio_set` and `worker_set` to Load the Multi-outputs Dataset with Numpy format into `torch.utils.data.DataLoader`.| +|set_freeze|require_grad: Dict[int, bool]|freeze some layers by given `requires_grad=False` if trained model will be loaded to execute experiments. | |train_val|num_epochs: int=2
interval: int=100
tolerance: float=1e-3
patience: int=10
backend: str='threading'
n_jobs: int=-1
early_stop: bool=False|Using `num_epochs`, `tolerance`, `patience` to Control Training Process and `interval` to Adjust Print Interval with Accelerated Validation Combined with `backend` and `n_jobs`.| |test|sort_by: str='accuracy'
sort_state: bool=True|Sort Returned Test Result about Correct Classes with `sort_by` and `sort_state` Which Only Appears in Classification.| |save|con: bool=True
dir: str='./model'|Save Trained Model Parameters with Model `state_dict` Control by `con`.| diff --git a/released_box/perming/__init__.py b/released_box/perming/__init__.py index 0bf12ec..5cfd496 100644 --- a/released_box/perming/__init__.py +++ b/released_box/perming/__init__.py @@ -27,4 +27,4 @@ 'Multi-outputs': Ranker } -__version__ = '1.7.1' \ No newline at end of file +__version__ = '1.8.0' \ No newline at end of file diff --git a/released_box/perming/_utils.py b/released_box/perming/_utils.py index d1f6110..ab88d1c 100644 --- a/released_box/perming/_utils.py +++ b/released_box/perming/_utils.py @@ -200,6 +200,20 @@ def data_loader(self, self.test_loader = torch.utils.data.DataLoader(TabularDataset(test_['features'], test_['target'], roc), batch_size=self.batch_size, shuffle=True, num_workers=worker_set['test']) self.val_loader = torch.utils.data.DataLoader(TabularDataset(val_['features'], val_['target'], roc), batch_size=self.batch_size, shuffle=False, num_workers=worker_set['val']) + def set_freeze(self, require_grad: Dict[int, bool]): + ''' + Freeze Network Layers of Trained Model by given `require_grad=False`. + :param require_grad: Dict[int, bool], whether to freeze grad on serial number of model layers. + ''' + operat: bool = False + for name, param in self.model.named_parameters(): + serial = int(name.strip('mlp.Linear.weight.bias')) + if serial in require_grad: + param.requires_grad = require_grad[serial] + operat: bool = True + if not operat: + raise ValueError('Please set require_grad states according to `self.model`.') + def train_val(self, num_epochs: int=2, interval: int=100, diff --git a/released_box/tests/Multi-classification Task.ipynb b/released_box/tests/Multi-classification Task.ipynb index 3d5b3cd..70d99d2 100644 --- a/released_box/tests/Multi-classification Task.ipynb +++ b/released_box/tests/Multi-classification Task.ipynb @@ -520,7 +520,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "id": "a771705d", "metadata": {}, "outputs": [], @@ -539,7 +539,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "id": "eee8abfb", "metadata": {}, "outputs": [ @@ -547,7 +547,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "loss of Box on the 104960 test dataset: 0.46509239077568054.\n" + "Epoch [1/1], Step [100/3277], Training Loss: 0.4768, Validation Loss: 0.4207\n", + "Epoch [1/1], Step [200/3277], Training Loss: 0.3780, Validation Loss: 0.3931\n", + "Process stop at epoch [1/1] with patience 10 within tolerance 0.001\n", + "loss of Box on the 104960 test dataset: 0.38367217779159546.\n" ] }, { @@ -588,9 +591,9 @@ " 'princetonLocky': [100761, 104857],\n", " 'white': [100761, 104857]}),\n", " ('loss',\n", - " {'train': 0.5337933897972107,\n", - " 'val': 0.4650214910507202,\n", - " 'test': 0.46509239077568054}),\n", + " {'train': 0.34570279717445374,\n", + " 'val': 0.3820456862449646,\n", + " 'test': 0.38367217779159546}),\n", " ('sorted',\n", " [('montrealAPT', [100761, 104857]),\n", " ('montrealComradeCircle', [100761, 104857]),\n", @@ -623,17 +626,19 @@ " ('white', [100761, 104857])])])" ] }, - "execution_count": 13, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "main.test()\n", "# main = perming.Box(8, 29, (40,), batch_size=256, activation='relu', inplace_on=True, solver='sgd', learning_rate_init=0.01)\n", "# main = perming.Multiple(8, 9, (40,), batch_size=256, activation='relu', solver='sgd', learning_rate_init=0.01)\n", "# main = perming.COMMON_MODELS['Mutiple Classifier'](8, 9, (40,), batch_size=256, activation='relu', solver='sgd', learning_rate_init=0.01)\n", - "# main.print_config()" + "# main.print_config()\n", + "main.set_freeze({0:False}) # freeze the first layer of `self.model`\n", + "main.train_val(num_epochs=1, interval=100, early_stop=True)\n", + "main.test()" ] }, {