From 48255b0806a72438ea949796bc133c8a5a256a25 Mon Sep 17 00:00:00 2001 From: Raphael Shu Date: Fri, 26 Aug 2016 18:37:13 +0900 Subject: [PATCH] speed monitoring --- deepy/dataset/ondisk_dataset.py | 9 ++++++--- deepy/layers/__init__.py | 2 +- deepy/layers/neural_var.py | 6 +++++- deepy/layers/plstm.py | 4 +++- deepy/layers/recurrent.py | 23 +++++++++++++++-------- deepy/trainers/base.py | 8 +++++++- 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/deepy/dataset/ondisk_dataset.py b/deepy/dataset/ondisk_dataset.py index 8b3206e..c92844e 100644 --- a/deepy/dataset/ondisk_dataset.py +++ b/deepy/dataset/ondisk_dataset.py @@ -38,9 +38,9 @@ def __init__(self, train_path, valid_path=None, test_path=None, train_size=None, logging.info("Cache on memory") self._cached_train_data = list(map(self._post_processing, StreamPickler.load(open(self._train_path)))) self._train_size = len(self._cached_train_data) - if self._shuffle_memory: - logging.info("Shuffle on-memory data") - global_rand.shuffle(self._cached_train_data) + # if self._shuffle_memory: + # logging.info("Shuffle on-memory data") + # global_rand.shuffle(self._cached_train_data) def curriculum_train_data(self): self._curriculum_count += 1 @@ -61,6 +61,9 @@ def generate_test_data(self): def train_set(self): if self._cache_on_memory: + if self._shuffle_memory: + logging.info("shuffle on-memory data") + global_rand.shuffle(self._cached_train_data) if self._curriculum: if not isinstance(self._curriculum(self._cached_train_data, 1), types.GeneratorType): raise Exception("Curriculum function must be a generator.") diff --git a/deepy/layers/__init__.py b/deepy/layers/__init__.py index 62aa21b..6d4d70d 100644 --- a/deepy/layers/__init__.py +++ b/deepy/layers/__init__.py @@ -15,7 +15,7 @@ from prelu import PRelu from irnn import IRNN from lstm import LSTM -from plstm import PeepholeLSTM +from plstm import PeepholeLSTM, PLSTM from gru import GRU from batch_norm import BatchNormalization from onehot_embed import OneHotEmbedding diff --git a/deepy/layers/neural_var.py b/deepy/layers/neural_var.py index 8fb69bf..bcf4868 100644 --- a/deepy/layers/neural_var.py +++ b/deepy/layers/neural_var.py @@ -27,7 +27,11 @@ def getitem_wrapper(t, index): index = tuple(index) return t.__getitem__(index) ret = getitem_wrapper(self, index) - ret.output_dim = self.dim() + if (hasattr(ret.tensor, 'tag') and hasattr(ret.tensor.tag, 'test_value') + and ret.tensor.tag.test_value is not None and len(ret.tensor.tag.test_value.shape) > 0): + ret.output_dim = ret.tensor.tag.test_value.shape[-1] + else: + ret.output_dim = self.dim() return ret def __call__(self, *args, **kwargs): diff --git a/deepy/layers/plstm.py b/deepy/layers/plstm.py index 7b3328e..b66d33b 100644 --- a/deepy/layers/plstm.py +++ b/deepy/layers/plstm.py @@ -77,4 +77,6 @@ def prepare(self): self.b_i, self.b_f, self.b_c, self.b_o) else: self.register_parameters(self.U, self.C, - self.b_i, self.b_f, self.b_c, self.b_o) \ No newline at end of file + self.b_i, self.b_f, self.b_c, self.b_o) + +PLSTM = PeepholeLSTM \ No newline at end of file diff --git a/deepy/layers/recurrent.py b/deepy/layers/recurrent.py index 1e0704b..a895995 100644 --- a/deepy/layers/recurrent.py +++ b/deepy/layers/recurrent.py @@ -113,13 +113,20 @@ def get_step_inputs(self, input_var, states=None, mask=None, additional_inputs=N return step_inputs - def compute(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, all_states=None, return_all=False): + def compute(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, init_states=None, return_all_states=False): if additional_inputs and not self.additional_input_dims: self.additional_input_dims = map(lambda var: var.dim(), additional_inputs) - return super(RecurrentLayer, self).compute(input_var, - mask=mask, additional_inputs=additional_inputs, steps=steps, backward=backward, all_states=all_states, return_all=return_all) + result_var = super(RecurrentLayer, self).compute(input_var, + mask=mask, additional_inputs=additional_inputs, steps=steps, backward=backward, init_states=init_states, return_all_states=return_all_states) + if return_all_states: + state_map = {} + for k in result_var.tensor: + state_map[k] = NeuralVariable(result_var.tensor[k], result_var.test_tensor[k], self.output_dim) + return state_map + else: + return result_var - def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, all_states=None, return_all=False): + def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, init_states=None, return_all_states=False): # prepare parameters backward = backward if backward else self._go_backwards steps = steps if steps else self._steps @@ -128,8 +135,8 @@ def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=Non raise Exception("Mask only works with sequence input") # get initial states init_state_map = self.get_initial_states(input_var) - if all_states: - for name, val in all_states.items(): + if init_states: + for name, val in init_states.items(): if name in init_state_map: init_state_map[name] = val # get input sequence map @@ -154,7 +161,7 @@ def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=Non # return main states main_states = retval_map[self.main_state] if self._output_type == "one": - if return_all: + if return_all_states: return_map = {} for name, val in retval_map.items(): return_map[name] = val[-1] @@ -162,7 +169,7 @@ def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=Non else: return main_states[-1] elif self._output_type == "sequence": - if return_all: + if return_all_states: return_map = {} for name, val in retval_map.items(): return_map[name] = val.dimshuffle((1,0,2)) diff --git a/deepy/trainers/base.py b/deepy/trainers/base.py index 89d44f5..42c0e96 100644 --- a/deepy/trainers/base.py +++ b/deepy/trainers/base.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import sys +import time import numpy as np import theano @@ -279,8 +280,11 @@ def train_step(self, train_set, train_size=None): network_callback = bool(self.network.training_callbacks) trainer_callback = bool(self._iter_callbacks) cost_matrix = [] + exec_count = 0 + start_time = time.time() self._progress = 0 + for x in train_set: if self._skip_batches == 0: @@ -301,6 +305,7 @@ def train_step(self, train_set, train_size=None): cost_x = self.learn(*[t[(t.shape[0]/2):] for t in x]) cost_matrix.append(cost_x) self.last_cost = cost_x[0] + exec_count += 1 if network_callback: self.network.training_callback() if trainer_callback: @@ -310,7 +315,8 @@ def train_step(self, train_set, train_size=None): self._skip_batches -= 1 if train_size: self._progress += 1 - sys.stdout.write("\x1b[2K\r> %d%% | J=%.2f" % (self._progress * 100 / train_size, self.last_cost)) + spd = float(exec_count) / (time.time() - start_time) + sys.stdout.write("\x1b[2K\r> %d%% | J=%.2f | spd=%.2f batch/s" % (self._progress * 100 / train_size, self.last_cost, spd)) sys.stdout.flush() self._progress = 0