train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorlayer as tl\n",
    "import tensorflow as tf\n",
    "import rawpy\n",
    "\n",
    "import model\n",
    "import utils\n",
    "\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import glob\n",
    "import importlib\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] [!] training_dir exists ...\n",
      "[TL] [!] checkpoint exists ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import config as _config\n",
    "importlib.reload(_config)\n",
    "from config import config, log_config\n",
    "\n",
    "###====================== HYPER-PARAMETERS ===========================###\n",
    "## Adam\n",
    "batch_size = config.TRAIN.batch_size\n",
    "lr_init = config.TRAIN.lr_init\n",
    "beta1 = config.TRAIN.beta1\n",
    "## initialize G\n",
    "n_epoch_init = config.TRAIN.n_epoch_init\n",
    "## adversarial learning (SRGAN)\n",
    "n_epoch = config.TRAIN.n_epoch\n",
    "lr_decay = config.TRAIN.lr_decay\n",
    "decay_every = config.TRAIN.decay_every\n",
    "training_dir = config.TRAIN.training_dir\n",
    "pretrain_checkpoint = config.TRAIN.pretrain_checkpoint\n",
    "train_data_dir = config.TRAIN.train_data_dir\n",
    "dark_model_dir = config.TRAIN.sample_lst\n",
    "\n",
    "crop_num = config.TRAIN.crop_num\n",
    "sample_img_size = config.TRAIN.sample_img_size\n",
    "sample_lst = config.TRAIN.sample_lst\n",
    "\n",
    "tl.files.exists_or_mkdir(training_dir)\n",
    "tl.files.exists_or_mkdir(pretrain_checkpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import model as _model\n",
    "importlib.reload(_model)\n",
    "import utils as _utils\n",
    "importlib.reload(_utils)\n",
    "import dark_utils as _dark_utils\n",
    "importlib.reload(_dark_utils)\n",
    "\n",
    "from model import *\n",
    "from utils import *\n",
    "from dark_utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] InputLayer  SRGAN_g/in: (?, 96, 96, 3)\n",
      "[TL] Conv2d SRGAN_g/n64s1/c: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/0: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/0: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/0: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/1: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/1: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/1: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/2: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/2: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/2: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/3: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/3: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/4: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/4: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/4: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/5: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/5: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/5: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/6: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/6: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/6: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/7: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/7: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/7: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/8: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/8: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/8: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/9: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/9: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/9: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/10: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/10: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/10: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/11: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/11: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/11: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/12: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/12: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/12: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/13: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/13: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/13: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/14: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/14: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/14: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/15: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/15: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/15: size: (?, 96, 96, 64) fn: add\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] Conv2d SRGAN_g/n64s1/c/m: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b/m: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/add3: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n256s1/1_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/1_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/n256s1/2_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/2_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/out_new: n_filter: 1 filter_size: (1, 1) strides: (1, 1) pad: SAME act: tanh\n",
      "[TL] WARNING: Function: `tensorlayer.layers.utils.set_name_reuse` (in file: /home/hwa125/.local/lib/python3.6/site-packages/tensorlayer/layers/utils.py) is deprecated and will be removed after 2018-06-30.\n",
      "Instructions for updating: TensorLayer relies on TensorFlow to check name reusing\n",
      "\n",
      "[TL] WARNING: this method is DEPRECATED and has no effect, please remove it from your code.\n",
      "[TL] InputLayer  SRGAN_d/input/images: (?, 384, 384, 1)\n",
      "[TL] Conv2d SRGAN_d/h0/c: n_filter: 64 filter_size: (4, 4) strides: (2, 2) pad: SAME act: <lambda>\n",
      "[TL] WARNING: Function: `tensorlayer.activation.leaky_relu` (in file: /home/hwa125/.local/lib/python3.6/site-packages/tensorlayer/activation.py) is deprecated and will be removed after 2018-09-30.\n",
      "Instructions for updating: This API is deprecated. Please use as `tf.nn.leaky_relu`\n",
      "\n",
      "[TL] Conv2d SRGAN_d/h1/c: n_filter: 128 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h1/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h2/c: n_filter: 256 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h2/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h3/c: n_filter: 512 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h3/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h4/c: n_filter: 1024 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h4/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h5/c: n_filter: 2048 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h5/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h6/c: n_filter: 1024 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h6/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h7/c: n_filter: 512 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h7/bn: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c: n_filter: 128 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c2: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn2: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_d/res/add: size: (?, 6, 6, 512) fn: add\n",
      "[TL] FlattenLayer SRGAN_d/ho/flatten: 18432\n",
      "[TL] DenseLayer  SRGAN_d/ho/dense: 1 No Activation\n",
      "[TL] WARNING: this method is DEPRECATED and has no effect, please remove it from your code.\n",
      "[TL] InputLayer  SRGAN_d/input/images: (?, 384, 384, 1)\n",
      "[TL] Conv2d SRGAN_d/h0/c: n_filter: 64 filter_size: (4, 4) strides: (2, 2) pad: SAME act: <lambda>\n",
      "[TL] Conv2d SRGAN_d/h1/c: n_filter: 128 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h1/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h2/c: n_filter: 256 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h2/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h3/c: n_filter: 512 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h3/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h4/c: n_filter: 1024 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h4/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h5/c: n_filter: 2048 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h5/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h6/c: n_filter: 1024 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h6/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h7/c: n_filter: 512 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h7/bn: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c: n_filter: 128 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c2: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn2: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_d/res/add: size: (?, 6, 6, 512) fn: add\n",
      "[TL] FlattenLayer SRGAN_d/ho/flatten: 18432\n",
      "[TL] DenseLayer  SRGAN_d/ho/dense: 1 No Activation\n",
      "[TL]   param   0: SRGAN_g/n64s1/c/kernel:0 (3, 3, 3, 64)      float32_ref\n",
      "[TL]   param   1: SRGAN_g/n64s1/c/bias:0 (64,)              float32_ref\n",
      "[TL]   param   2: SRGAN_g/n64s1/c1/0/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param   3: SRGAN_g/n64s1/b1/0/beta:0 (64,)              float32_ref\n",
      "[TL]   param   4: SRGAN_g/n64s1/b1/0/gamma:0 (64,)              float32_ref\n",
      "[TL]   param   5: SRGAN_g/n64s1/b1/0/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param   6: SRGAN_g/n64s1/b1/0/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param   7: SRGAN_g/n64s1/c2/0/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param   8: SRGAN_g/n64s1/b2/0/beta:0 (64,)              float32_ref\n",
      "[TL]   param   9: SRGAN_g/n64s1/b2/0/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  10: SRGAN_g/n64s1/b2/0/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  11: SRGAN_g/n64s1/b2/0/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  12: SRGAN_g/n64s1/c1/1/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  13: SRGAN_g/n64s1/b1/1/beta:0 (64,)              float32_ref\n",
      "[TL]   param  14: SRGAN_g/n64s1/b1/1/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  15: SRGAN_g/n64s1/b1/1/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  16: SRGAN_g/n64s1/b1/1/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  17: SRGAN_g/n64s1/c2/1/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  18: SRGAN_g/n64s1/b2/1/beta:0 (64,)              float32_ref\n",
      "[TL]   param  19: SRGAN_g/n64s1/b2/1/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  20: SRGAN_g/n64s1/b2/1/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  21: SRGAN_g/n64s1/b2/1/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  22: SRGAN_g/n64s1/c1/2/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  23: SRGAN_g/n64s1/b1/2/beta:0 (64,)              float32_ref\n",
      "[TL]   param  24: SRGAN_g/n64s1/b1/2/gamma:0 (64,)              float32_ref\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL]   param  25: SRGAN_g/n64s1/b1/2/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  26: SRGAN_g/n64s1/b1/2/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  27: SRGAN_g/n64s1/c2/2/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  28: SRGAN_g/n64s1/b2/2/beta:0 (64,)              float32_ref\n",
      "[TL]   param  29: SRGAN_g/n64s1/b2/2/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  30: SRGAN_g/n64s1/b2/2/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  31: SRGAN_g/n64s1/b2/2/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  32: SRGAN_g/n64s1/c1/3/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  33: SRGAN_g/n64s1/b1/3/beta:0 (64,)              float32_ref\n",
      "[TL]   param  34: SRGAN_g/n64s1/b1/3/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  35: SRGAN_g/n64s1/b1/3/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  36: SRGAN_g/n64s1/b1/3/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  37: SRGAN_g/n64s1/c2/3/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  38: SRGAN_g/n64s1/b2/3/beta:0 (64,)              float32_ref\n",
      "[TL]   param  39: SRGAN_g/n64s1/b2/3/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  40: SRGAN_g/n64s1/b2/3/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  41: SRGAN_g/n64s1/b2/3/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  42: SRGAN_g/n64s1/c1/4/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  43: SRGAN_g/n64s1/b1/4/beta:0 (64,)              float32_ref\n",
      "[TL]   param  44: SRGAN_g/n64s1/b1/4/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  45: SRGAN_g/n64s1/b1/4/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  46: SRGAN_g/n64s1/b1/4/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  47: SRGAN_g/n64s1/c2/4/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  48: SRGAN_g/n64s1/b2/4/beta:0 (64,)              float32_ref\n",
      "[TL]   param  49: SRGAN_g/n64s1/b2/4/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  50: SRGAN_g/n64s1/b2/4/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  51: SRGAN_g/n64s1/b2/4/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  52: SRGAN_g/n64s1/c1/5/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  53: SRGAN_g/n64s1/b1/5/beta:0 (64,)              float32_ref\n",
      "[TL]   param  54: SRGAN_g/n64s1/b1/5/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  55: SRGAN_g/n64s1/b1/5/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  56: SRGAN_g/n64s1/b1/5/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  57: SRGAN_g/n64s1/c2/5/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  58: SRGAN_g/n64s1/b2/5/beta:0 (64,)              float32_ref\n",
      "[TL]   param  59: SRGAN_g/n64s1/b2/5/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  60: SRGAN_g/n64s1/b2/5/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  61: SRGAN_g/n64s1/b2/5/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  62: SRGAN_g/n64s1/c1/6/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  63: SRGAN_g/n64s1/b1/6/beta:0 (64,)              float32_ref\n",
      "[TL]   param  64: SRGAN_g/n64s1/b1/6/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  65: SRGAN_g/n64s1/b1/6/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  66: SRGAN_g/n64s1/b1/6/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  67: SRGAN_g/n64s1/c2/6/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  68: SRGAN_g/n64s1/b2/6/beta:0 (64,)              float32_ref\n",
      "[TL]   param  69: SRGAN_g/n64s1/b2/6/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  70: SRGAN_g/n64s1/b2/6/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  71: SRGAN_g/n64s1/b2/6/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  72: SRGAN_g/n64s1/c1/7/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  73: SRGAN_g/n64s1/b1/7/beta:0 (64,)              float32_ref\n",
      "[TL]   param  74: SRGAN_g/n64s1/b1/7/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  75: SRGAN_g/n64s1/b1/7/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  76: SRGAN_g/n64s1/b1/7/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  77: SRGAN_g/n64s1/c2/7/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  78: SRGAN_g/n64s1/b2/7/beta:0 (64,)              float32_ref\n",
      "[TL]   param  79: SRGAN_g/n64s1/b2/7/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  80: SRGAN_g/n64s1/b2/7/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  81: SRGAN_g/n64s1/b2/7/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  82: SRGAN_g/n64s1/c1/8/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  83: SRGAN_g/n64s1/b1/8/beta:0 (64,)              float32_ref\n",
      "[TL]   param  84: SRGAN_g/n64s1/b1/8/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  85: SRGAN_g/n64s1/b1/8/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  86: SRGAN_g/n64s1/b1/8/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  87: SRGAN_g/n64s1/c2/8/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  88: SRGAN_g/n64s1/b2/8/beta:0 (64,)              float32_ref\n",
      "[TL]   param  89: SRGAN_g/n64s1/b2/8/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  90: SRGAN_g/n64s1/b2/8/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  91: SRGAN_g/n64s1/b2/8/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  92: SRGAN_g/n64s1/c1/9/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  93: SRGAN_g/n64s1/b1/9/beta:0 (64,)              float32_ref\n",
      "[TL]   param  94: SRGAN_g/n64s1/b1/9/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  95: SRGAN_g/n64s1/b1/9/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  96: SRGAN_g/n64s1/b1/9/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  97: SRGAN_g/n64s1/c2/9/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  98: SRGAN_g/n64s1/b2/9/beta:0 (64,)              float32_ref\n",
      "[TL]   param  99: SRGAN_g/n64s1/b2/9/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 100: SRGAN_g/n64s1/b2/9/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 101: SRGAN_g/n64s1/b2/9/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 102: SRGAN_g/n64s1/c1/10/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 103: SRGAN_g/n64s1/b1/10/beta:0 (64,)              float32_ref\n",
      "[TL]   param 104: SRGAN_g/n64s1/b1/10/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 105: SRGAN_g/n64s1/b1/10/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 106: SRGAN_g/n64s1/b1/10/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 107: SRGAN_g/n64s1/c2/10/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 108: SRGAN_g/n64s1/b2/10/beta:0 (64,)              float32_ref\n",
      "[TL]   param 109: SRGAN_g/n64s1/b2/10/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 110: SRGAN_g/n64s1/b2/10/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 111: SRGAN_g/n64s1/b2/10/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 112: SRGAN_g/n64s1/c1/11/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 113: SRGAN_g/n64s1/b1/11/beta:0 (64,)              float32_ref\n",
      "[TL]   param 114: SRGAN_g/n64s1/b1/11/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 115: SRGAN_g/n64s1/b1/11/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 116: SRGAN_g/n64s1/b1/11/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 117: SRGAN_g/n64s1/c2/11/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 118: SRGAN_g/n64s1/b2/11/beta:0 (64,)              float32_ref\n",
      "[TL]   param 119: SRGAN_g/n64s1/b2/11/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 120: SRGAN_g/n64s1/b2/11/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 121: SRGAN_g/n64s1/b2/11/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 122: SRGAN_g/n64s1/c1/12/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 123: SRGAN_g/n64s1/b1/12/beta:0 (64,)              float32_ref\n",
      "[TL]   param 124: SRGAN_g/n64s1/b1/12/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 125: SRGAN_g/n64s1/b1/12/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 126: SRGAN_g/n64s1/b1/12/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 127: SRGAN_g/n64s1/c2/12/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 128: SRGAN_g/n64s1/b2/12/beta:0 (64,)              float32_ref\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL]   param 129: SRGAN_g/n64s1/b2/12/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 130: SRGAN_g/n64s1/b2/12/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 131: SRGAN_g/n64s1/b2/12/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 132: SRGAN_g/n64s1/c1/13/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 133: SRGAN_g/n64s1/b1/13/beta:0 (64,)              float32_ref\n",
      "[TL]   param 134: SRGAN_g/n64s1/b1/13/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 135: SRGAN_g/n64s1/b1/13/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 136: SRGAN_g/n64s1/b1/13/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 137: SRGAN_g/n64s1/c2/13/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 138: SRGAN_g/n64s1/b2/13/beta:0 (64,)              float32_ref\n",
      "[TL]   param 139: SRGAN_g/n64s1/b2/13/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 140: SRGAN_g/n64s1/b2/13/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 141: SRGAN_g/n64s1/b2/13/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 142: SRGAN_g/n64s1/c1/14/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 143: SRGAN_g/n64s1/b1/14/beta:0 (64,)              float32_ref\n",
      "[TL]   param 144: SRGAN_g/n64s1/b1/14/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 145: SRGAN_g/n64s1/b1/14/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 146: SRGAN_g/n64s1/b1/14/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 147: SRGAN_g/n64s1/c2/14/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 148: SRGAN_g/n64s1/b2/14/beta:0 (64,)              float32_ref\n",
      "[TL]   param 149: SRGAN_g/n64s1/b2/14/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 150: SRGAN_g/n64s1/b2/14/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 151: SRGAN_g/n64s1/b2/14/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 152: SRGAN_g/n64s1/c1/15/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 153: SRGAN_g/n64s1/b1/15/beta:0 (64,)              float32_ref\n",
      "[TL]   param 154: SRGAN_g/n64s1/b1/15/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 155: SRGAN_g/n64s1/b1/15/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 156: SRGAN_g/n64s1/b1/15/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 157: SRGAN_g/n64s1/c2/15/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 158: SRGAN_g/n64s1/b2/15/beta:0 (64,)              float32_ref\n",
      "[TL]   param 159: SRGAN_g/n64s1/b2/15/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 160: SRGAN_g/n64s1/b2/15/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 161: SRGAN_g/n64s1/b2/15/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 162: SRGAN_g/n64s1/c/m/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 163: SRGAN_g/n64s1/b/m/beta:0 (64,)              float32_ref\n",
      "[TL]   param 164: SRGAN_g/n64s1/b/m/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 165: SRGAN_g/n64s1/b/m/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 166: SRGAN_g/n64s1/b/m/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 167: SRGAN_g/n256s1/1_new/kernel:0 (3, 3, 64, 256)    float32_ref\n",
      "[TL]   param 168: SRGAN_g/n256s1/1_new/bias:0 (256,)             float32_ref\n",
      "[TL]   param 169: SRGAN_g/n256s1/2_new/kernel:0 (3, 3, 64, 256)    float32_ref\n",
      "[TL]   param 170: SRGAN_g/n256s1/2_new/bias:0 (256,)             float32_ref\n",
      "[TL]   param 171: SRGAN_g/out_new/kernel:0 (1, 1, 64, 1)      float32_ref\n",
      "[TL]   param 172: SRGAN_g/out_new/bias:0 (1,)               float32_ref\n",
      "[TL]   num of params: 1522241\n",
      "[TL]   param   0: SRGAN_d/h0/c/kernel:0 (4, 4, 1, 64)      float32_ref\n",
      "[TL]   param   1: SRGAN_d/h0/c/bias:0  (64,)              float32_ref\n",
      "[TL]   param   2: SRGAN_d/h1/c/kernel:0 (4, 4, 64, 128)    float32_ref\n",
      "[TL]   param   3: SRGAN_d/h1/bn/beta:0 (128,)             float32_ref\n",
      "[TL]   param   4: SRGAN_d/h1/bn/gamma:0 (128,)             float32_ref\n",
      "[TL]   param   5: SRGAN_d/h1/bn/moving_mean:0 (128,)             float32_ref\n",
      "[TL]   param   6: SRGAN_d/h1/bn/moving_variance:0 (128,)             float32_ref\n",
      "[TL]   param   7: SRGAN_d/h2/c/kernel:0 (4, 4, 128, 256)    float32_ref\n",
      "[TL]   param   8: SRGAN_d/h2/bn/beta:0 (256,)             float32_ref\n",
      "[TL]   param   9: SRGAN_d/h2/bn/gamma:0 (256,)             float32_ref\n",
      "[TL]   param  10: SRGAN_d/h2/bn/moving_mean:0 (256,)             float32_ref\n",
      "[TL]   param  11: SRGAN_d/h2/bn/moving_variance:0 (256,)             float32_ref\n",
      "[TL]   param  12: SRGAN_d/h3/c/kernel:0 (4, 4, 256, 512)    float32_ref\n",
      "[TL]   param  13: SRGAN_d/h3/bn/beta:0 (512,)             float32_ref\n",
      "[TL]   param  14: SRGAN_d/h3/bn/gamma:0 (512,)             float32_ref\n",
      "[TL]   param  15: SRGAN_d/h3/bn/moving_mean:0 (512,)             float32_ref\n",
      "[TL]   param  16: SRGAN_d/h3/bn/moving_variance:0 (512,)             float32_ref\n",
      "[TL]   param  17: SRGAN_d/h4/c/kernel:0 (4, 4, 512, 1024)    float32_ref\n",
      "[TL]   param  18: SRGAN_d/h4/bn/beta:0 (1024,)            float32_ref\n",
      "[TL]   param  19: SRGAN_d/h4/bn/gamma:0 (1024,)            float32_ref\n",
      "[TL]   param  20: SRGAN_d/h4/bn/moving_mean:0 (1024,)            float32_ref\n",
      "[TL]   param  21: SRGAN_d/h4/bn/moving_variance:0 (1024,)            float32_ref\n",
      "[TL]   param  22: SRGAN_d/h5/c/kernel:0 (4, 4, 1024, 2048)    float32_ref\n",
      "[TL]   param  23: SRGAN_d/h5/bn/beta:0 (2048,)            float32_ref\n",
      "[TL]   param  24: SRGAN_d/h5/bn/gamma:0 (2048,)            float32_ref\n",
      "[TL]   param  25: SRGAN_d/h5/bn/moving_mean:0 (2048,)            float32_ref\n",
      "[TL]   param  26: SRGAN_d/h5/bn/moving_variance:0 (2048,)            float32_ref\n",
      "[TL]   param  27: SRGAN_d/h6/c/kernel:0 (1, 1, 2048, 1024)    float32_ref\n",
      "[TL]   param  28: SRGAN_d/h6/bn/beta:0 (1024,)            float32_ref\n",
      "[TL]   param  29: SRGAN_d/h6/bn/gamma:0 (1024,)            float32_ref\n",
      "[TL]   param  30: SRGAN_d/h6/bn/moving_mean:0 (1024,)            float32_ref\n",
      "[TL]   param  31: SRGAN_d/h6/bn/moving_variance:0 (1024,)            float32_ref\n",
      "[TL]   param  32: SRGAN_d/h7/c/kernel:0 (1, 1, 1024, 512)    float32_ref\n",
      "[TL]   param  33: SRGAN_d/h7/bn/beta:0 (512,)             float32_ref\n",
      "[TL]   param  34: SRGAN_d/h7/bn/gamma:0 (512,)             float32_ref\n",
      "[TL]   param  35: SRGAN_d/h7/bn/moving_mean:0 (512,)             float32_ref\n",
      "[TL]   param  36: SRGAN_d/h7/bn/moving_variance:0 (512,)             float32_ref\n",
      "[TL]   param  37: SRGAN_d/res/c/kernel:0 (1, 1, 512, 128)    float32_ref\n",
      "[TL]   param  38: SRGAN_d/res/bn/beta:0 (128,)             float32_ref\n",
      "[TL]   param  39: SRGAN_d/res/bn/gamma:0 (128,)             float32_ref\n",
      "[TL]   param  40: SRGAN_d/res/bn/moving_mean:0 (128,)             float32_ref\n",
      "[TL]   param  41: SRGAN_d/res/bn/moving_variance:0 (128,)             float32_ref\n",
      "[TL]   param  42: SRGAN_d/res/c2/kernel:0 (3, 3, 128, 128)    float32_ref\n",
      "[TL]   param  43: SRGAN_d/res/bn2/beta:0 (128,)             float32_ref\n",
      "[TL]   param  44: SRGAN_d/res/bn2/gamma:0 (128,)             float32_ref\n",
      "[TL]   param  45: SRGAN_d/res/bn2/moving_mean:0 (128,)             float32_ref\n",
      "[TL]   param  46: SRGAN_d/res/bn2/moving_variance:0 (128,)             float32_ref\n",
      "[TL]   param  47: SRGAN_d/res/c3/kernel:0 (3, 3, 128, 512)    float32_ref\n",
      "[TL]   param  48: SRGAN_d/res/bn3/beta:0 (512,)             float32_ref\n",
      "[TL]   param  49: SRGAN_d/res/bn3/gamma:0 (512,)             float32_ref\n",
      "[TL]   param  50: SRGAN_d/res/bn3/moving_mean:0 (512,)             float32_ref\n",
      "[TL]   param  51: SRGAN_d/res/bn3/moving_variance:0 (512,)             float32_ref\n",
      "[TL]   param  52: SRGAN_d/ho/dense/W:0 (18432, 1)         float32_ref\n",
      "[TL]   param  53: SRGAN_d/ho/dense/b:0 (1,)               float32_ref\n",
      "[TL]   num of params: 48164417\n",
      "[TL] InputLayer  SRGAN_g/in: (?, ?, ?, 3)\n",
      "[TL] Conv2d SRGAN_g/n64s1/c: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/0: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/0: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/0: size: (?, ?, ?, 64) fn: add\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] Conv2d SRGAN_g/n64s1/c1/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/1: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/1: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/1: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/2: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/2: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/2: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/3: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/3: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/4: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/4: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/4: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/5: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/5: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/5: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/6: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/6: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/6: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/7: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/7: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/7: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/8: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/8: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/8: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/9: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/9: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/9: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/10: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/10: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/10: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/11: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/11: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/11: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/12: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/12: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/12: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/13: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/13: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/13: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/14: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/14: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/14: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/15: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/15: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/15: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c/m: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b/m: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/add3: size: (?, ?, ?, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n256s1/1_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/1_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/n256s1/2_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/2_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/out_new: n_filter: 1 filter_size: (1, 1) strides: (1, 1) pad: SAME act: tanh\n"
     ]
    }
   ],
   "source": [
    "t_image = tf.placeholder('float32', [None, 96, 96, 3], name='t_image_input_to_SRGAN_generator')\n",
    "t_image_sample = tf.placeholder('float32', [None, None, None, 3], name='t_image_sample_to_SRGAN_g_test')\n",
    "t_target_image = tf.placeholder('float32', [None, 384, 384, 1], name='t_target_image')\n",
    "\n",
    "net_g = SRGAN_g(t_image, is_train=True, reuse=False)\n",
    "net_d, logits_real = SRGAN_d(t_target_image, is_train=True, reuse=False)\n",
    "_, logits_fake = SRGAN_d(net_g.outputs, is_train=True, reuse=True)\n",
    "\n",
    "# net_g.print_params(False)￼\n",
    "\n",
    "net_g.print_params(False)\n",
    "# net_g.print_layers()\n",
    "net_d.print_params(False)\n",
    "# net_d.print_layers()\n",
    "\n",
    "## test inference\n",
    "net_g_test = SRGAN_g(t_image_sample, is_train=False, reuse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_raw_for_dark = tf.placeholder(tf.float32, [None, None, None, 4])\n",
    "out_image = dark_network(t_raw_for_dark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define training op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL]   [*] geting variables with SRGAN_g\n",
      "[TL]   got   0: SRGAN_g/n64s1/c/kernel:0   (3, 3, 3, 64)\n",
      "[TL]   got   1: SRGAN_g/n64s1/c/bias:0   (64,)\n",
      "[TL]   got   2: SRGAN_g/n64s1/c1/0/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got   3: SRGAN_g/n64s1/b1/0/beta:0   (64,)\n",
      "[TL]   got   4: SRGAN_g/n64s1/b1/0/gamma:0   (64,)\n",
      "[TL]   got   5: SRGAN_g/n64s1/c2/0/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got   6: SRGAN_g/n64s1/b2/0/beta:0   (64,)\n",
      "[TL]   got   7: SRGAN_g/n64s1/b2/0/gamma:0   (64,)\n",
      "[TL]   got   8: SRGAN_g/n64s1/c1/1/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got   9: SRGAN_g/n64s1/b1/1/beta:0   (64,)\n",
      "[TL]   got  10: SRGAN_g/n64s1/b1/1/gamma:0   (64,)\n",
      "[TL]   got  11: SRGAN_g/n64s1/c2/1/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  12: SRGAN_g/n64s1/b2/1/beta:0   (64,)\n",
      "[TL]   got  13: SRGAN_g/n64s1/b2/1/gamma:0   (64,)\n",
      "[TL]   got  14: SRGAN_g/n64s1/c1/2/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  15: SRGAN_g/n64s1/b1/2/beta:0   (64,)\n",
      "[TL]   got  16: SRGAN_g/n64s1/b1/2/gamma:0   (64,)\n",
      "[TL]   got  17: SRGAN_g/n64s1/c2/2/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  18: SRGAN_g/n64s1/b2/2/beta:0   (64,)\n",
      "[TL]   got  19: SRGAN_g/n64s1/b2/2/gamma:0   (64,)\n",
      "[TL]   got  20: SRGAN_g/n64s1/c1/3/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  21: SRGAN_g/n64s1/b1/3/beta:0   (64,)\n",
      "[TL]   got  22: SRGAN_g/n64s1/b1/3/gamma:0   (64,)\n",
      "[TL]   got  23: SRGAN_g/n64s1/c2/3/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  24: SRGAN_g/n64s1/b2/3/beta:0   (64,)\n",
      "[TL]   got  25: SRGAN_g/n64s1/b2/3/gamma:0   (64,)\n",
      "[TL]   got  26: SRGAN_g/n64s1/c1/4/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  27: SRGAN_g/n64s1/b1/4/beta:0   (64,)\n",
      "[TL]   got  28: SRGAN_g/n64s1/b1/4/gamma:0   (64,)\n",
      "[TL]   got  29: SRGAN_g/n64s1/c2/4/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  30: SRGAN_g/n64s1/b2/4/beta:0   (64,)\n",
      "[TL]   got  31: SRGAN_g/n64s1/b2/4/gamma:0   (64,)\n",
      "[TL]   got  32: SRGAN_g/n64s1/c1/5/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  33: SRGAN_g/n64s1/b1/5/beta:0   (64,)\n",
      "[TL]   got  34: SRGAN_g/n64s1/b1/5/gamma:0   (64,)\n",
      "[TL]   got  35: SRGAN_g/n64s1/c2/5/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  36: SRGAN_g/n64s1/b2/5/beta:0   (64,)\n",
      "[TL]   got  37: SRGAN_g/n64s1/b2/5/gamma:0   (64,)\n",
      "[TL]   got  38: SRGAN_g/n64s1/c1/6/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  39: SRGAN_g/n64s1/b1/6/beta:0   (64,)\n",
      "[TL]   got  40: SRGAN_g/n64s1/b1/6/gamma:0   (64,)\n",
      "[TL]   got  41: SRGAN_g/n64s1/c2/6/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  42: SRGAN_g/n64s1/b2/6/beta:0   (64,)\n",
      "[TL]   got  43: SRGAN_g/n64s1/b2/6/gamma:0   (64,)\n",
      "[TL]   got  44: SRGAN_g/n64s1/c1/7/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  45: SRGAN_g/n64s1/b1/7/beta:0   (64,)\n",
      "[TL]   got  46: SRGAN_g/n64s1/b1/7/gamma:0   (64,)\n",
      "[TL]   got  47: SRGAN_g/n64s1/c2/7/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  48: SRGAN_g/n64s1/b2/7/beta:0   (64,)\n",
      "[TL]   got  49: SRGAN_g/n64s1/b2/7/gamma:0   (64,)\n",
      "[TL]   got  50: SRGAN_g/n64s1/c1/8/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  51: SRGAN_g/n64s1/b1/8/beta:0   (64,)\n",
      "[TL]   got  52: SRGAN_g/n64s1/b1/8/gamma:0   (64,)\n",
      "[TL]   got  53: SRGAN_g/n64s1/c2/8/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  54: SRGAN_g/n64s1/b2/8/beta:0   (64,)\n",
      "[TL]   got  55: SRGAN_g/n64s1/b2/8/gamma:0   (64,)\n",
      "[TL]   got  56: SRGAN_g/n64s1/c1/9/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  57: SRGAN_g/n64s1/b1/9/beta:0   (64,)\n",
      "[TL]   got  58: SRGAN_g/n64s1/b1/9/gamma:0   (64,)\n",
      "[TL]   got  59: SRGAN_g/n64s1/c2/9/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  60: SRGAN_g/n64s1/b2/9/beta:0   (64,)\n",
      "[TL]   got  61: SRGAN_g/n64s1/b2/9/gamma:0   (64,)\n",
      "[TL]   got  62: SRGAN_g/n64s1/c1/10/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  63: SRGAN_g/n64s1/b1/10/beta:0   (64,)\n",
      "[TL]   got  64: SRGAN_g/n64s1/b1/10/gamma:0   (64,)\n",
      "[TL]   got  65: SRGAN_g/n64s1/c2/10/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  66: SRGAN_g/n64s1/b2/10/beta:0   (64,)\n",
      "[TL]   got  67: SRGAN_g/n64s1/b2/10/gamma:0   (64,)\n",
      "[TL]   got  68: SRGAN_g/n64s1/c1/11/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  69: SRGAN_g/n64s1/b1/11/beta:0   (64,)\n",
      "[TL]   got  70: SRGAN_g/n64s1/b1/11/gamma:0   (64,)\n",
      "[TL]   got  71: SRGAN_g/n64s1/c2/11/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  72: SRGAN_g/n64s1/b2/11/beta:0   (64,)\n",
      "[TL]   got  73: SRGAN_g/n64s1/b2/11/gamma:0   (64,)\n",
      "[TL]   got  74: SRGAN_g/n64s1/c1/12/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  75: SRGAN_g/n64s1/b1/12/beta:0   (64,)\n",
      "[TL]   got  76: SRGAN_g/n64s1/b1/12/gamma:0   (64,)\n",
      "[TL]   got  77: SRGAN_g/n64s1/c2/12/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  78: SRGAN_g/n64s1/b2/12/beta:0   (64,)\n",
      "[TL]   got  79: SRGAN_g/n64s1/b2/12/gamma:0   (64,)\n",
      "[TL]   got  80: SRGAN_g/n64s1/c1/13/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  81: SRGAN_g/n64s1/b1/13/beta:0   (64,)\n",
      "[TL]   got  82: SRGAN_g/n64s1/b1/13/gamma:0   (64,)\n",
      "[TL]   got  83: SRGAN_g/n64s1/c2/13/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  84: SRGAN_g/n64s1/b2/13/beta:0   (64,)\n",
      "[TL]   got  85: SRGAN_g/n64s1/b2/13/gamma:0   (64,)\n",
      "[TL]   got  86: SRGAN_g/n64s1/c1/14/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  87: SRGAN_g/n64s1/b1/14/beta:0   (64,)\n",
      "[TL]   got  88: SRGAN_g/n64s1/b1/14/gamma:0   (64,)\n",
      "[TL]   got  89: SRGAN_g/n64s1/c2/14/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  90: SRGAN_g/n64s1/b2/14/beta:0   (64,)\n",
      "[TL]   got  91: SRGAN_g/n64s1/b2/14/gamma:0   (64,)\n",
      "[TL]   got  92: SRGAN_g/n64s1/c1/15/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  93: SRGAN_g/n64s1/b1/15/beta:0   (64,)\n",
      "[TL]   got  94: SRGAN_g/n64s1/b1/15/gamma:0   (64,)\n",
      "[TL]   got  95: SRGAN_g/n64s1/c2/15/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  96: SRGAN_g/n64s1/b2/15/beta:0   (64,)\n",
      "[TL]   got  97: SRGAN_g/n64s1/b2/15/gamma:0   (64,)\n",
      "[TL]   got  98: SRGAN_g/n64s1/c/m/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  99: SRGAN_g/n64s1/b/m/beta:0   (64,)\n",
      "[TL]   got 100: SRGAN_g/n64s1/b/m/gamma:0   (64,)\n",
      "[TL]   got 101: SRGAN_g/n256s1/1_new/kernel:0   (3, 3, 64, 256)\n",
      "[TL]   got 102: SRGAN_g/n256s1/1_new/bias:0   (256,)\n",
      "[TL]   got 103: SRGAN_g/n256s1/2_new/kernel:0   (3, 3, 64, 256)\n",
      "[TL]   got 104: SRGAN_g/n256s1/2_new/bias:0   (256,)\n",
      "[TL]   got 105: SRGAN_g/out_new/kernel:0   (1, 1, 64, 1)\n",
      "[TL]   got 106: SRGAN_g/out_new/bias:0   (1,)\n",
      "[TL]   [*] geting variables with SRGAN_d\n",
      "[TL]   got   0: SRGAN_d/h0/c/kernel:0   (4, 4, 1, 64)\n",
      "[TL]   got   1: SRGAN_d/h0/c/bias:0   (64,)\n",
      "[TL]   got   2: SRGAN_d/h1/c/kernel:0   (4, 4, 64, 128)\n",
      "[TL]   got   3: SRGAN_d/h1/bn/beta:0   (128,)\n",
      "[TL]   got   4: SRGAN_d/h1/bn/gamma:0   (128,)\n",
      "[TL]   got   5: SRGAN_d/h2/c/kernel:0   (4, 4, 128, 256)\n",
      "[TL]   got   6: SRGAN_d/h2/bn/beta:0   (256,)\n",
      "[TL]   got   7: SRGAN_d/h2/bn/gamma:0   (256,)\n",
      "[TL]   got   8: SRGAN_d/h3/c/kernel:0   (4, 4, 256, 512)\n",
      "[TL]   got   9: SRGAN_d/h3/bn/beta:0   (512,)\n",
      "[TL]   got  10: SRGAN_d/h3/bn/gamma:0   (512,)\n",
      "[TL]   got  11: SRGAN_d/h4/c/kernel:0   (4, 4, 512, 1024)\n",
      "[TL]   got  12: SRGAN_d/h4/bn/beta:0   (1024,)\n",
      "[TL]   got  13: SRGAN_d/h4/bn/gamma:0   (1024,)\n",
      "[TL]   got  14: SRGAN_d/h5/c/kernel:0   (4, 4, 1024, 2048)\n",
      "[TL]   got  15: SRGAN_d/h5/bn/beta:0   (2048,)\n",
      "[TL]   got  16: SRGAN_d/h5/bn/gamma:0   (2048,)\n",
      "[TL]   got  17: SRGAN_d/h6/c/kernel:0   (1, 1, 2048, 1024)\n",
      "[TL]   got  18: SRGAN_d/h6/bn/beta:0   (1024,)\n",
      "[TL]   got  19: SRGAN_d/h6/bn/gamma:0   (1024,)\n",
      "[TL]   got  20: SRGAN_d/h7/c/kernel:0   (1, 1, 1024, 512)\n",
      "[TL]   got  21: SRGAN_d/h7/bn/beta:0   (512,)\n",
      "[TL]   got  22: SRGAN_d/h7/bn/gamma:0   (512,)\n",
      "[TL]   got  23: SRGAN_d/res/c/kernel:0   (1, 1, 512, 128)\n",
      "[TL]   got  24: SRGAN_d/res/bn/beta:0   (128,)\n",
      "[TL]   got  25: SRGAN_d/res/bn/gamma:0   (128,)\n",
      "[TL]   got  26: SRGAN_d/res/c2/kernel:0   (3, 3, 128, 128)\n",
      "[TL]   got  27: SRGAN_d/res/bn2/beta:0   (128,)\n",
      "[TL]   got  28: SRGAN_d/res/bn2/gamma:0   (128,)\n",
      "[TL]   got  29: SRGAN_d/res/c3/kernel:0   (3, 3, 128, 512)\n",
      "[TL]   got  30: SRGAN_d/res/bn3/beta:0   (512,)\n",
      "[TL]   got  31: SRGAN_d/res/bn3/gamma:0   (512,)\n",
      "[TL]   got  32: SRGAN_d/ho/dense/W:0   (18432, 1)\n",
      "[TL]   got  33: SRGAN_d/ho/dense/b:0   (1,)\n"
     ]
    }
   ],
   "source": [
    "d_loss1 = tl.cost.sigmoid_cross_entropy(logits_real, tf.ones_like(logits_real), name='d1')\n",
    "d_loss2 = tl.cost.sigmoid_cross_entropy(logits_fake, tf.zeros_like(logits_fake), name='d2')\n",
    "d_loss = d_loss1 + d_loss2\n",
    "\n",
    "g_gan_loss = 1e-3 * tl.cost.sigmoid_cross_entropy(logits_fake, tf.ones_like(logits_fake), name='g')\n",
    "mse_loss = tl.cost.mean_squared_error(net_g.outputs, t_target_image, is_mean=True)\n",
    "# vgg_loss = 2e-6 * tl.cost.mean_squared_error(vgg_predict_emb.outputs, vgg_target_emb.outputs, is_mean=True)\n",
    "\n",
    "# g_loss = mse_loss + vgg_loss + g_gan_loss\n",
    "g_loss = mse_loss + g_gan_loss\n",
    "\n",
    "g_vars = tl.layers.get_variables_with_name('SRGAN_g', True, True)\n",
    "d_vars = tl.layers.get_variables_with_name('SRGAN_d', True, True)\n",
    "\n",
    "with tf.variable_scope('learning_rate'):\n",
    "    lr_v = tf.Variable(lr_init, trainable=False)\n",
    "## Pretrain\n",
    "g_optim_init = tf.train.AdamOptimizer(lr_v, beta1=beta1).minimize(mse_loss, var_list=g_vars)\n",
    "## SRGAN\n",
    "g_optim = tf.train.AdamOptimizer(lr_v, beta1=beta1).minimize(g_loss, var_list=g_vars)\n",
    "d_optim = tf.train.AdamOptimizer(lr_v, beta1=beta1).minimize(d_loss, var_list=d_vars)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## restore model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,\n",
    "                                        log_device_placement=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] WARNING: Function: `tensorlayer.layers.utils.initialize_global_variables` (in file: /home/hwa125/.local/lib/python3.6/site-packages/tensorlayer/layers/utils.py) is deprecated and will be removed after 2018-09-30.\n",
      "Instructions for updating: This API is deprecated in favor of `tf.global_variables_initializer`\n",
      "\n",
      "[TL] ERROR: file training_dir/g_srgan.npz doesn't exist.\n",
      "[TL] [*] Load checkpoint/g_srgan.npz SUCCESS!\n",
      "[TL] ERROR: file training_dir/d_srgan.npz doesn't exist.\n"
     ]
    }
   ],
   "source": [
    "tl.layers.initialize_global_variables(sess)\n",
    "if tl.files.load_and_assign_npz(sess=sess, name=training_dir + '/g_srgan.npz', network=net_g) is None:\n",
    "    load_pretrain_model(sess=sess, npz_file=pretrain_checkpoint + '/g_srgan.npz', network=net_g)\n",
    "tl.files.load_and_assign_npz(sess=sess, name=training_dir + '/d_srgan.npz', network=net_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "var_dict = dark_model_var_dict(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='DARK'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded dark_model/Sony/model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from dark_model/Sony/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "load the pre-train dark model\n",
    "saver = tf.train.Saver(var_dict)\n",
    "ckpt = tf.train.get_checkpoint_state(dark_model_dir)\n",
    "if ckpt:\n",
    "    print('loaded ' + ckpt.model_checkpoint_path)\n",
    "    saver.restore(sess, ckpt.model_checkpoint_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## get a sub set of sample, test for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = Pool(3)\n",
    "\n",
    "# train_data_list = sorted(tl.files.load_file_list('test_data', regx = '^0.*.ARW', printable = False))\n",
    "train_data_list = sorted(tl.files.load_file_list('test_data', regx = '.*.arw', printable = False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample_index = list(range(0, 320, 80))\n",
    "sample_lst = [6]\n",
    "ni = int(np.sqrt(len(sample_lst)))\n",
    "\n",
    "sample_file_name = [train_data_list[i] for i in sample_lst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rgb_sample, raw_sample = get_inputs_labels(p = p,\n",
    "#                                           file_dir = train_data_dir,\n",
    "#                                           raw_file_list = sample_file_name,\n",
    "#                                           crop_num = None,\n",
    "#                                           crop_size = 1000)\n",
    "rgb_sample, raw_sample = get_inputs_labels(p = p,\n",
    "                                          file_dir = 'test_data',\n",
    "                                          raw_file_list = sample_file_name,\n",
    "                                          crop_num = None,\n",
    "                                          crop_size = 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the sample input rgb\n",
    "rgb_sample_x200 = (rgb_sample + 1) * 200\n",
    "rgb_sample_x200 = (np.minimum(rgb_sample_x200, 2) * 127.5).astype(np.uint8)\n",
    "rgb_sample_out_filename = training_dir + os.path.sep + 'rgb_sample.png'\n",
    "if not os.path.exists(rgb_sample_out_filename):\n",
    "    tl.vis.save_images(rgb_sample_x200, [ni, ni], rgb_sample_out_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the sample label raw\n",
    "raw_sample = pack_raw_matrix(raw_sample) * 200\n",
    "\n",
    "label_raw_sample = sess.run(out_image, feed_dict={t_raw_for_dark: raw_sample})\n",
    "label_raw_sample = np.minimum(np.maximum(label_raw_sample, 0), 1)\n",
    "label_raw_sample = (label_raw_sample * 255).astype(np.uint8)\n",
    "label_raw_sample_out_file_name = training_dir + os.path.sep + 'label_raw_sample.png'\n",
    "if not os.path.exists(label_raw_sample_out_file_name):\n",
    "    tl.vis.save_images(label_raw_sample, [ni, ni], label_raw_sample_out_file_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n",
    "### initialize G"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ** fixed learning rate: 0.000100 (for init G)\n"
     ]
    }
   ],
   "source": [
    "sess.run(tf.assign(lr_v, lr_init))\n",
    "print(\" ** fixed learning rate: %f (for init G)\" % lr_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 2\n",
    "crop_num = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [ 0/100]    0 time: 4.2848s, mse: 0.82553345 \n",
      "Epoch [ 0/100]    1 time: 1.3810s, mse: 0.79789329 \n",
      "Epoch [ 0/100]    2 time: 1.2712s, mse: 0.72667342 \n",
      "Epoch [ 0/100]    3 time: 1.2810s, mse: 0.82550436 \n",
      "[*] Epoch: [ 0/100] time: 8.2182s, mse: 0.79390113\n",
      "Epoch [ 1/100]    0 time: 1.3650s, mse: 0.77747792 \n",
      "Epoch [ 1/100]    1 time: 1.4443s, mse: 0.74258214 \n",
      "Epoch [ 1/100]    2 time: 1.3296s, mse: 0.69772798 \n",
      "Epoch [ 1/100]    3 time: 1.4641s, mse: 0.49750084 \n",
      "[*] Epoch: [ 1/100] time: 5.6032s, mse: 0.67882222\n",
      "Epoch [ 2/100]    0 time: 1.2875s, mse: 0.68143964 \n",
      "Epoch [ 2/100]    1 time: 1.3845s, mse: 0.54008365 \n",
      "Epoch [ 2/100]    2 time: 1.3346s, mse: 0.56234610 \n",
      "Epoch [ 2/100]    3 time: 1.2997s, mse: 0.46491694 \n",
      "[*] Epoch: [ 2/100] time: 5.3065s, mse: 0.56219658\n",
      "Epoch [ 3/100]    0 time: 1.3828s, mse: 0.62870270 \n",
      "Epoch [ 3/100]    1 time: 1.4353s, mse: 0.54932535 \n",
      "Epoch [ 3/100]    2 time: 1.2011s, mse: 0.49835545 \n",
      "Epoch [ 3/100]    3 time: 1.3952s, mse: 0.55458212 \n",
      "[*] Epoch: [ 3/100] time: 5.4150s, mse: 0.55774140\n",
      "Epoch [ 4/100]    0 time: 1.4341s, mse: 0.56195617 \n",
      "Epoch [ 4/100]    1 time: 1.3611s, mse: 0.43847898 \n",
      "Epoch [ 4/100]    2 time: 1.3874s, mse: 0.31543931 \n",
      "Epoch [ 4/100]    3 time: 1.3216s, mse: 0.47019830 \n",
      "[*] Epoch: [ 4/100] time: 5.5045s, mse: 0.44651819\n",
      "Epoch [ 5/100]    0 time: 1.4112s, mse: 0.42398098 \n",
      "Epoch [ 5/100]    1 time: 1.3362s, mse: 0.34836978 \n",
      "Epoch [ 5/100]    2 time: 1.3940s, mse: 0.28815660 \n",
      "Epoch [ 5/100]    3 time: 1.3036s, mse: 0.33684179 \n",
      "[*] Epoch: [ 5/100] time: 5.4453s, mse: 0.34933729\n",
      "Epoch [ 6/100]    0 time: 1.3155s, mse: 0.23407665 \n",
      "Epoch [ 6/100]    1 time: 1.3557s, mse: 0.20357363 \n",
      "Epoch [ 6/100]    2 time: 1.3249s, mse: 0.18989116 \n",
      "Epoch [ 6/100]    3 time: 1.2377s, mse: 0.23031050 \n",
      "[*] Epoch: [ 6/100] time: 5.2341s, mse: 0.21446298\n",
      "Epoch [ 7/100]    0 time: 1.2673s, mse: 0.15296349 \n",
      "Epoch [ 7/100]    1 time: 1.3185s, mse: 0.11975673 \n",
      "Epoch [ 7/100]    2 time: 1.3462s, mse: 0.09595062 \n",
      "Epoch [ 7/100]    3 time: 1.3240s, mse: 0.12801883 \n",
      "[*] Epoch: [ 7/100] time: 5.2566s, mse: 0.12417242\n",
      "Epoch [ 8/100]    0 time: 1.3087s, mse: 0.07140300 \n",
      "Epoch [ 8/100]    1 time: 1.3751s, mse: 0.06629663 \n",
      "Epoch [ 8/100]    2 time: 1.5040s, mse: 0.03670394 \n",
      "Epoch [ 8/100]    3 time: 1.3292s, mse: 0.03330489 \n",
      "[*] Epoch: [ 8/100] time: 5.5173s, mse: 0.05192711\n",
      "Epoch [ 9/100]    0 time: 1.3724s, mse: 0.03914219 \n",
      "Epoch [ 9/100]    1 time: 1.3679s, mse: 0.02792051 \n",
      "Epoch [ 9/100]    2 time: 1.4081s, mse: 0.02588195 \n",
      "Epoch [ 9/100]    3 time: 1.3009s, mse: 0.02690695 \n",
      "[*] Epoch: [ 9/100] time: 5.4496s, mse: 0.02996290\n",
      "Epoch [10/100]    0 time: 1.2718s, mse: 0.01501335 \n",
      "Epoch [10/100]    1 time: 1.5058s, mse: 0.01517644 \n",
      "Epoch [10/100]    2 time: 1.3935s, mse: 0.02773876 \n",
      "Epoch [10/100]    3 time: 1.2672s, mse: 0.01741625 \n",
      "[*] Epoch: [10/100] time: 5.4386s, mse: 0.01883620\n",
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [11/100]    0 time: 1.4163s, mse: 0.00913958 \n",
      "Epoch [11/100]    1 time: 1.3610s, mse: 0.00665353 \n",
      "Epoch [11/100]    2 time: 1.4147s, mse: 0.01552587 \n",
      "Epoch [11/100]    3 time: 1.3949s, mse: 0.01223743 \n",
      "[*] Epoch: [11/100] time: 5.5873s, mse: 0.01088910\n",
      "Epoch [12/100]    0 time: 1.3512s, mse: 0.00807967 \n",
      "Epoch [12/100]    1 time: 1.4119s, mse: 0.00681106 \n",
      "Epoch [12/100]    2 time: 1.4978s, mse: 0.00835824 \n",
      "Epoch [12/100]    3 time: 1.3262s, mse: 0.01228174 \n",
      "[*] Epoch: [12/100] time: 5.5873s, mse: 0.00888268\n",
      "Epoch [13/100]    0 time: 1.3166s, mse: 0.00550809 \n",
      "Epoch [13/100]    1 time: 1.4426s, mse: 0.00758788 \n",
      "Epoch [13/100]    2 time: 1.2829s, mse: 0.01768565 \n",
      "Epoch [13/100]    3 time: 1.3518s, mse: 0.00646657 \n",
      "[*] Epoch: [13/100] time: 5.3943s, mse: 0.00931205\n",
      "Epoch [14/100]    0 time: 1.3644s, mse: 0.00468823 \n",
      "Epoch [14/100]    1 time: 1.3828s, mse: 0.08619891 \n",
      "Epoch [14/100]    2 time: 1.2942s, mse: 0.00895539 \n",
      "Epoch [14/100]    3 time: 1.2884s, mse: 0.00490628 \n",
      "[*] Epoch: [14/100] time: 5.3300s, mse: 0.02618720\n",
      "Epoch [15/100]    0 time: 1.3126s, mse: 0.00660232 \n",
      "Epoch [15/100]    1 time: 1.4130s, mse: 0.01603442 \n",
      "Epoch [15/100]    2 time: 1.3271s, mse: 0.01179508 \n",
      "Epoch [15/100]    3 time: 1.2356s, mse: 0.00840486 \n",
      "[*] Epoch: [15/100] time: 5.2886s, mse: 0.01070917\n",
      "Epoch [16/100]    0 time: 1.3301s, mse: 0.04072621 \n",
      "Epoch [16/100]    1 time: 1.3921s, mse: 0.01032383 \n",
      "Epoch [16/100]    2 time: 1.3252s, mse: 0.01378933 \n",
      "Epoch [16/100]    3 time: 1.2795s, mse: 0.00996285 \n",
      "[*] Epoch: [16/100] time: 5.3271s, mse: 0.01870056\n",
      "Epoch [17/100]    0 time: 1.3651s, mse: 0.00724623 \n",
      "Epoch [17/100]    1 time: 1.4226s, mse: 0.01268548 \n",
      "Epoch [17/100]    2 time: 1.2796s, mse: 0.01849799 \n",
      "Epoch [17/100]    3 time: 1.1856s, mse: 0.01805532 \n",
      "[*] Epoch: [17/100] time: 5.2530s, mse: 0.01412125\n",
      "Epoch [18/100]    0 time: 1.3292s, mse: 0.01920722 \n",
      "Epoch [18/100]    1 time: 1.3823s, mse: 0.01354741 \n",
      "Epoch [18/100]    2 time: 1.3619s, mse: 0.01655986 \n",
      "Epoch [18/100]    3 time: 1.3540s, mse: 0.00632243 \n",
      "[*] Epoch: [18/100] time: 5.4276s, mse: 0.01390923\n",
      "Epoch [19/100]    0 time: 1.3661s, mse: 0.02747270 \n",
      "Epoch [19/100]    1 time: 1.2474s, mse: 0.00543032 \n",
      "Epoch [19/100]    2 time: 1.3082s, mse: 0.00996320 \n",
      "Epoch [19/100]    3 time: 1.3611s, mse: 0.11737706 \n",
      "[*] Epoch: [19/100] time: 5.2830s, mse: 0.04006082\n",
      "Epoch [20/100]    0 time: 1.3661s, mse: 0.00458054 \n",
      "Epoch [20/100]    1 time: 1.2688s, mse: 0.02389439 \n",
      "Epoch [20/100]    2 time: 1.2612s, mse: 0.00719591 \n",
      "Epoch [20/100]    3 time: 1.3244s, mse: 0.01647079 \n",
      "[*] Epoch: [20/100] time: 5.2208s, mse: 0.01303541\n",
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [21/100]    0 time: 1.3277s, mse: 0.01122353 \n",
      "Epoch [21/100]    1 time: 1.3006s, mse: 0.00767541 \n",
      "Epoch [21/100]    2 time: 1.3958s, mse: 0.03388417 \n",
      "Epoch [21/100]    3 time: 1.2802s, mse: 0.00740073 \n",
      "[*] Epoch: [21/100] time: 5.3046s, mse: 0.01504596\n",
      "Epoch [22/100]    0 time: 1.3780s, mse: 0.00930453 \n",
      "Epoch [22/100]    1 time: 1.3469s, mse: 0.01234885 \n",
      "Epoch [22/100]    2 time: 1.3066s, mse: 0.00994679 \n",
      "Epoch [22/100]    3 time: 1.2392s, mse: 0.00564869 \n",
      "[*] Epoch: [22/100] time: 5.2709s, mse: 0.00931221\n",
      "Epoch [23/100]    0 time: 1.4078s, mse: 0.00696885 \n",
      "Epoch [23/100]    1 time: 1.3062s, mse: 0.02686431 \n",
      "Epoch [23/100]    2 time: 1.3726s, mse: 0.00478815 \n",
      "Epoch [23/100]    3 time: 1.2696s, mse: 0.00805268 \n",
      "[*] Epoch: [23/100] time: 5.3564s, mse: 0.01166850\n",
      "Epoch [24/100]    0 time: 1.3063s, mse: 0.00450808 \n",
      "Epoch [24/100]    1 time: 1.3149s, mse: 0.00368595 \n",
      "Epoch [24/100]    2 time: 1.4228s, mse: 0.00678668 \n",
      "Epoch [24/100]    3 time: 1.2514s, mse: 0.00754801 \n",
      "[*] Epoch: [24/100] time: 5.2956s, mse: 0.00563218\n",
      "Epoch [25/100]    0 time: 1.3374s, mse: 0.00344732 \n",
      "Epoch [25/100]    1 time: 1.3615s, mse: 0.05376309 \n",
      "Epoch [25/100]    2 time: 1.3534s, mse: 0.05461688 \n",
      "Epoch [25/100]    3 time: 1.2691s, mse: 0.00436138 \n",
      "[*] Epoch: [25/100] time: 5.3217s, mse: 0.02904717\n",
      "Epoch [26/100]    0 time: 1.3318s, mse: 0.00361425 \n",
      "Epoch [26/100]    1 time: 1.3571s, mse: 0.00313745 \n",
      "Epoch [26/100]    2 time: 1.2978s, mse: 0.00625736 \n",
      "Epoch [26/100]    3 time: 1.3989s, mse: 0.04302895 \n",
      "[*] Epoch: [26/100] time: 5.3859s, mse: 0.01400950\n",
      "Epoch [27/100]    0 time: 1.3573s, mse: 0.00305404 \n",
      "Epoch [27/100]    1 time: 1.3737s, mse: 0.00345998 \n",
      "Epoch [27/100]    2 time: 1.3744s, mse: 0.01066604 \n",
      "Epoch [27/100]    3 time: 1.2674s, mse: 0.00465445 \n",
      "[*] Epoch: [27/100] time: 5.3731s, mse: 0.00545863\n",
      "Epoch [28/100]    0 time: 1.3593s, mse: 0.00293625 \n",
      "Epoch [28/100]    1 time: 1.3749s, mse: 0.04089608 \n",
      "Epoch [28/100]    2 time: 1.3150s, mse: 0.00674109 \n",
      "Epoch [28/100]    3 time: 1.2729s, mse: 0.01884352 \n",
      "[*] Epoch: [28/100] time: 5.3223s, mse: 0.01735423\n",
      "Epoch [29/100]    0 time: 1.3729s, mse: 0.00491309 \n",
      "Epoch [29/100]    1 time: 1.2673s, mse: 0.03074512 \n",
      "Epoch [29/100]    2 time: 1.3068s, mse: 0.00832325 \n",
      "Epoch [29/100]    3 time: 1.2621s, mse: 0.00465207 \n",
      "[*] Epoch: [29/100] time: 5.2094s, mse: 0.01215838\n",
      "Epoch [30/100]    0 time: 1.2690s, mse: 0.00743901 \n",
      "Epoch [30/100]    1 time: 1.4081s, mse: 0.00795155 \n",
      "Epoch [30/100]    2 time: 1.3380s, mse: 0.00578347 \n",
      "Epoch [30/100]    3 time: 1.2803s, mse: 0.00582106 \n",
      "[*] Epoch: [30/100] time: 5.2957s, mse: 0.00674877\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [31/100]    0 time: 1.3886s, mse: 0.00462554 \n",
      "Epoch [31/100]    1 time: 1.3737s, mse: 0.01797284 \n",
      "Epoch [31/100]    2 time: 1.3927s, mse: 0.00368195 \n",
      "Epoch [31/100]    3 time: 1.3109s, mse: 0.00322281 \n",
      "[*] Epoch: [31/100] time: 5.4662s, mse: 0.00737579\n",
      "Epoch [32/100]    0 time: 1.2444s, mse: 0.00576146 \n",
      "Epoch [32/100]    1 time: 1.3361s, mse: 0.00367709 \n",
      "Epoch [32/100]    2 time: 1.3201s, mse: 0.02259646 \n",
      "Epoch [32/100]    3 time: 1.2731s, mse: 0.00392294 \n",
      "[*] Epoch: [32/100] time: 5.1739s, mse: 0.00898949\n",
      "Epoch [33/100]    0 time: 1.3282s, mse: 0.00499573 \n",
      "Epoch [33/100]    1 time: 1.3182s, mse: 0.00477029 \n",
      "Epoch [33/100]    2 time: 1.3526s, mse: 0.00404269 \n",
      "Epoch [33/100]    3 time: 1.2430s, mse: 0.00418914 \n",
      "[*] Epoch: [33/100] time: 5.2423s, mse: 0.00449946\n",
      "Epoch [34/100]    0 time: 1.3730s, mse: 0.00359722 \n",
      "Epoch [34/100]    1 time: 1.3653s, mse: 0.00328161 \n",
      "Epoch [34/100]    2 time: 1.3622s, mse: 0.13278180 \n",
      "Epoch [34/100]    3 time: 1.2661s, mse: 0.00317317 \n",
      "[*] Epoch: [34/100] time: 5.3668s, mse: 0.03570845\n",
      "Epoch [35/100]    0 time: 1.3137s, mse: 0.01586042 \n",
      "Epoch [35/100]    1 time: 1.4206s, mse: 0.00455736 \n",
      "Epoch [35/100]    2 time: 1.3547s, mse: 0.07059616 \n",
      "Epoch [35/100]    3 time: 1.2811s, mse: 0.00439859 \n",
      "[*] Epoch: [35/100] time: 5.3704s, mse: 0.02385313\n",
      "Epoch [36/100]    0 time: 1.3404s, mse: 0.00341981 \n",
      "Epoch [36/100]    1 time: 1.2656s, mse: 0.00602232 \n",
      "Epoch [36/100]    2 time: 1.3177s, mse: 0.07713132 \n",
      "Epoch [36/100]    3 time: 1.2443s, mse: 0.00490756 \n",
      "[*] Epoch: [36/100] time: 5.1683s, mse: 0.02287025\n",
      "Epoch [37/100]    0 time: 1.3529s, mse: 0.00413081 \n",
      "Epoch [37/100]    1 time: 1.2254s, mse: 0.00393077 \n",
      "Epoch [37/100]    2 time: 1.3428s, mse: 0.00735888 \n",
      "Epoch [37/100]    3 time: 1.2436s, mse: 0.00530153 \n",
      "[*] Epoch: [37/100] time: 5.1649s, mse: 0.00518050\n",
      "Epoch [38/100]    0 time: 1.3114s, mse: 0.01228063 \n",
      "Epoch [38/100]    1 time: 1.2710s, mse: 0.00800009 \n",
      "Epoch [38/100]    2 time: 1.3180s, mse: 0.03951130 \n",
      "Epoch [38/100]    3 time: 1.3158s, mse: 0.00470912 \n",
      "[*] Epoch: [38/100] time: 5.2165s, mse: 0.01612528\n",
      "Epoch [39/100]    0 time: 1.3120s, mse: 0.02147736 \n",
      "Epoch [39/100]    1 time: 1.3922s, mse: 0.03658637 \n",
      "Epoch [39/100]    2 time: 1.2890s, mse: 0.04325344 \n",
      "Epoch [39/100]    3 time: 1.3538s, mse: 0.00366603 \n",
      "[*] Epoch: [39/100] time: 5.3472s, mse: 0.02624580\n",
      "Epoch [40/100]    0 time: 1.3245s, mse: 0.00440816 \n",
      "Epoch [40/100]    1 time: 1.5486s, mse: 0.00845187 \n",
      "Epoch [40/100]    2 time: 1.3309s, mse: 0.04518341 \n",
      "Epoch [40/100]    3 time: 1.3561s, mse: 0.00326232 \n",
      "[*] Epoch: [40/100] time: 5.5604s, mse: 0.01532644\n",
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(0, 40 + 1):\n",
    "    epoch_time = time.time()\n",
    "    total_mse_loss, n_iter = 0, 0\n",
    "\n",
    "    for idx in range(0, len(train_data_list), batch_size):\n",
    "        step_time = time.time()\n",
    "        \n",
    "        batch_file_name = train_data_list[idx: idx+batch_size]\n",
    "#         inputs_rgbs, label_raws = get_inputs_labels(p, train_data_dir, batch_file_name, crop_num)\n",
    "        inputs_rgbs, label_raws = get_inputs_labels(p, 'test_data', batch_file_name, crop_num)\n",
    "        \n",
    "        ## update G\n",
    "        errM, _ = sess.run([mse_loss, g_optim_init], {t_image: inputs_rgbs, t_target_image: label_raws})\n",
    "        print(\"Epoch [%2d/%2d] %4d time: %4.4fs, mse: %.8f \" % (epoch, n_epoch_init, n_iter, time.time() - step_time, errM))\n",
    "        total_mse_loss += errM\n",
    "        n_iter += 1\n",
    "    log = \"[*] Epoch: [%2d/%2d] time: %4.4fs, mse: %.8f\" % (epoch, n_epoch_init, time.time() - epoch_time, total_mse_loss / n_iter)\n",
    "    print(log)\n",
    "\n",
    "    # quick evaluation on train set\n",
    "    if (epoch != 0) and (epoch % 10 == 0):\n",
    "        sample_out = sess.run(net_g_test.outputs, {t_image_sample: rgb_sample})\n",
    "        sample_out = pack_raw_matrix(sample_out) * 200\n",
    "        \n",
    "        train_raw_sample = sess.run(out_image, feed_dict={t_raw_for_dark: sample_out})\n",
    "        train_raw_sample = np.minimum(np.maximum(train_raw_sample, 0), 1)\n",
    "        train_raw_sample = (train_raw_sample * 255).astype(np.uint8)\n",
    "        train_raw_sample_out_file_name = training_dir + os.path.sep + '/train_raw_sample_%d.png' % epoch\n",
    "        \n",
    "        print(\"[*] save images\")\n",
    "        tl.vis.save_images(train_raw_sample, [ni, ni], train_raw_sample_out_file_name)\n",
    "            \n",
    "    ## save model\n",
    "    if (epoch != 0) and (epoch % 10 == 0):\n",
    "        tl.files.save_npz(net_g.all_params, name=training_dir + '/g_srgan_init.npz', sess=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ** init lr: 0.000100  decay_every_init: 1000, lr_decay: 0.100000 (for GAN)\n",
      "Epoch [ 0/2000]    0 time: 1.9502s, d_loss: 2.52115321 g_loss: 0.00374823 (mse: 0.002150 adv: 0.001598)\n",
      "Epoch [ 0/2000]    1 time: 1.9604s, d_loss: 0.38484010 g_loss: 0.02049297 (mse: 0.014041 adv: 0.006452)\n",
      "Epoch [ 0/2000]    2 time: 1.8715s, d_loss: 2.44681311 g_loss: 0.00923260 (mse: 0.007457 adv: 0.001775)\n",
      "Epoch [ 0/2000]    3 time: 1.9933s, d_loss: 3.55068922 g_loss: 0.00211774 (mse: 0.000801 adv: 0.001317)\n",
      "[*] Epoch: [ 0/2000] time: 7.7757s, d_loss: 2.22587391 g_loss: 0.00889789\n",
      "Epoch [ 1/2000]    0 time: 1.9835s, d_loss: 0.14413087 g_loss: 0.00623906 (mse: 0.001515 adv: 0.004724)\n",
      "Epoch [ 1/2000]    1 time: 1.9573s, d_loss: 2.16561866 g_loss: 0.00857060 (mse: 0.004541 adv: 0.004029)\n",
      "Epoch [ 1/2000]    2 time: 1.9668s, d_loss: 7.51290512 g_loss: 0.00624027 (mse: 0.002106 adv: 0.004135)\n",
      "Epoch [ 1/2000]    3 time: 1.8213s, d_loss: 3.38742065 g_loss: 0.03051618 (mse: 0.029201 adv: 0.001315)\n",
      "[*] Epoch: [ 1/2000] time: 7.7292s, d_loss: 3.30251883 g_loss: 0.01289153\n",
      "Epoch [ 2/2000]    0 time: 1.9232s, d_loss: 2.64028287 g_loss: 0.00964958 (mse: 0.005953 adv: 0.003697)\n",
      "Epoch [ 2/2000]    1 time: 2.0145s, d_loss: 6.10762024 g_loss: 0.01271532 (mse: 0.009264 adv: 0.003452)\n",
      "Epoch [ 2/2000]    2 time: 1.9398s, d_loss: 0.21571854 g_loss: 0.00996205 (mse: 0.006860 adv: 0.003102)\n",
      "Epoch [ 2/2000]    3 time: 1.8282s, d_loss: 0.45750675 g_loss: 0.00338310 (mse: 0.002267 adv: 0.001117)\n",
      "[*] Epoch: [ 2/2000] time: 7.7059s, d_loss: 2.35528210 g_loss: 0.00892751\n",
      "Epoch [ 3/2000]    0 time: 1.8844s, d_loss: 3.06233120 g_loss: 0.00365124 (mse: 0.002093 adv: 0.001558)\n",
      "Epoch [ 3/2000]    1 time: 1.9741s, d_loss: 1.89966977 g_loss: 0.01508676 (mse: 0.009110 adv: 0.005977)\n",
      "Epoch [ 3/2000]    2 time: 1.8671s, d_loss: 1.47272789 g_loss: 0.00735866 (mse: 0.005920 adv: 0.001439)\n",
      "Epoch [ 3/2000]    3 time: 1.8587s, d_loss: 4.41768980 g_loss: 0.00203803 (mse: 0.001452 adv: 0.000586)\n",
      "[*] Epoch: [ 3/2000] time: 7.5845s, d_loss: 2.71310467 g_loss: 0.00703367\n",
      "Epoch [ 4/2000]    0 time: 1.8473s, d_loss: 2.02201438 g_loss: 0.01305425 (mse: 0.010156 adv: 0.002898)\n",
      "Epoch [ 4/2000]    1 time: 1.9745s, d_loss: 7.22981262 g_loss: 0.01838386 (mse: 0.018149 adv: 0.000235)\n",
      "Epoch [ 4/2000]    2 time: 1.9949s, d_loss: 0.91278249 g_loss: 0.01204753 (mse: 0.006841 adv: 0.005206)\n",
      "Epoch [ 4/2000]    3 time: 1.8822s, d_loss: 5.28696966 g_loss: 0.00289053 (mse: 0.001191 adv: 0.001699)\n",
      "[*] Epoch: [ 4/2000] time: 7.6991s, d_loss: 3.86289479 g_loss: 0.01159404\n",
      "Epoch [ 5/2000]    0 time: 1.9745s, d_loss: 5.00100756 g_loss: 0.00729762 (mse: 0.001939 adv: 0.005359)\n",
      "Epoch [ 5/2000]    1 time: 1.9251s, d_loss: 1.61071038 g_loss: 0.00974129 (mse: 0.005753 adv: 0.003989)\n",
      "Epoch [ 5/2000]    2 time: 1.9273s, d_loss: 1.41340768 g_loss: 0.05219458 (mse: 0.048105 adv: 0.004089)\n",
      "Epoch [ 5/2000]    3 time: 1.9145s, d_loss: 1.85636973 g_loss: 0.01186977 (mse: 0.001797 adv: 0.010073)\n",
      "[*] Epoch: [ 5/2000] time: 7.7418s, d_loss: 2.47037384 g_loss: 0.02027581\n",
      "Epoch [ 6/2000]    0 time: 1.9589s, d_loss: 3.98565722 g_loss: 0.00760905 (mse: 0.002612 adv: 0.004997)\n",
      "Epoch [ 6/2000]    1 time: 1.8894s, d_loss: 3.95101786 g_loss: 0.00776070 (mse: 0.001226 adv: 0.006534)\n",
      "Epoch [ 6/2000]    2 time: 1.9033s, d_loss: 1.58193111 g_loss: 0.05660206 (mse: 0.049954 adv: 0.006648)\n",
      "Epoch [ 6/2000]    3 time: 1.8915s, d_loss: 1.02987432 g_loss: 0.01574657 (mse: 0.009935 adv: 0.005811)\n",
      "[*] Epoch: [ 6/2000] time: 7.6433s, d_loss: 2.63712013 g_loss: 0.02192960\n",
      "Epoch [ 7/2000]    0 time: 1.9175s, d_loss: 0.09169019 g_loss: 0.01705021 (mse: 0.010036 adv: 0.007015)\n",
      "Epoch [ 7/2000]    1 time: 2.0621s, d_loss: 0.27640259 g_loss: 0.01077576 (mse: 0.004279 adv: 0.006497)\n",
      "Epoch [ 7/2000]    2 time: 1.9766s, d_loss: 1.96560490 g_loss: 0.01633241 (mse: 0.012873 adv: 0.003459)\n",
      "Epoch [ 7/2000]    3 time: 1.8777s, d_loss: 0.89456630 g_loss: 0.00285296 (mse: 0.000959 adv: 0.001894)\n",
      "[*] Epoch: [ 7/2000] time: 7.8342s, d_loss: 0.80706600 g_loss: 0.01175284\n",
      "Epoch [ 8/2000]    0 time: 1.9863s, d_loss: 0.60953873 g_loss: 0.00444714 (mse: 0.002501 adv: 0.001946)\n",
      "Epoch [ 8/2000]    1 time: 1.8395s, d_loss: 1.87186635 g_loss: 0.02155778 (mse: 0.011720 adv: 0.009838)\n",
      "Epoch [ 8/2000]    2 time: 1.8328s, d_loss: 2.90816164 g_loss: 0.00506902 (mse: 0.002898 adv: 0.002171)\n",
      "Epoch [ 8/2000]    3 time: 1.9458s, d_loss: 2.07203960 g_loss: 0.00701700 (mse: 0.005518 adv: 0.001499)\n",
      "[*] Epoch: [ 8/2000] time: 7.6046s, d_loss: 1.86540158 g_loss: 0.00952273\n",
      "Epoch [ 9/2000]    0 time: 1.8562s, d_loss: 1.79337490 g_loss: 0.00372719 (mse: 0.002419 adv: 0.001308)\n",
      "Epoch [ 9/2000]    1 time: 1.8912s, d_loss: 0.79990578 g_loss: 0.00359778 (mse: 0.002345 adv: 0.001253)\n",
      "Epoch [ 9/2000]    2 time: 1.9592s, d_loss: 2.81441426 g_loss: 0.00571865 (mse: 0.005008 adv: 0.000711)\n",
      "Epoch [ 9/2000]    3 time: 1.8557s, d_loss: 1.91645265 g_loss: 0.02715865 (mse: 0.023502 adv: 0.003656)\n",
      "[*] Epoch: [ 9/2000] time: 7.5626s, d_loss: 1.83103690 g_loss: 0.01005057\n",
      "Epoch [10/2000]    0 time: 1.9187s, d_loss: 0.16764230 g_loss: 0.00677042 (mse: 0.002892 adv: 0.003878)\n",
      "Epoch [10/2000]    1 time: 1.9275s, d_loss: 3.06745124 g_loss: 0.00487770 (mse: 0.002087 adv: 0.002791)\n",
      "Epoch [10/2000]    2 time: 1.9359s, d_loss: 1.85692596 g_loss: 0.00875460 (mse: 0.003376 adv: 0.005379)\n",
      "Epoch [10/2000]    3 time: 1.9130s, d_loss: 0.00711571 g_loss: 0.00820603 (mse: 0.001036 adv: 0.007170)\n",
      "[*] Epoch: [10/2000] time: 7.6953s, d_loss: 1.27478380 g_loss: 0.00715219\n",
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan.npz\n",
      "[TL] [*] Saved\n",
      "[TL] [*] Saving TL params into training_dir/d_srgan.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [11/2000]    0 time: 1.9000s, d_loss: 2.32948089 g_loss: 0.00236089 (mse: 0.000639 adv: 0.001722)\n",
      "Epoch [11/2000]    1 time: 1.8666s, d_loss: 3.76875329 g_loss: 0.01100481 (mse: 0.009379 adv: 0.001626)\n",
      "Epoch [11/2000]    2 time: 1.9844s, d_loss: 1.84408903 g_loss: 0.00536774 (mse: 0.001921 adv: 0.003447)\n",
      "Epoch [11/2000]    3 time: 1.8535s, d_loss: 1.76976323 g_loss: 0.00432107 (mse: 0.000975 adv: 0.003346)\n",
      "[*] Epoch: [11/2000] time: 7.6048s, d_loss: 2.42802161 g_loss: 0.00576363\n",
      "Epoch [12/2000]    0 time: 1.9566s, d_loss: 0.01317082 g_loss: 0.01082101 (mse: 0.003525 adv: 0.007296)\n",
      "Epoch [12/2000]    1 time: 1.9139s, d_loss: 10.40152645 g_loss: 0.00964551 (mse: 0.000949 adv: 0.008697)\n",
      "Epoch [12/2000]    2 time: 2.0089s, d_loss: 1.50874496 g_loss: 0.00890374 (mse: 0.003893 adv: 0.005011)\n",
      "Epoch [12/2000]    3 time: 1.8519s, d_loss: 0.01696936 g_loss: 0.00559981 (mse: 0.000980 adv: 0.004620)\n",
      "[*] Epoch: [12/2000] time: 7.7315s, d_loss: 2.98510290 g_loss: 0.00874252\n",
      "Epoch [13/2000]    0 time: 1.9295s, d_loss: 1.69094431 g_loss: 0.00245967 (mse: 0.000840 adv: 0.001620)\n",
      "Epoch [13/2000]    1 time: 1.9603s, d_loss: 0.81809270 g_loss: 0.00352544 (mse: 0.001775 adv: 0.001750)\n",
      "Epoch [13/2000]    2 time: 1.8618s, d_loss: 4.12017441 g_loss: 0.00308790 (mse: 0.002203 adv: 0.000884)\n",
      "Epoch [13/2000]    3 time: 2.0422s, d_loss: 3.93852115 g_loss: 0.00230836 (mse: 0.000343 adv: 0.001965)\n",
      "[*] Epoch: [13/2000] time: 7.7941s, d_loss: 2.64193314 g_loss: 0.00284534\n",
      "Epoch [14/2000]    0 time: 1.9219s, d_loss: 3.13860893 g_loss: 0.00333430 (mse: 0.000667 adv: 0.002667)\n",
      "Epoch [14/2000]    1 time: 1.9865s, d_loss: 0.77579927 g_loss: 0.01570563 (mse: 0.011651 adv: 0.004054)\n",
      "Epoch [14/2000]    2 time: 1.9814s, d_loss: 2.26582170 g_loss: 0.00726847 (mse: 0.004313 adv: 0.002955)\n",
      "Epoch [14/2000]    3 time: 1.9319s, d_loss: 5.33604336 g_loss: 0.00888373 (mse: 0.002915 adv: 0.005969)\n",
      "[*] Epoch: [14/2000] time: 7.8219s, d_loss: 2.87906832 g_loss: 0.00879803\n",
      "Epoch [15/2000]    0 time: 1.9060s, d_loss: 1.60262191 g_loss: 0.00838294 (mse: 0.001323 adv: 0.007060)\n",
      "Epoch [15/2000]    1 time: 1.8487s, d_loss: 0.63178867 g_loss: 0.01007162 (mse: 0.006399 adv: 0.003672)\n",
      "Epoch [15/2000]    2 time: 1.9395s, d_loss: 2.15522003 g_loss: 0.00859738 (mse: 0.005288 adv: 0.003309)\n",
      "Epoch [15/2000]    3 time: 1.8991s, d_loss: 0.21756575 g_loss: 0.02199238 (mse: 0.018435 adv: 0.003557)\n",
      "[*] Epoch: [15/2000] time: 7.5935s, d_loss: 1.15179909 g_loss: 0.01226108\n",
      "Epoch [16/2000]    0 time: 1.9995s, d_loss: 1.74074531 g_loss: 0.00401420 (mse: 0.001362 adv: 0.002652)\n",
      "Epoch [16/2000]    1 time: 1.9290s, d_loss: 1.26736712 g_loss: 0.00365033 (mse: 0.002250 adv: 0.001400)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [16/2000]    2 time: 1.9155s, d_loss: 3.12672067 g_loss: 0.00359301 (mse: 0.002507 adv: 0.001086)\n",
      "Epoch [16/2000]    3 time: 1.9720s, d_loss: 2.18370509 g_loss: 0.00401797 (mse: 0.000908 adv: 0.003110)\n",
      "[*] Epoch: [16/2000] time: 7.8164s, d_loss: 2.07963455 g_loss: 0.00381888\n",
      "Epoch [17/2000]    0 time: 2.0329s, d_loss: 2.66017056 g_loss: 0.00195472 (mse: 0.000615 adv: 0.001340)\n",
      "Epoch [17/2000]    1 time: 1.9494s, d_loss: 2.42755389 g_loss: 0.00844134 (mse: 0.006270 adv: 0.002171)\n",
      "Epoch [17/2000]    2 time: 1.9348s, d_loss: 3.04890609 g_loss: 0.00426535 (mse: 0.003159 adv: 0.001107)\n",
      "Epoch [17/2000]    3 time: 1.8577s, d_loss: 1.60033560 g_loss: 0.00281314 (mse: 0.000753 adv: 0.002060)\n",
      "[*] Epoch: [17/2000] time: 7.7751s, d_loss: 2.43424153 g_loss: 0.00436864\n",
      "Epoch [18/2000]    0 time: 1.8836s, d_loss: 5.64814138 g_loss: 0.00666251 (mse: 0.002869 adv: 0.003794)\n",
      "Epoch [18/2000]    1 time: 1.9535s, d_loss: 0.91201079 g_loss: 0.01069581 (mse: 0.009197 adv: 0.001499)\n",
      "Epoch [18/2000]    2 time: 2.0140s, d_loss: 0.85790712 g_loss: 0.01874626 (mse: 0.013696 adv: 0.005050)\n",
      "Epoch [18/2000]    3 time: 1.9151s, d_loss: 0.36856490 g_loss: 0.04553358 (mse: 0.041449 adv: 0.004084)\n",
      "[*] Epoch: [18/2000] time: 7.7665s, d_loss: 1.94665605 g_loss: 0.02040954\n",
      "Epoch [19/2000]    0 time: 1.9120s, d_loss: 3.57077384 g_loss: 0.00462809 (mse: 0.003240 adv: 0.001388)\n",
      "Epoch [19/2000]    1 time: 1.9468s, d_loss: 1.84456277 g_loss: 0.00354113 (mse: 0.001954 adv: 0.001588)\n",
      "Epoch [19/2000]    2 time: 1.9068s, d_loss: 0.00136710 g_loss: 0.01285411 (mse: 0.004862 adv: 0.007992)\n",
      "Epoch [19/2000]    3 time: 1.9289s, d_loss: 2.35925579 g_loss: 0.06885694 (mse: 0.065858 adv: 0.002999)\n",
      "[*] Epoch: [19/2000] time: 7.6947s, d_loss: 1.94398988 g_loss: 0.02247007\n",
      "Epoch [20/2000]    0 time: 1.8871s, d_loss: 0.93647277 g_loss: 0.00540765 (mse: 0.002322 adv: 0.003086)\n",
      "Epoch [20/2000]    1 time: 1.9065s, d_loss: 0.11238705 g_loss: 0.02281948 (mse: 0.015674 adv: 0.007146)\n",
      "Epoch [20/2000]    2 time: 1.9300s, d_loss: 0.15240887 g_loss: 0.10452254 (mse: 0.100012 adv: 0.004511)\n",
      "Epoch [20/2000]    3 time: 1.8114s, d_loss: 0.16517106 g_loss: 0.07551574 (mse: 0.065455 adv: 0.010061)\n",
      "[*] Epoch: [20/2000] time: 7.5352s, d_loss: 0.34160994 g_loss: 0.05206635\n",
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan.npz\n",
      "[TL] [*] Saved\n",
      "[TL] [*] Saving TL params into training_dir/d_srgan.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [21/2000]    0 time: 1.9413s, d_loss: 2.61150455 g_loss: 0.00432724 (mse: 0.000933 adv: 0.003395)\n",
      "Epoch [21/2000]    1 time: 1.8707s, d_loss: 0.38464579 g_loss: 0.01999028 (mse: 0.010338 adv: 0.009652)\n",
      "Epoch [21/2000]    2 time: 1.9597s, d_loss: 6.50528955 g_loss: 0.01146406 (mse: 0.005005 adv: 0.006459)\n",
      "Epoch [21/2000]    3 time: 1.8787s, d_loss: 0.00904114 g_loss: 0.05886959 (mse: 0.051274 adv: 0.007595)\n",
      "[*] Epoch: [21/2000] time: 7.6507s, d_loss: 2.37762026 g_loss: 0.02366279\n",
      "Epoch [22/2000]    0 time: 2.0549s, d_loss: 1.69525826 g_loss: 0.00475795 (mse: 0.001670 adv: 0.003088)\n",
      "Epoch [22/2000]    1 time: 1.9412s, d_loss: 2.88113880 g_loss: 0.01117289 (mse: 0.006777 adv: 0.004396)\n",
      "Epoch [22/2000]    2 time: 1.9320s, d_loss: 0.47052091 g_loss: 0.00791239 (mse: 0.003676 adv: 0.004237)\n",
      "Epoch [22/2000]    3 time: 1.8742s, d_loss: 1.59542799 g_loss: 0.03695432 (mse: 0.035192 adv: 0.001762)\n",
      "[*] Epoch: [22/2000] time: 7.8025s, d_loss: 1.66058649 g_loss: 0.01519939\n",
      "Epoch [23/2000]    0 time: 1.8536s, d_loss: 3.05002546 g_loss: 0.00380325 (mse: 0.002905 adv: 0.000898)\n",
      "Epoch [23/2000]    1 time: 2.0772s, d_loss: 4.61051941 g_loss: 0.00362873 (mse: 0.003150 adv: 0.000479)\n",
      "Epoch [23/2000]    2 time: 1.9859s, d_loss: 7.59207869 g_loss: 0.00558537 (mse: 0.004899 adv: 0.000686)\n",
      "Epoch [23/2000]    3 time: 1.8863s, d_loss: 1.09761596 g_loss: 0.00922485 (mse: 0.004393 adv: 0.004831)\n",
      "[*] Epoch: [23/2000] time: 7.8034s, d_loss: 4.08755988 g_loss: 0.00556055\n",
      "Epoch [24/2000]    0 time: 1.9924s, d_loss: 3.03630424 g_loss: 0.00712684 (mse: 0.001741 adv: 0.005385)\n",
      "Epoch [24/2000]    1 time: 2.0152s, d_loss: 0.49311534 g_loss: 0.01070694 (mse: 0.006846 adv: 0.003861)\n",
      "Epoch [24/2000]    2 time: 2.0135s, d_loss: 2.86387491 g_loss: 0.00810839 (mse: 0.001947 adv: 0.006161)\n",
      "Epoch [24/2000]    3 time: 1.8987s, d_loss: 0.12239038 g_loss: 0.03418442 (mse: 0.029067 adv: 0.005117)\n",
      "[*] Epoch: [24/2000] time: 7.9202s, d_loss: 1.62892122 g_loss: 0.01503165\n",
      "Epoch [25/2000]    0 time: 2.0749s, d_loss: 3.63168859 g_loss: 0.00686943 (mse: 0.000905 adv: 0.005965)\n",
      "Epoch [25/2000]    1 time: 2.1658s, d_loss: 0.00368551 g_loss: 0.02174563 (mse: 0.015482 adv: 0.006264)\n",
      "Epoch [25/2000]    2 time: 2.0480s, d_loss: 0.64421129 g_loss: 0.01087752 (mse: 0.004013 adv: 0.006864)\n",
      "Epoch [25/2000]    3 time: 1.9609s, d_loss: 0.17089370 g_loss: 0.00753269 (mse: 0.001041 adv: 0.006492)\n",
      "[*] Epoch: [25/2000] time: 8.2497s, d_loss: 1.11261977 g_loss: 0.01175632\n",
      "Epoch [26/2000]    0 time: 1.9931s, d_loss: 3.97854209 g_loss: 0.00631910 (mse: 0.003521 adv: 0.002798)\n",
      "Epoch [26/2000]    1 time: 2.0010s, d_loss: 4.24964046 g_loss: 0.00776260 (mse: 0.006089 adv: 0.001674)\n",
      "Epoch [26/2000]    2 time: 2.0393s, d_loss: 7.21549463 g_loss: 0.00655998 (mse: 0.001238 adv: 0.005322)\n",
      "Epoch [26/2000]    3 time: 1.9171s, d_loss: 0.32441363 g_loss: 0.00594155 (mse: 0.001041 adv: 0.004900)\n",
      "[*] Epoch: [26/2000] time: 7.9507s, d_loss: 3.94202270 g_loss: 0.00664581\n",
      "Epoch [27/2000]    0 time: 1.9714s, d_loss: 0.12414417 g_loss: 0.00376466 (mse: 0.000564 adv: 0.003200)\n",
      "Epoch [27/2000]    1 time: 1.9190s, d_loss: 0.00369583 g_loss: 0.01971397 (mse: 0.013488 adv: 0.006226)\n",
      "Epoch [27/2000]    2 time: 1.9746s, d_loss: 0.95843178 g_loss: 0.00437416 (mse: 0.001518 adv: 0.002856)\n",
      "Epoch [27/2000]    3 time: 1.9002s, d_loss: 0.20224160 g_loss: 0.00319773 (mse: 0.001028 adv: 0.002170)\n",
      "[*] Epoch: [27/2000] time: 7.7655s, d_loss: 0.32212834 g_loss: 0.00776263\n",
      "Epoch [28/2000]    0 time: 1.8937s, d_loss: 0.18131495 g_loss: 0.00752910 (mse: 0.002832 adv: 0.004697)\n",
      "Epoch [28/2000]    1 time: 1.9468s, d_loss: 1.64464259 g_loss: 0.00418805 (mse: 0.002695 adv: 0.001493)\n",
      "Epoch [28/2000]    2 time: 1.8909s, d_loss: 3.84633899 g_loss: 0.00528420 (mse: 0.001934 adv: 0.003351)\n",
      "Epoch [28/2000]    3 time: 1.8745s, d_loss: 1.04004276 g_loss: 0.00379230 (mse: 0.000412 adv: 0.003381)\n",
      "[*] Epoch: [28/2000] time: 7.6061s, d_loss: 1.67808482 g_loss: 0.00519841\n",
      "Epoch [29/2000]    0 time: 1.7967s, d_loss: 0.17536183 g_loss: 0.00963510 (mse: 0.001978 adv: 0.007658)\n",
      "Epoch [29/2000]    1 time: 1.9858s, d_loss: 0.04829849 g_loss: 0.00592444 (mse: 0.000834 adv: 0.005090)\n",
      "Epoch [29/2000]    2 time: 2.0473s, d_loss: 2.94660759 g_loss: 0.00422559 (mse: 0.001795 adv: 0.002430)\n",
      "Epoch [29/2000]    3 time: 1.8568s, d_loss: 0.37170219 g_loss: 0.00418861 (mse: 0.000439 adv: 0.003749)\n",
      "[*] Epoch: [29/2000] time: 7.6869s, d_loss: 0.88549252 g_loss: 0.00599343\n",
      "Epoch [30/2000]    0 time: 1.8690s, d_loss: 5.80536461 g_loss: 0.00429368 (mse: 0.000923 adv: 0.003371)\n",
      "Epoch [30/2000]    1 time: 1.9106s, d_loss: 1.92834365 g_loss: 0.00836588 (mse: 0.001376 adv: 0.006990)\n",
      "Epoch [30/2000]    2 time: 1.8938s, d_loss: 0.91011649 g_loss: 0.00636621 (mse: 0.001647 adv: 0.004719)\n",
      "Epoch [30/2000]    3 time: 1.8136s, d_loss: 1.03262532 g_loss: 0.00883090 (mse: 0.006335 adv: 0.002496)\n",
      "[*] Epoch: [30/2000] time: 7.4872s, d_loss: 2.41911252 g_loss: 0.00696417\n",
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan.npz\n",
      "[TL] [*] Saved\n",
      "[TL] [*] Saving TL params into training_dir/d_srgan.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [31/2000]    0 time: 2.0471s, d_loss: 0.16376324 g_loss: 0.00630835 (mse: 0.001546 adv: 0.004762)\n",
      "Epoch [31/2000]    1 time: 1.9464s, d_loss: 1.35326540 g_loss: 0.00511237 (mse: 0.002075 adv: 0.003037)\n",
      "Epoch [31/2000]    2 time: 2.0028s, d_loss: 1.87564826 g_loss: 0.00710994 (mse: 0.005786 adv: 0.001323)\n",
      "Epoch [31/2000]    3 time: 1.9059s, d_loss: 2.51609993 g_loss: 0.00242022 (mse: 0.000701 adv: 0.001719)\n",
      "[*] Epoch: [31/2000] time: 7.9027s, d_loss: 1.47719421 g_loss: 0.00523772\n",
      "Epoch [32/2000]    0 time: 1.8829s, d_loss: 2.14721823 g_loss: 0.00468456 (mse: 0.002006 adv: 0.002679)\n",
      "Epoch [32/2000]    1 time: 1.9233s, d_loss: 3.70388794 g_loss: 0.00334579 (mse: 0.002146 adv: 0.001200)\n",
      "Epoch [32/2000]    2 time: 1.9596s, d_loss: 1.43337750 g_loss: 0.00866730 (mse: 0.003327 adv: 0.005340)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [32/2000]    3 time: 2.0660s, d_loss: 12.50316620 g_loss: 0.01567148 (mse: 0.012927 adv: 0.002744)\n",
      "[*] Epoch: [32/2000] time: 7.8320s, d_loss: 4.94691247 g_loss: 0.00809228\n",
      "Epoch [33/2000]    0 time: 1.9699s, d_loss: 1.11265266 g_loss: 0.00474504 (mse: 0.001754 adv: 0.002991)\n",
      "Epoch [33/2000]    1 time: 1.9608s, d_loss: 3.37703562 g_loss: 0.00502800 (mse: 0.001673 adv: 0.003355)\n",
      "Epoch [33/2000]    2 time: 1.9302s, d_loss: 9.54619312 g_loss: 0.01261944 (mse: 0.004697 adv: 0.007923)\n",
      "Epoch [33/2000]    3 time: 1.9678s, d_loss: 3.58529615 g_loss: 0.07749112 (mse: 0.070908 adv: 0.006584)\n",
      "[*] Epoch: [33/2000] time: 7.8289s, d_loss: 4.40529439 g_loss: 0.02497090\n",
      "Epoch [34/2000]    0 time: 2.0040s, d_loss: 12.63529682 g_loss: 0.00972597 (mse: 0.001050 adv: 0.008676)\n",
      "Epoch [34/2000]    1 time: 1.9176s, d_loss: 0.90438032 g_loss: 0.01266002 (mse: 0.006358 adv: 0.006302)\n",
      "Epoch [34/2000]    2 time: 1.8936s, d_loss: 2.38615632 g_loss: 0.01082266 (mse: 0.005838 adv: 0.004984)\n",
      "Epoch [34/2000]    3 time: 1.8465s, d_loss: 1.17513931 g_loss: 0.09718731 (mse: 0.095837 adv: 0.001350)\n",
      "[*] Epoch: [34/2000] time: 7.6620s, d_loss: 4.27524319 g_loss: 0.03259899\n",
      "Epoch [35/2000]    0 time: 1.9826s, d_loss: 6.26263094 g_loss: 0.00394278 (mse: 0.003159 adv: 0.000784)\n",
      "Epoch [35/2000]    1 time: 1.9313s, d_loss: 1.15196407 g_loss: 0.00534510 (mse: 0.003021 adv: 0.002324)\n",
      "Epoch [35/2000]    2 time: 1.9596s, d_loss: 6.63072872 g_loss: 0.00265340 (mse: 0.002615 adv: 0.000039)\n",
      "Epoch [35/2000]    3 time: 1.9667s, d_loss: 5.54380798 g_loss: 0.00130070 (mse: 0.001109 adv: 0.000192)\n",
      "[*] Epoch: [35/2000] time: 7.8404s, d_loss: 4.89728293 g_loss: 0.00331049\n",
      "Epoch [36/2000]    0 time: 2.0088s, d_loss: 9.37384224 g_loss: 0.00152936 (mse: 0.001526 adv: 0.000004)\n",
      "Epoch [36/2000]    1 time: 1.9230s, d_loss: 6.13171959 g_loss: 0.00733423 (mse: 0.007080 adv: 0.000254)\n",
      "Epoch [36/2000]    2 time: 1.9154s, d_loss: 2.34862685 g_loss: 0.00940272 (mse: 0.004323 adv: 0.005079)\n",
      "Epoch [36/2000]    3 time: 1.8458s, d_loss: 0.05173577 g_loss: 0.00615396 (mse: 0.000569 adv: 0.005585)\n",
      "[*] Epoch: [36/2000] time: 7.6932s, d_loss: 4.47648111 g_loss: 0.00610507\n",
      "Epoch [37/2000]    0 time: 1.9335s, d_loss: 3.70296645 g_loss: 0.00310646 (mse: 0.001193 adv: 0.001914)\n",
      "Epoch [37/2000]    1 time: 1.9518s, d_loss: 0.23640046 g_loss: 0.00690405 (mse: 0.002032 adv: 0.004872)\n",
      "Epoch [37/2000]    2 time: 2.0152s, d_loss: 0.03780053 g_loss: 0.00966183 (mse: 0.005184 adv: 0.004478)\n",
      "Epoch [37/2000]    3 time: 1.8377s, d_loss: 0.02156714 g_loss: 0.00753529 (mse: 0.001257 adv: 0.006279)\n",
      "[*] Epoch: [37/2000] time: 7.7385s, d_loss: 0.99968364 g_loss: 0.00680191\n",
      "Epoch [38/2000]    0 time: 1.9075s, d_loss: 1.72198546 g_loss: 0.00242761 (mse: 0.000871 adv: 0.001557)\n",
      "Epoch [38/2000]    1 time: 1.9357s, d_loss: 0.39229003 g_loss: 0.00780821 (mse: 0.002585 adv: 0.005223)\n",
      "Epoch [38/2000]    2 time: 1.9176s, d_loss: 0.61654049 g_loss: 0.05044765 (mse: 0.046944 adv: 0.003504)\n",
      "Epoch [38/2000]    3 time: 1.8355s, d_loss: 1.15763128 g_loss: 0.00365390 (mse: 0.000766 adv: 0.002888)\n",
      "[*] Epoch: [38/2000] time: 7.5967s, d_loss: 0.97211181 g_loss: 0.01608434\n",
      "Epoch [39/2000]    0 time: 1.8773s, d_loss: 2.35380864 g_loss: 0.00855170 (mse: 0.004076 adv: 0.004475)\n",
      "Epoch [39/2000]    1 time: 1.9118s, d_loss: 5.25843191 g_loss: 0.00647386 (mse: 0.001384 adv: 0.005090)\n",
      "Epoch [39/2000]    2 time: 1.9079s, d_loss: 3.51571369 g_loss: 0.00747634 (mse: 0.002037 adv: 0.005439)\n",
      "Epoch [39/2000]    3 time: 1.9522s, d_loss: 0.01623849 g_loss: 0.00831471 (mse: 0.000308 adv: 0.008007)\n",
      "[*] Epoch: [39/2000] time: 7.6494s, d_loss: 2.78604818 g_loss: 0.00770415\n",
      "Epoch [40/2000]    0 time: 1.9855s, d_loss: 0.45935136 g_loss: 0.01211352 (mse: 0.001857 adv: 0.010256)\n",
      "Epoch [40/2000]    1 time: 1.9512s, d_loss: 2.19547272 g_loss: 0.00456629 (mse: 0.001878 adv: 0.002689)\n",
      "Epoch [40/2000]    2 time: 1.9261s, d_loss: 0.00388820 g_loss: 0.01288406 (mse: 0.003599 adv: 0.009285)\n",
      "Epoch [40/2000]    3 time: 1.8447s, d_loss: 1.67437196 g_loss: 0.00415846 (mse: 0.000642 adv: 0.003516)\n",
      "[*] Epoch: [40/2000] time: 7.7078s, d_loss: 1.08327106 g_loss: 0.00843058\n",
      "[*] save images\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan.npz\n",
      "[TL] [*] Saved\n",
      "[TL] [*] Saving TL params into training_dir/d_srgan.npz\n",
      "[TL] [*] Saved\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(0, 40 + 1):\n",
    "    ## update learning rate\n",
    "    if epoch != 0 and (epoch % decay_every == 0):\n",
    "        new_lr_decay = lr_decay**(epoch // decay_every)\n",
    "        sess.run(tf.assign(lr_v, lr_init * new_lr_decay))\n",
    "        log = \" ** new learning rate: %f (for GAN)\" % (lr_init * new_lr_decay)\n",
    "        print(log)\n",
    "    elif epoch == 0:\n",
    "        sess.run(tf.assign(lr_v, lr_init))\n",
    "        log = \" ** init lr: %f  decay_every_init: %d, lr_decay: %f (for GAN)\" % (lr_init, decay_every, lr_decay)\n",
    "        print(log)\n",
    "\n",
    "    epoch_time = time.time()\n",
    "    total_d_loss, total_g_loss, n_iter = 0, 0, 0\n",
    "\n",
    "    for idx in range(0, len(train_data_list), batch_size):\n",
    "        step_time = time.time()\n",
    "        batch_file_name = train_data_list[idx: idx+batch_size]\n",
    "#         inputs_rgbs, label_raws = get_inputs_labels(p, train_data_dir, batch_file_name, crop_num)\n",
    "        inputs_rgbs, label_raws = get_inputs_labels(p, 'test_data', batch_file_name, crop_num)\n",
    "        ## update D\n",
    "        errD, _ = sess.run([d_loss, d_optim], {t_image: inputs_rgbs, t_target_image: label_raws})\n",
    "        ## update G\n",
    "        errG, errM, errA, _ = sess.run([g_loss, mse_loss, g_gan_loss, g_optim],\n",
    "                                             {t_image: inputs_rgbs, t_target_image: label_raws})\n",
    "        print(\"Epoch [%2d/%2d] %4d time: %4.4fs, d_loss: %.8f g_loss: %.8f (mse: %.6f adv: %.6f)\" %\n",
    "              (epoch, n_epoch, n_iter, time.time() - step_time, errD, errG, errM, errA))\n",
    "        total_d_loss += errD\n",
    "        total_g_loss += errG\n",
    "        n_iter += 1\n",
    "\n",
    "    log = \"[*] Epoch: [%2d/%2d] time: %4.4fs, d_loss: %.8f g_loss: %.8f\" % (epoch, n_epoch, time.time() - epoch_time, total_d_loss / n_iter,\n",
    "                                                                            total_g_loss / n_iter)\n",
    "    print(log)\n",
    "\n",
    "    # quick evaluation on train set\n",
    "    if (epoch != 0) and (epoch % 10 == 0):\n",
    "        sample_out = sess.run(net_g_test.outputs, {t_image_sample: rgb_sample})\n",
    "        sample_out = pack_raw_matrix(sample_out) * 200\n",
    "        \n",
    "        train_raw_sample = sess.run(out_image, feed_dict={t_raw_for_dark: sample_out})\n",
    "        train_raw_sample = np.minimum(np.maximum(train_raw_sample, 0), 1)\n",
    "        train_raw_sample = (train_raw_sample * 255).astype(np.uint8)\n",
    "        train_raw_sample_out_file_name = training_dir + os.path.sep + '/train_raw_sample_%d(GAN).png' % epoch\n",
    "        \n",
    "        print(\"[*] save images\")\n",
    "        tl.vis.save_images(train_raw_sample, [ni, ni], train_raw_sample_out_file_name)\n",
    "\n",
    "    ## save model\n",
    "    if (epoch != 0) and (epoch % 10 == 0):\n",
    "        tl.files.save_npz(net_g.all_params, name=training_dir + '/g_srgan.npz', sess=sess)\n",
    "        tl.files.save_npz(net_d.all_params, name=training_dir + '/d_srgan.npz', sess=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_out.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rgb_sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tl.vis.save_images(rgb_sample, [1, 1], 'test_png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rgb_sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}