LSTM EVAL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, MaxPooling2D, Dropout, Conv2D, Conv2DTranspose, add, Lambda, TimeDistributed, Bidirectional, ConvLSTM2D\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "from keras.optimizers import RMSprop, Adam, SGD\n",
    "from keras.losses import binary_crossentropy\n",
    "from losses import *\n",
    "import math\n",
    "\n",
    "from datahandler import DataHandler\n",
    "from models import *\n",
    "\n",
    "from generator import *\n",
    "from params import *\n",
    "from callbacks import getCallbacks\n",
    "from kfold_data_loader import *\n",
    "\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import skimage.io as io\n",
    "\n",
    "from keras.models import *\n",
    "from keras import backend as K\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import random\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from keras.models import *\n",
    "from keras import backend as K\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "from models.unet import *\n",
    "from models.unet_se import *\n",
    "\n",
    "from datahandler import DataHandler\n",
    "from kfold_data_loader import *\n",
    "from params import *\n",
    "import os\n",
    "import cv2\n",
    "import skimage.io as io\n",
    "from tqdm import tqdm\n",
    "\n",
    "from medpy.io import save\n",
    "\n",
    "from math import ceil, floor\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.metrics import f1_score, jaccard_similarity_score\n",
    "\n",
    "from scipy.ndimage import _ni_support\n",
    "from scipy.ndimage.morphology import distance_transform_edt, binary_erosion,\\\n",
    "     generate_binary_structure\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "lstm_graph = tf.get_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstmGenerator(images, batch_size, pre_model, pre_graph):\n",
    "    i=0\n",
    "    c=0\n",
    "    bs = batch_size\n",
    "    while True:\n",
    "        with pre_graph.as_default():\n",
    "            batch_features = []\n",
    "\n",
    "            j = i\n",
    "\n",
    "            if j == 0:\n",
    "                res1 =  np.expand_dims(np.zeros(images[j].shape), axis=0)\n",
    "            else:\n",
    "                img1 = np.expand_dims(images[j-1], axis=0)\n",
    "                res1 = pre_model.predict(img1)\n",
    "\n",
    "            img2 = np.expand_dims(images[j], axis=0)\n",
    "            res2 = pre_model.predict(img2)\n",
    "\n",
    "            if j == images.shape[0]-1:\n",
    "                res3 = np.expand_dims(np.zeros(images[j].shape), axis=0)\n",
    "            else:\n",
    "                img3 = np.expand_dims(images[j+1], axis=0)\n",
    "                res3 = pre_model.predict(img3)\n",
    "\n",
    "            res = np.concatenate((res1,res2,res3), axis=0)\n",
    "            \n",
    "            images[j] = res2[0]\n",
    "            \n",
    "            res[res>=0.5] = 1\n",
    "            res[res<0.5] = 0\n",
    "\n",
    "            batch_features.append(res)\n",
    "            \n",
    "            i += 1\n",
    "            yield np.array(batch_features)\n",
    "\n",
    "def lstmModel():\n",
    "\n",
    "    with lstm_graph.as_default():\n",
    "\n",
    "        inputs = Input((3, 256, 256, 1))\n",
    "\n",
    "        original = Lambda(lambda x : x[:,1,:,:,:] * 0.5)(inputs)\n",
    "\n",
    "        pool = TimeDistributed(MaxPooling2D(pool_size=2))(inputs)\n",
    "        bclstm = Bidirectional(ConvLSTM2D(64, 3, return_sequences = True,\n",
    "                                          padding='same', activation = 'relu'))(pool)\n",
    "        bclstm = Bidirectional(ConvLSTM2D(64, 3, padding='same', activation = 'relu'))(bclstm)\n",
    "\n",
    "        up = Conv2DTranspose(64,3, strides=2, padding='same', activation = 'relu')(bclstm)\n",
    "        drop = Dropout(0.5)(up)\n",
    "        outputs = Conv2D(1, (1,1), activation = 'sigmoid')(drop)\n",
    "\n",
    "        outputs = Lambda(lambda x : x * 0.5)(outputs)\n",
    "\n",
    "        outputs = add([outputs, original])\n",
    "\n",
    "        model = Model(inputs = inputs, outputs = outputs)\n",
    "\n",
    "        model.compile(optimizer = Adam(lr = 1e-4),\n",
    "                loss = binary_crossentropy, metrics = [dice_coef])\n",
    "\n",
    "        return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getDiceScore(ground_truth, prediction):\n",
    "    #convert to boolean values and flatten\n",
    "    ground_truth = np.asarray(ground_truth, dtype=np.bool).flatten()\n",
    "    prediction = np.asarray(prediction, dtype=np.bool).flatten()    \n",
    "    return f1_score(ground_truth, prediction)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    " def hd(result, reference, voxelspacing=None, connectivity=1):\n",
    "    hd1 = __surface_distances(result, reference, voxelspacing, connectivity).max()\n",
    "    hd2 = __surface_distances(reference, result, voxelspacing, connectivity).max()\n",
    "    hd = max(hd1, hd2)\n",
    "    return hd\n",
    "\n",
    "def hd95(result, reference, voxelspacing=None, connectivity=1):\n",
    "    hd1 = __surface_distances(result, reference, voxelspacing, connectivity)\n",
    "    hd2 = __surface_distances(reference, result, voxelspacing, connectivity)\n",
    "    hd95 = np.percentile(np.hstack((hd1, hd2)), 95)\n",
    "    return hd95\n",
    "\n",
    "def __surface_distances(result, reference, voxelspacing=None, connectivity=1):\n",
    "    result = np.atleast_1d(result.astype(np.bool))\n",
    "    reference = np.atleast_1d(reference.astype(np.bool))\n",
    "    if voxelspacing is not None:\n",
    "        voxelspacing = _ni_support._normalize_sequence(voxelspacing, result.ndim)\n",
    "        voxelspacing = np.asarray(voxelspacing, dtype=np.float64)\n",
    "        if not voxelspacing.flags.contiguous:\n",
    "            voxelspacing = voxelspacing.copy()\n",
    "\n",
    "    footprint = generate_binary_structure(result.ndim, connectivity)\n",
    "\n",
    "    if 0 == np.count_nonzero(result):\n",
    "        raise RuntimeError('The first supplied array does not contain any binary object.')\n",
    "    if 0 == np.count_nonzero(reference):\n",
    "        raise RuntimeError('The second supplied array does not contain any binary object.')\n",
    "\n",
    "    result_border = result ^ binary_erosion(result, structure=footprint, iterations=1)\n",
    "    reference_border = reference ^ binary_erosion(reference, structure=footprint, iterations=1)\n",
    "\n",
    "    dt = distance_transform_edt(~reference_border, sampling=voxelspacing)\n",
    "    sds = dt[result_border]\n",
    "\n",
    "    return sds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_files, mask_files = load_data_files('data/kfold_data/')\n",
    "\n",
    "skf = getKFolds(image_files, mask_files, n=10)\n",
    "\n",
    "kfold_indices = []\n",
    "for train_index, val_index in skf.split(image_files, mask_files):\n",
    "    kfold_indices.append({'train': train_index, 'val': val_index})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictMask(model, cur_graph, pre_model, pre_graph, image): \n",
    "    image_gen = lstmGenerator(image, 1, pre_model, pre_graph)\n",
    "    return model.predict_generator(image_gen, steps=len(image))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictAll(model, cur_graph, data, num_data=0):\n",
    "    dice_scores = []\n",
    "    hd_scores = []\n",
    "    hd95_scores = []\n",
    "    \n",
    "    pre_graph = tf.get_default_graph()\n",
    "    with pre_graph.as_default():\n",
    "        pre_model = getUnet()\n",
    "        print('loading pre weights %d'%i)\n",
    "        pre_model.load_weights('logs/unet/kfold_unet/kfold_unet_dice_DA_K%d/kfold_unet_dice_DA_K%d_weights.h5'%(i,i))\n",
    "\n",
    "\n",
    "    for image_file, mask_file in tqdm(data, total=num_data):\n",
    "        \n",
    "        fname = image_file[image_file.rindex('/')+1 : image_file.index('.')]\n",
    "        \n",
    "        image, hdr = dh.getImageData(image_file)\n",
    "        gt_mask, _ = dh.getImageData(mask_file, is_mask=True)\n",
    "\n",
    "        assert image.shape == gt_mask.shape\n",
    "        \n",
    "        if image.shape[1] != 256:\n",
    "            continue\n",
    "        \n",
    "        pred_mask = predictMask(model, cur_graph, pre_model, pre_graph, image)\n",
    "        pred_mask[pred_mask>=0.5] = 1\n",
    "        pred_mask[pred_mask<0.5] = 0\n",
    "            \n",
    "        dice_score = getDiceScore(gt_mask, pred_mask)\n",
    "        \n",
    "        if dice_score == 0:\n",
    "            continue\n",
    "            \n",
    "        dice_scores.append(dice_score)\n",
    "        \n",
    "        hd_score = hd(gt_mask, pred_mask)\n",
    "        hd_scores.append(hd_score)\n",
    "        \n",
    "        hd95_score = hd95(gt_mask, pred_mask)\n",
    "        hd95_scores.append(hd95_score)\n",
    "\n",
    "    return dice_scores, hd_scores, hd95_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K0/kfold_unet_BiCLSTM_K0_weights.h5\n",
      "loading pre weights 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30/30 [02:45<00:00,  5.12s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K0\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K1/kfold_unet_BiCLSTM_K1_weights.h5\n",
      "loading pre weights 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:51<00:00,  5.67s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K1\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K2/kfold_unet_BiCLSTM_K2_weights.h5\n",
      "loading pre weights 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:33<00:00,  4.86s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K2\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K3/kfold_unet_BiCLSTM_K3_weights.h5\n",
      "loading pre weights 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:36<00:00,  5.03s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K3\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K4/kfold_unet_BiCLSTM_K4_weights.h5\n",
      "loading pre weights 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:41<00:00,  5.35s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K4\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K5/kfold_unet_BiCLSTM_K5_weights.h5\n",
      "loading pre weights 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:30<00:00,  5.47s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K5\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K6/kfold_unet_BiCLSTM_K6_weights.h5\n",
      "loading pre weights 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:28<00:00,  5.14s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K6\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K7/kfold_unet_BiCLSTM_K7_weights.h5\n",
      "loading pre weights 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:48<00:00,  5.55s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K7\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K8/kfold_unet_BiCLSTM_K8_weights.h5\n",
      "loading pre weights 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:53<00:00,  7.25s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K8\n",
      "loading weights from ./logs/unet_LSTM/kfold_unet_LSTM/kfold_unet_BiCLSTM_K9/kfold_unet_BiCLSTM_K9_weights.h5\n",
      "loading pre weights 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [02:31<00:00,  4.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished K9\n",
      "dice\n",
      "0.9518160316025411\n",
      "0.9533490128229389\n",
      "0.891307393079965\n",
      "0.9371863598645996\n",
      "0.9154275906758437\n",
      "0.9677843470966389\n",
      "0.978547729604772\n",
      "0.9456817147705893\n",
      "0.9651825980455755\n",
      "0.9660379010022321\n",
      "0.9630109667865719\n",
      "0.9566243700812466\n",
      "0.964672231908278\n",
      "0.9734761762158228\n",
      "0.9475582577416328\n",
      "0.9228104876014602\n",
      "0.925295930712243\n",
      "0.9005662850218848\n",
      "0.009566161642277264\n",
      "0.9702943740360758\n",
      "0.6413960090172831\n",
      "0.9747516880602862\n",
      "0.9762035397205728\n",
      "0.9720842364490473\n",
      "0.9462630098756375\n",
      "0.848376581709915\n",
      "0.9079392269784149\n",
      "0.7046877291933812\n",
      "0.760500956921734\n",
      "0.9055784624278139\n",
      "0.9376427013039726\n",
      "0.9681657537255318\n",
      "0.9346970225983939\n",
      "0.963741754508996\n",
      "0.9604681623555186\n",
      "0.9720424270263498\n",
      "0.928584777834883\n",
      "0.9662158947077913\n",
      "0.7178121014595438\n",
      "0.968710457139947\n",
      "0.9581199685490662\n",
      "0.9578522968312311\n",
      "0.9415694804957409\n",
      "0.9016611440530171\n",
      "0.950492256787177\n",
      "0.8947131386139701\n",
      "0.9570458103473982\n",
      "0.9601591766851875\n",
      "0.9638381595173646\n",
      "0.953191323193471\n",
      "0.9602110608454011\n",
      "0.9545330013936703\n",
      "0.957139720769875\n",
      "0.9451528030763005\n",
      "0.8909327529828629\n",
      "0.9260365951341148\n",
      "0.9064969186484554\n",
      "0.9501407655509694\n",
      "0.9589546298752986\n",
      "0.9500141123341801\n",
      "0.9643652429696222\n",
      "0.961532413958952\n",
      "0.9360790300607239\n",
      "0.9684962131581348\n",
      "0.9634076505336464\n",
      "0.9720479806316716\n",
      "0.9126367992634289\n",
      "0.8453900709219859\n",
      "0.9691241594226668\n",
      "0.9281435655717581\n",
      "0.9587022685199711\n",
      "0.9692548812819498\n",
      "0.9439833662397569\n",
      "0.5197288493705432\n",
      "0.8555348697866512\n",
      "0.9681052122232613\n",
      "0.9496112183812072\n",
      "0.978501007562133\n",
      "0.9698735026869804\n",
      "0.9638435305210656\n",
      "0.9229443475821157\n",
      "0.9435793827743475\n",
      "0.9489379253454322\n",
      "0.925619675044918\n",
      "0.9335320537894012\n",
      "0.9484393097698987\n",
      "0.9519190799501178\n",
      "0.9216067741526719\n",
      "0.9228214498358542\n",
      "0.9535349556899705\n",
      "0.9536744685474696\n",
      "0.9467493069361417\n",
      "0.971488928064029\n",
      "0.9682876142975895\n",
      "0.7356193293529825\n",
      "0.9689841637697953\n",
      "0.9718775176948323\n",
      "0.9538464414923951\n",
      "0.48653210857160617\n",
      "0.9134278917584783\n",
      "0.8452933775591902\n",
      "0.9573113453710469\n",
      "0.9651806584075934\n",
      "0.8716253262276027\n",
      "0.9158448725154565\n",
      "0.7459013797544524\n",
      "0.8814096756658815\n",
      "0.8073592670527551\n",
      "0.8514797102135518\n",
      "0.8627216099664591\n",
      "0.8990565867561375\n",
      "0.9151342039782682\n",
      "0.9442873407029271\n",
      "0.9256849848446728\n",
      "0.9270118863791432\n",
      "0.9090120574818495\n",
      "0.9453271977778589\n",
      "0.934864520603672\n",
      "0.965945163867754\n",
      "0.9035964334869248\n",
      "0.9123131939039888\n",
      "0.9228883559818811\n",
      "0.9409710788080267\n",
      "0.9648803693857562\n",
      "0.9414458053028375\n",
      "0.921022574753541\n",
      "0.9678208898586472\n",
      "0.9631242950723122\n",
      "0.8444638440273534\n",
      "0.8075534373194685\n",
      "0.9597814572262311\n",
      "0.9593369739355141\n",
      "0.9506538028889935\n",
      "0.9494016426673074\n",
      "0.9566767052189792\n",
      "0.9592254640171682\n",
      "0.9335782844904655\n",
      "0.877871066992201\n",
      "0.8885087305332703\n",
      "0.8725733032985316\n",
      "0.9097759853993111\n",
      "0.9473361144896729\n",
      "0.8845315070589587\n",
      "0.937401677286463\n",
      "0.921475658065799\n",
      "0.6786029186835333\n",
      "0.8874354757445837\n",
      "0.9718475827271851\n",
      "0.9541368408012453\n",
      "0.9451981724993622\n",
      "0.9574328749181401\n",
      "0.963075152725365\n",
      "0.9525021756280855\n",
      "0.891539892743789\n",
      "0.9512797122153565\n",
      "0.9642337405303819\n",
      "0.931470090409941\n",
      "0.9785332897461044\n",
      "0.8360590347028322\n",
      "0.963697542908165\n",
      "0.9728088299746872\n",
      "0.9592510242754031\n",
      "0.9735294621886197\n",
      "0.9719156792831553\n",
      "0.939378302877173\n",
      "0.9166175805810878\n",
      "0.9350501775513355\n",
      "0.9356109612671681\n",
      "0.9332442523089015\n",
      "0.9527423047421641\n",
      "0.9614413110624243\n",
      "0.9371747708852173\n",
      "0.9546808066032807\n",
      "0.9572315860007216\n",
      "0.9610533815732251\n",
      "0.9623330385286982\n",
      "0.963752303365338\n",
      "0.9362724555080603\n",
      "0.9509448368009162\n",
      "0.9496352965442239\n",
      "0.9652684283237859\n",
      "0.953946723320804\n",
      "0.9724003375971414\n",
      "0.9197977743209286\n",
      "0.7679362145310582\n",
      "0.8915961516170163\n",
      "0.9431461749713059\n",
      "0.9393473866589662\n",
      "0.9218544830140399\n",
      "0.8805454121668868\n",
      "0.9670212529703079\n",
      "0.8862566959502893\n",
      "0.9472800762733486\n",
      "0.9409655355601302\n",
      "0.9145449520381268\n",
      "0.8570644674515626\n",
      "0.9280775589959417\n",
      "0.9435735038202717\n",
      "0.9220207411001273\n",
      "0.9228533830463866\n",
      "0.9237834334865087\n",
      "0.9500080364021305\n",
      "0.934068826237116\n",
      "0.9346577852402459\n",
      "0.9614757184088343\n",
      "0.969054687562181\n",
      "0.9612223636995023\n",
      "0.948062321364988\n",
      "0.9347817580360179\n",
      "0.9183208007569343\n",
      "0.9598495405956882\n",
      "0.9331667268858646\n",
      "0.8612989633758154\n",
      "0.9551304250161452\n",
      "0.9303906699440406\n",
      "0.954440405032649\n",
      "0.9048320326541971\n",
      "0.97318730680621\n",
      "0.9776945737115776\n",
      "0.970056152818664\n",
      "0.9699813867108611\n",
      "0.6005870330244495\n",
      "0.9371955012826517\n",
      "0.9486319414992739\n",
      "0.9004590406358791\n",
      "0.8934953244107935\n",
      "0.922266918044466\n",
      "0.8987345950704225\n",
      "0.9509795522988014\n",
      "0.9106025482369755\n",
      "0.954215621955826\n",
      "0.9667613843351548\n",
      "0.9474514621573444\n",
      "0.9131290112961661\n",
      "0.9602690075148005\n",
      "0.943106880771705\n",
      "0.9660617707618394\n",
      "0.9678848636501284\n",
      "0.7413445117217823\n",
      "0.9189242455347977\n",
      "0.9637116222150236\n",
      "0.9628149286873346\n",
      "0.8813258051024676\n",
      "0.9668142793105319\n",
      "0.9398695560698506\n",
      "0.9554365509761388\n",
      "0.9361788775549782\n",
      "0.9136843315984703\n",
      "0.8738737543291623\n",
      "0.9493932120505171\n",
      "0.948975195420693\n",
      "0.8926126474569716\n",
      "0.7023163729954464\n",
      "0.9492821463732839\n",
      "0.9528823014657624\n",
      "0.9624785044655239\n",
      "0.9237733152006672\n",
      "0.882336188131904\n",
      "0.9596885484585682\n",
      "0.9643499674632846\n",
      "0.9273471292424216\n",
      "0.948227946021832\n",
      "0.884298491105153\n",
      "0.9668673390837494\n",
      "0.8951095267315353\n",
      "0.9696697587531168\n",
      "0.9502937009503494\n",
      "0.9299928553416827\n",
      "0.9013515612863134\n",
      "0.32727916445388566\n",
      "0.9359229586469208\n",
      "0.8692657157950343\n",
      "0.9775663596654747\n",
      "0.9458133863246319\n",
      "0.9796060871880141\n",
      "0.9449273468612002\n",
      "0.9499159643973107\n",
      "0.9125203141062007\n",
      "0.9666548326178209\n",
      "0.9113840769903762\n",
      "0.9327524054047045\n",
      "0.9488195067611473\n",
      "0.8769216079442375\n",
      "0.9245051821848568\n",
      "0.8964510265502547\n",
      "\n",
      "hd\n",
      "7.211102550927978\n",
      "17.88854381999832\n",
      "150.07664708408169\n",
      "143.68716017793656\n",
      "197.73972792537162\n",
      "94.03722667114339\n",
      "4.242640687119285\n",
      "20.074859899884732\n",
      "149.06709898565813\n",
      "5.830951894845301\n",
      "120.31209415515964\n",
      "13.74772708486752\n",
      "7.0710678118654755\n",
      "182.56231812726304\n",
      "142.18649724921138\n",
      "9.433981132056603\n",
      "139.53494186045302\n",
      "127.94139283281233\n",
      "227.0990973121646\n",
      "13.152946437965905\n",
      "87.49857141690943\n",
      "7.0\n",
      "8.0\n",
      "69.77105417004964\n",
      "97.63708311906906\n",
      "16.55294535724685\n",
      "111.86152153444007\n",
      "194.74342094150447\n",
      "132.47263868437136\n",
      "12.727922061357855\n",
      "68.01470429252781\n",
      "55.5517776493246\n",
      "7.14142842854285\n",
      "8.246211251235321\n",
      "4.0\n",
      "4.58257569495584\n",
      "98.37682653958706\n",
      "110.18620603324175\n",
      "116.55899793666725\n",
      "13.601470508735444\n",
      "71.01408311032397\n",
      "51.487862647424\n",
      "89.58794561770016\n",
      "163.33401360402553\n",
      "72.61542535852834\n",
      "160.51168181786645\n",
      "8.06225774829855\n",
      "7.615773105863909\n",
      "98.19877799647\n",
      "10.44030650891055\n",
      "62.12889826803627\n",
      "77.78174593052023\n",
      "81.39410298049853\n",
      "6.4031242374328485\n",
      "79.8373346248483\n",
      "79.08223567907018\n",
      "137.29166034395533\n",
      "5.830951894845301\n",
      "91.18662182579197\n",
      "155.89098755219945\n",
      "8.06225774829855\n",
      "107.2846680565308\n",
      "98.8635423197045\n",
      "51.86520991955976\n",
      "55.018178813915675\n",
      "74.91995728776145\n",
      "192.52792005317048\n",
      "130.49904214207857\n",
      "68.21290200541243\n",
      "20.223748416156685\n",
      "6.164414002968976\n",
      "71.13367697511495\n",
      "98.37174391053561\n",
      "154.18819669481837\n",
      "72.56721022610694\n",
      "5.196152422706632\n",
      "4.0\n",
      "5.744562646538029\n",
      "4.69041575982343\n",
      "118.10588469674151\n",
      "146.85026387446499\n",
      "7.681145747868608\n",
      "121.65936051122412\n",
      "140.92196422133776\n",
      "116.84605256490268\n",
      "3.1622776601683795\n",
      "133.5402561027947\n",
      "28.792360097775937\n",
      "7.54983443527075\n",
      "11.445523142259598\n",
      "134.08206442324789\n",
      "33.075670817082454\n",
      "8.246211251235321\n",
      "14.966629547095765\n",
      "148.19581640518737\n",
      "89.3420393767682\n",
      "6.6332495807108\n",
      "113.52973178863765\n",
      "199.93248860552904\n",
      "90.81850031794183\n",
      "143.03845636750978\n",
      "58.215118311311535\n",
      "8.48528137423857\n",
      "134.33167906342868\n",
      "60.149812967290266\n",
      "142.38679714074615\n",
      "93.8296328459192\n",
      "126.92123541787639\n",
      "8.54400374531753\n",
      "141.45317246354003\n",
      "196.33135256499406\n",
      "146.3318147225681\n",
      "4.358898943540674\n",
      "58.66856057549052\n",
      "47.3392015141785\n",
      "31.400636936215164\n",
      "5.830951894845301\n",
      "43.78355855797927\n",
      "15.684387141358123\n",
      "26.43860813280457\n",
      "137.64083696345355\n",
      "5.0\n",
      "6.0\n",
      "74.63913182774837\n",
      "176.90958142508845\n",
      "33.436506994600975\n",
      "64.4437739428721\n",
      "94.5568612000208\n",
      "34.713109915419565\n",
      "68.99275324264136\n",
      "17.916472867168917\n",
      "5.916079783099616\n",
      "29.68164415931166\n",
      "103.9759587597056\n",
      "11.357816691600547\n",
      "5.0\n",
      "87.14929718592113\n",
      "103.67738422626219\n",
      "11.489125293076057\n",
      "128.191263352851\n",
      "87.96590248499699\n",
      "114.04385121522334\n",
      "107.8563859954523\n",
      "17.60681686165901\n",
      "35.35533905932738\n",
      "53.45091205957107\n",
      "95.13148795220224\n",
      "64.07807737440318\n",
      "67.68308503607086\n",
      "122.46223907801131\n",
      "76.4591394144611\n",
      "47.45524207081869\n",
      "79.73079706111058\n",
      "136.3451502621197\n",
      "33.27160951922825\n",
      "56.83308895353129\n",
      "46.52956049652737\n",
      "3.0\n",
      "59.62382074305537\n",
      "20.396078054371138\n",
      "59.481089431852205\n",
      "7.280109889280518\n",
      "46.78675026115834\n",
      "13.076696830622021\n",
      "63.06346010171025\n",
      "81.93289937503738\n",
      "9.055385138137417\n",
      "54.59853477887479\n",
      "147.27185746095552\n",
      "111.6870628139177\n",
      "48.85693400122443\n",
      "55.5517776493246\n",
      "136.14697940094007\n",
      "84.04760555780278\n",
      "96.33794683301072\n",
      "96.23408959407264\n",
      "34.14674215792775\n",
      "53.40411969127476\n",
      "110.95945205344158\n",
      "120.124934963562\n",
      "142.10559454152394\n",
      "4.69041575982343\n",
      "52.1344415909483\n",
      "103.14068062602651\n",
      "163.49923547221866\n",
      "127.41271522104849\n",
      "131.62066707018317\n",
      "33.63034344160047\n",
      "124.43873994861889\n",
      "98.5393322486001\n",
      "118.06777714516353\n",
      "43.3589667773576\n",
      "15.033296378372908\n",
      "92.2062904578641\n",
      "10.198039027185569\n",
      "7.0710678118654755\n",
      "5.830951894845301\n",
      "13.92838827718412\n",
      "119.54915307102765\n",
      "19.339079605813716\n",
      "28.61817604250837\n",
      "58.52349955359813\n",
      "64.32728814430156\n",
      "95.90099061010788\n",
      "4.58257569495584\n",
      "100.772019926168\n",
      "12.083045973594572\n",
      "111.83022847155415\n",
      "76.97402159170326\n",
      "169.1715106038839\n",
      "83.02409288875127\n",
      "88.66228059327146\n",
      "41.30375285612676\n",
      "6.0\n",
      "42.720018726587654\n",
      "92.74157643689264\n",
      "136.47710430691296\n",
      "5.916079783099616\n",
      "6.4031242374328485\n",
      "129.47200469599596\n",
      "3.0\n",
      "13.74772708486752\n",
      "8.06225774829855\n",
      "131.4610208388783\n",
      "89.3364427319557\n",
      "120.68968472906042\n",
      "196.85781671043696\n",
      "102.79105019407089\n",
      "112.05802068571441\n",
      "119.03780911962384\n",
      "86.31917515824627\n",
      "8.12403840463596\n",
      "5.830951894845301\n",
      "53.9351462406472\n",
      "6.164414002968976\n",
      "12.449899597988733\n",
      "5.477225575051661\n",
      "18.35755975068582\n",
      "182.67183690979843\n",
      "122.07374820165063\n",
      "9.0\n",
      "23.53720459187964\n",
      "32.57299494980466\n",
      "31.32091952673165\n",
      "11.661903789690601\n",
      "54.120236510939236\n",
      "40.755367744629666\n",
      "43.977266854592045\n",
      "96.88137075826292\n",
      "25.238858928247925\n",
      "5.656854249492381\n",
      "138.01811475310043\n",
      "107.8424777163433\n",
      "106.48474069086143\n",
      "96.4157663455516\n",
      "9.0\n",
      "96.32237538599222\n",
      "116.93587986584785\n",
      "95.54580053565934\n",
      "7.0710678118654755\n",
      "29.631064780058107\n",
      "40.36087214122113\n",
      "58.957611891934704\n",
      "107.77754868245984\n",
      "42.0\n",
      "56.089214649520635\n",
      "40.0374824383352\n",
      "87.3040663428686\n",
      "96.27564593395361\n",
      "131.5066538240556\n",
      "53.009433122794285\n",
      "36.578682316343766\n",
      "104.66613587975817\n",
      "2.8284271247461903\n",
      "2.449489742783178\n",
      "8.06225774829855\n",
      "7.0\n",
      "107.17275773255068\n",
      "80.73413156775763\n",
      "7.211102550927978\n",
      "84.85870609430714\n",
      "55.72252686302013\n",
      "98.92421341612983\n",
      "106.94391053257777\n",
      "90.61456836513652\n",
      "\n",
      "hd95\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "194.85892332659543\n",
      "0.0\n",
      "53.40411640846444\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "3.0\n",
      "1.0\n",
      "170.496627530359\n",
      "86.33075929238663\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "58.056437900048536\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.4142135623730951\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.7320508075688772\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "2.23606797749979\n",
      "2.23606797749979\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "135.86390249069103\n",
      "2.8284271247461903\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "79.36088453979872\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "171.92149363771955\n",
      "1.0\n",
      "3.3166247903554\n",
      "0.0\n",
      "0.0\n",
      "2.23606797749979\n",
      "1.0\n",
      "109.93634521849451\n",
      "2.0\n",
      "82.49242389456137\n",
      "1.0\n",
      "2.0\n",
      "85.17041739947034\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.4142135623730951\n",
      "26.27926774296469\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "46.79743582719036\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "2.0\n",
      "1.0\n",
      "1.4142135623730951\n",
      "1.0\n",
      "1.0\n",
      "2.449489742783178\n",
      "1.0\n",
      "1.0\n",
      "4.0\n",
      "2.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "3.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "2.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "3.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.7320508075688772\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "2.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "2.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "6.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "2.8284271247461903\n",
      "1.0\n",
      "1.4142135623730951\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "2.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "4.123105625617661\n",
      "1.0\n",
      "1.0\n",
      "3.605551275463989\n",
      "91.53141537199127\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "2.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "2.0\n",
      "0.0\n",
      "1.4142135623730951\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "2.8284271247461903\n",
      "12.68857754044952\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "\n",
      "Final results for unet\n",
      "dice 0.919547\n",
      "hd 69.215383\n",
      "hd95 5.618991\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#Get data and generators\n",
    "\n",
    "unet_type = 'unet'\n",
    "dh = DataHandler()\n",
    "all_dice = []\n",
    "all_hd = []\n",
    "all_hd95 = []\n",
    "\n",
    "for i in range(10):\n",
    "    exp_name = 'kfold_%s_BiCLSTM_K%d'%(unet_type, i)\n",
    "\n",
    "    #get parameters\n",
    "    params = getParams(exp_name, unet_type=unet_type, is_lstm = True)\n",
    "    \n",
    "    val_img_files = np.take(image_files, kfold_indices[i]['val'])\n",
    "    val_mask_files = np.take(mask_files, kfold_indices[i]['val'])\n",
    "    \n",
    "    \n",
    "    \n",
    "    with lstm_graph.as_default():\n",
    "        model = lstmModel()\n",
    "        print('loading weights from %s'%params['checkpoint']['name'])\n",
    "        model.load_weights(params['checkpoint']['name'])\n",
    "        \n",
    "    data = zip(val_img_files, val_mask_files)\n",
    "    dice_score, hd_score, hd95_score = predictAll(model, lstm_graph, data, num_data=len(val_mask_files))\n",
    "    \n",
    "    print('Finished K%d'%i)\n",
    "    \n",
    "    all_dice += dice_score\n",
    "    all_hd += hd_score\n",
    "    all_hd95 += hd95_score\n",
    "\n",
    "print('dice')\n",
    "for i in range(len(all_dice)):\n",
    "    print(all_dice[i])\n",
    "print()\n",
    "\n",
    "print('hd')\n",
    "for i in range(len(all_hd)):\n",
    "    print(all_hd[i])\n",
    "print()\n",
    "\n",
    "print('hd95')\n",
    "for i in range(len(all_hd95)):\n",
    "    print(all_hd95[i])\n",
    "print()\n",
    "    \n",
    "print('Final results for %s'%unet_type)\n",
    "print('dice %f'%np.mean(all_dice))\n",
    "print('hd %f'%np.mean(all_hd))\n",
    "print('hd95 %f'%np.mean(all_hd95))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}